gavindiaz/manifoldbt
1
0
Fork 0
mirror of https://github.com/manifoldbt/manifoldbt.git synced 2026-08-24 14:38:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: add bt.ingest() and manifoldbt CLI for data ingestion

Browse Source 
- Add ingest() function for downloading bar data from providers
- Add manifoldbt CLI entry point (manifoldbt ingest ...)
- Supported providers: binance, hyperliquid (free), databento, massive (Pro)
- Update type stubs for py_ingest
This commit is contained in:
Jimmy7892
2026-03-18 02:53:28 +01:00
parent b71461c9a7
commit b917d7975d
4 changed files with 156 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pyproject.toml
+3 -1
View File
@@ -9,7 +9,6 @@ classifiers = [
"Development Status :: 4 - Beta",
"Intended Audience :: Financial and Insurance Industry",
"Intended Audience :: Science/Research",
"License :: OSI Approved :: MIT License",
"Programming Language :: Python :: 3",
"Programming Language :: Rust",
"Topic :: Office/Business :: Financial :: Investment",
@@ -24,6 +23,9 @@ plotly = ["plotly>=5.0"]
all = ["matplotlib>=3.7", "plotly>=5.0", "pandas>=1.5", "polars>=0.20"]
dev = ["pytest>=7.0", "polars>=0.20", "pandas>=1.5", "pyarrow>=14.0", "matplotlib>=3.7"]
[project.scripts]
manifoldbt = "manifoldbt.cli:main"
[project.urls]
Homepage = "https://manifold-bt.com"
Repository = "https://github.com/manifoldbt/manifoldbt"
python/manifoldbt/__init__.py
+63 -1
View File
@@ -25,6 +25,7 @@ from manifoldbt._native import (
py_replay as _replay_native,
py_run_monte_carlo,
run_portfolio as _run_portfolio_native,
py_ingest as _ingest_native,
)
from manifoldbt._serde import scalar_value_to_json
from manifoldbt.config import (
@@ -119,8 +120,10 @@ def _is_pro() -> bool:
def _require_pro(feature: str) -> None:
"""Warn and raise if not Pro. Use _gate_pro for graceful skip."""
if _is_pro():
return
_warn_pro(feature)
raise LicenseError(f"{feature} — Pro license required")
raise LicenseError(f"{feature} -- Pro license required")
def _gate_pro(feature: str) -> bool:
@@ -296,6 +299,63 @@ def _cap_output_resolution(config: BacktestConfig) -> BacktestConfig:
return config
# ---------------------------------------------------------------------------
# Data Ingestion
# ---------------------------------------------------------------------------
def ingest(
provider: str,
symbol: str,
symbol_id: int,
start: str,
end: str,
*,
interval: str = "1m",
dataset: Optional[str] = None,
data_root: str = "data",
metadata_db: str = "metadata/metadata.sqlite",
exchange: Optional[str] = None,
asset_class: str = "crypto_spot",
) -> DataStore:
"""Ingest bars from a data provider into the local Parquet store.
Providers: ``"binance"``, ``"hyperliquid"`` (free), ``"databento"``, ``"massive"`` (Pro).
Returns a :class:`DataStore` ready for :func:`run`.
Example::
store = bt.ingest(
provider="databento",
symbol="ESH5",
symbol_id=1,
start="2025-01-01T00:00:00Z",
end="2025-01-31T00:00:00Z",
dataset="GLBX.MDP3",
exchange="CME",
asset_class="future",
)
result = bt.run(strategy, config, store)
"""
_PRO_PROVIDERS = {"databento", "massive"}
if provider in _PRO_PROVIDERS:
_require_pro(f"Data connector: {provider}")
return _ingest_native(
provider=provider,
symbol=symbol,
symbol_id=symbol_id,
start=start,
end=end,
interval=interval,
dataset=dataset,
data_root=data_root,
metadata_db=metadata_db,
exchange=exchange,
asset_class=asset_class,
)
# ---------------------------------------------------------------------------
# Core API
# ---------------------------------------------------------------------------
@@ -669,6 +729,8 @@ __all__ = [
"DataStore",
"Result",
"SweepResult",
# Data ingestion
"ingest",
# Run functions
"run",
"run_sweep",
python/manifoldbt/_native.pyi
+13
View File
@@ -121,3 +121,16 @@ def py_run_monte_carlo(
result: BacktestResult,
mc_config_json: str,
) -> Dict[str, Any]: ...
def py_ingest(
provider: str,
symbol: str,
symbol_id: int,
start: str,
end: str,
interval: str = "1m",
dataset: Optional[str] = None,
data_root: str = "data",
metadata_db: str = "metadata/metadata.sqlite",
exchange: Optional[str] = None,
asset_class: str = "crypto_spot",
) -> DataStore: ...
python/manifoldbt/cli.py
+77
View File
@@ -0,0 +1,77 @@
"""manifoldbt CLI — data ingestion from the command line.
Usage:
manifoldbt ingest --provider binance --symbol BTCUSDT --symbol-id 1 \
--start 2025-01-01T00:00:00Z --end 2025-01-31T00:00:00Z
manifoldbt ingest --provider databento --dataset GLBX.MDP3 --symbol ESH5 \
--symbol-id 1 --start 2025-01-01T00:00:00Z --end 2025-01-31T00:00:00Z \
--exchange CME --asset-class future
"""
import argparse
import sys
def main() -> None:
parser = argparse.ArgumentParser(prog="manifoldbt", description="ManifoldBT CLI")
sub = parser.add_subparsers(dest="command")
# ── ingest ────────────────────────────────────────────────────────────
ing = sub.add_parser("ingest", help="Ingest bars from a data provider")
ing.add_argument("--provider", required=True, help="binance | hyperliquid | databento")
ing.add_argument("--symbol", required=True, help="e.g. BTCUSDT, ESH5")
ing.add_argument("--symbol-id", required=True, type=int, help="Unique integer ID for this symbol")
ing.add_argument("--start", required=True, help="RFC3339 start (e.g. 2025-01-01T00:00:00Z)")
ing.add_argument("--end", required=True, help="RFC3339 end")
ing.add_argument("--interval", default="1m", help="Bar interval: 1s, 1m, 5m, 15m, 1h, 1d (default: 1m)")
ing.add_argument("--dataset", default=None, help="Databento dataset (e.g. GLBX.MDP3)")
ing.add_argument("--data-root", default="data", help="Parquet store directory (default: data)")
ing.add_argument("--metadata-db", default="metadata/metadata.sqlite", help="Metadata SQLite path")
ing.add_argument("--exchange", default=None, help="Exchange name for metadata")
ing.add_argument("--asset-class", default="crypto_spot",
help="crypto_spot | crypto_perp | equity | future | option | forex | index")
ing.add_argument("--license-key", default=None, help="Pro license key (or set via bt.activate())")
args = parser.parse_args()
if args.command is None:
parser.print_help()
sys.exit(1)
if args.command == "ingest":
_cmd_ingest(args)
def _cmd_ingest(args: argparse.Namespace) -> None:
import manifoldbt as bt
if args.license_key:
try:
bt.activate(args.license_key)
except Exception as e:
print(f"License activation failed: {e}", file=sys.stderr)
sys.exit(1)
from manifoldbt.exceptions import LicenseError
try:
store = bt.ingest(
provider=args.provider,
symbol=args.symbol,
symbol_id=args.symbol_id,
start=args.start,
end=args.end,
interval=args.interval,
dataset=args.dataset,
data_root=args.data_root,
metadata_db=args.metadata_db,
exchange=args.exchange,
asset_class=args.asset_class,
)
print(f"Ingested to {store.data_root()} (symbols: {store.list_symbols()})")
except LicenseError:
# Message already handled by _warn_pro atexit summary
sys.exit(1)
except Exception as e:
print(f"Error: {e}", file=sys.stderr)
sys.exit(1)