diff --git a/pyproject.toml b/pyproject.toml index af4a31f..f117a7e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,6 @@ classifiers = [ "Development Status :: 4 - Beta", "Intended Audience :: Financial and Insurance Industry", "Intended Audience :: Science/Research", - "License :: OSI Approved :: MIT License", "Programming Language :: Python :: 3", "Programming Language :: Rust", "Topic :: Office/Business :: Financial :: Investment", @@ -24,6 +23,9 @@ plotly = ["plotly>=5.0"] all = ["matplotlib>=3.7", "plotly>=5.0", "pandas>=1.5", "polars>=0.20"] dev = ["pytest>=7.0", "polars>=0.20", "pandas>=1.5", "pyarrow>=14.0", "matplotlib>=3.7"] +[project.scripts] +manifoldbt = "manifoldbt.cli:main" + [project.urls] Homepage = "https://manifold-bt.com" Repository = "https://github.com/manifoldbt/manifoldbt" diff --git a/python/manifoldbt/__init__.py b/python/manifoldbt/__init__.py index 514f962..2daf11a 100644 --- a/python/manifoldbt/__init__.py +++ b/python/manifoldbt/__init__.py @@ -25,6 +25,7 @@ from manifoldbt._native import ( py_replay as _replay_native, py_run_monte_carlo, run_portfolio as _run_portfolio_native, + py_ingest as _ingest_native, ) from manifoldbt._serde import scalar_value_to_json from manifoldbt.config import ( @@ -119,8 +120,10 @@ def _is_pro() -> bool: def _require_pro(feature: str) -> None: """Warn and raise if not Pro. Use _gate_pro for graceful skip.""" + if _is_pro(): + return _warn_pro(feature) - raise LicenseError(f"{feature} — Pro license required") + raise LicenseError(f"{feature} -- Pro license required") def _gate_pro(feature: str) -> bool: @@ -296,6 +299,63 @@ def _cap_output_resolution(config: BacktestConfig) -> BacktestConfig: return config +# --------------------------------------------------------------------------- +# Data Ingestion +# --------------------------------------------------------------------------- + +def ingest( + provider: str, + symbol: str, + symbol_id: int, + start: str, + end: str, + *, + interval: str = "1m", + dataset: Optional[str] = None, + data_root: str = "data", + metadata_db: str = "metadata/metadata.sqlite", + exchange: Optional[str] = None, + asset_class: str = "crypto_spot", +) -> DataStore: + """Ingest bars from a data provider into the local Parquet store. + + Providers: ``"binance"``, ``"hyperliquid"`` (free), ``"databento"``, ``"massive"`` (Pro). + + Returns a :class:`DataStore` ready for :func:`run`. + + Example:: + + store = bt.ingest( + provider="databento", + symbol="ESH5", + symbol_id=1, + start="2025-01-01T00:00:00Z", + end="2025-01-31T00:00:00Z", + dataset="GLBX.MDP3", + exchange="CME", + asset_class="future", + ) + result = bt.run(strategy, config, store) + """ + _PRO_PROVIDERS = {"databento", "massive"} + if provider in _PRO_PROVIDERS: + _require_pro(f"Data connector: {provider}") + + return _ingest_native( + provider=provider, + symbol=symbol, + symbol_id=symbol_id, + start=start, + end=end, + interval=interval, + dataset=dataset, + data_root=data_root, + metadata_db=metadata_db, + exchange=exchange, + asset_class=asset_class, + ) + + # --------------------------------------------------------------------------- # Core API # --------------------------------------------------------------------------- @@ -669,6 +729,8 @@ __all__ = [ "DataStore", "Result", "SweepResult", + # Data ingestion + "ingest", # Run functions "run", "run_sweep", diff --git a/python/manifoldbt/_native.pyi b/python/manifoldbt/_native.pyi index c5bd214..c6fbac0 100644 --- a/python/manifoldbt/_native.pyi +++ b/python/manifoldbt/_native.pyi @@ -121,3 +121,16 @@ def py_run_monte_carlo( result: BacktestResult, mc_config_json: str, ) -> Dict[str, Any]: ... +def py_ingest( + provider: str, + symbol: str, + symbol_id: int, + start: str, + end: str, + interval: str = "1m", + dataset: Optional[str] = None, + data_root: str = "data", + metadata_db: str = "metadata/metadata.sqlite", + exchange: Optional[str] = None, + asset_class: str = "crypto_spot", +) -> DataStore: ... diff --git a/python/manifoldbt/cli.py b/python/manifoldbt/cli.py new file mode 100644 index 0000000..e74baba --- /dev/null +++ b/python/manifoldbt/cli.py @@ -0,0 +1,77 @@ +"""manifoldbt CLI — data ingestion from the command line. + +Usage: + manifoldbt ingest --provider binance --symbol BTCUSDT --symbol-id 1 \ + --start 2025-01-01T00:00:00Z --end 2025-01-31T00:00:00Z + + manifoldbt ingest --provider databento --dataset GLBX.MDP3 --symbol ESH5 \ + --symbol-id 1 --start 2025-01-01T00:00:00Z --end 2025-01-31T00:00:00Z \ + --exchange CME --asset-class future +""" +import argparse +import sys + + +def main() -> None: + parser = argparse.ArgumentParser(prog="manifoldbt", description="ManifoldBT CLI") + sub = parser.add_subparsers(dest="command") + + # ── ingest ──────────────────────────────────────────────────────────── + ing = sub.add_parser("ingest", help="Ingest bars from a data provider") + ing.add_argument("--provider", required=True, help="binance | hyperliquid | databento") + ing.add_argument("--symbol", required=True, help="e.g. BTCUSDT, ESH5") + ing.add_argument("--symbol-id", required=True, type=int, help="Unique integer ID for this symbol") + ing.add_argument("--start", required=True, help="RFC3339 start (e.g. 2025-01-01T00:00:00Z)") + ing.add_argument("--end", required=True, help="RFC3339 end") + ing.add_argument("--interval", default="1m", help="Bar interval: 1s, 1m, 5m, 15m, 1h, 1d (default: 1m)") + ing.add_argument("--dataset", default=None, help="Databento dataset (e.g. GLBX.MDP3)") + ing.add_argument("--data-root", default="data", help="Parquet store directory (default: data)") + ing.add_argument("--metadata-db", default="metadata/metadata.sqlite", help="Metadata SQLite path") + ing.add_argument("--exchange", default=None, help="Exchange name for metadata") + ing.add_argument("--asset-class", default="crypto_spot", + help="crypto_spot | crypto_perp | equity | future | option | forex | index") + ing.add_argument("--license-key", default=None, help="Pro license key (or set via bt.activate())") + + args = parser.parse_args() + + if args.command is None: + parser.print_help() + sys.exit(1) + + if args.command == "ingest": + _cmd_ingest(args) + + +def _cmd_ingest(args: argparse.Namespace) -> None: + import manifoldbt as bt + + if args.license_key: + try: + bt.activate(args.license_key) + except Exception as e: + print(f"License activation failed: {e}", file=sys.stderr) + sys.exit(1) + + from manifoldbt.exceptions import LicenseError + + try: + store = bt.ingest( + provider=args.provider, + symbol=args.symbol, + symbol_id=args.symbol_id, + start=args.start, + end=args.end, + interval=args.interval, + dataset=args.dataset, + data_root=args.data_root, + metadata_db=args.metadata_db, + exchange=args.exchange, + asset_class=args.asset_class, + ) + print(f"Ingested to {store.data_root()} (symbols: {store.list_symbols()})") + except LicenseError: + # Message already handled by _warn_pro atexit summary + sys.exit(1) + except Exception as e: + print(f"Error: {e}", file=sys.stderr) + sys.exit(1)