From 6ba4691a02b801b1ac054b7401809a6438fa9252 Mon Sep 17 00:00:00 2001 From: Jimmy7892 Date: Wed, 1 Apr 2026 01:18:05 +0200 Subject: [PATCH] release: v0.4.6 - Cross-exchange backtesting (Pro) - Dict universe format (provider-based symbol resolution) - Exogenous data support (register_exo + exo() expressions) - Provider-based data layout (binance/1h/TICKER.arrow) - Preload fix for provider layout - Exo column resampling for multi-resolution - Pro gate for cross-exchange (clean exit) - ATR/ADX rolling SMA fix - Precise mode hybrid fills --- examples/00_template.py | 6 +- examples/01_trend_following.py | 6 +- examples/02_mean_reversion.py | 6 +- examples/03_multi_asset_momentum.py | 10 +- examples/04_linear_regression.py | 6 +- examples/05_stat_arb.py | 9 +- examples/06_full_visualization.py | 16 +- examples/07_walk_forward.py | 6 +- examples/08_sweep_2d_heatmap.py | 6 +- examples/09_surface_3d.py | 6 +- examples/10_monte_carlo.py | 6 +- examples/11_portfolio.py | 6 +- examples/12_diagnostics.py | 6 +- examples/14_multi_timeframe.py | 6 +- examples/15_cross_exchange.py | 92 +++++++++++ examples/16_hashrate_exogene.py | 214 ++++++++++++++++++++++++ examples/metadata/metadata.sqlite | Bin 0 -> 77824 bytes pyproject.toml | 2 +- python/manifoldbt/__init__.py | 243 ++++++++++++++++++++++++++-- python/manifoldbt/config.py | 49 +++++- python/manifoldbt/expr.py | 29 +++- python/manifoldbt/indicators.py | 138 +++++++++++----- python/manifoldbt/plot/chart.py | 42 ++--- 23 files changed, 792 insertions(+), 118 deletions(-) create mode 100644 examples/15_cross_exchange.py create mode 100644 examples/16_hashrate_exogene.py create mode 100644 examples/metadata/metadata.sqlite diff --git a/examples/00_template.py b/examples/00_template.py index 8ceed59..d786280 100644 --- a/examples/00_template.py +++ b/examples/00_template.py @@ -42,7 +42,7 @@ strategy = ( start, end = time_range("2021-01-01", "2026-01-01") config = mbt.BacktestConfig( - universe=[1], # symbol IDs (1=BTC, 2=ETH, etc.) + universe={"binance": ["BTC-USDT:perp"]}, time_range_start=start, time_range_end=end, bar_interval=Interval.minutes(1), # bar resolution @@ -60,9 +60,11 @@ config = mbt.BacktestConfig( # -- Run ---------------------------------------------------------------------- if __name__ == "__main__": root = os.path.join(os.path.dirname(__file__), "..") + data_root = os.path.abspath(os.path.join(root, "data")) store = mbt.DataStore( - data_root=os.path.abspath(os.path.join(root, "data")), + data_root=data_root, metadata_db=os.path.abspath(os.path.join(root, "metadata", "metadata.sqlite")), + arrow_dir=os.path.join(data_root, "mega"), ) t0 = perf_counter() diff --git a/examples/01_trend_following.py b/examples/01_trend_following.py index 9ba5f01..d72ba8f 100644 --- a/examples/01_trend_following.py +++ b/examples/01_trend_following.py @@ -39,7 +39,7 @@ strategy = ( start, end = time_range("2022-01-01", "2025-01-01") config = mbt.BacktestConfig( - universe=[1], + universe={"binance": ["BTC-USDT:perp"]}, time_range_start=start, time_range_end=end, bar_interval=Interval.hours(1), @@ -58,9 +58,11 @@ config = mbt.BacktestConfig( # -- Run ---------------------------------------------------------------------- if __name__ == "__main__": root = os.path.join(os.path.dirname(__file__), "..") + data_root = os.path.abspath(os.path.join(root, "data")) store = mbt.DataStore( - data_root=os.path.abspath(os.path.join(root, "data")), + data_root=data_root, metadata_db=os.path.abspath(os.path.join(root, "metadata", "metadata.sqlite")), + arrow_dir=os.path.join(data_root, "mega"), ) diff --git a/examples/02_mean_reversion.py b/examples/02_mean_reversion.py index b3bba7c..88f6c2b 100644 --- a/examples/02_mean_reversion.py +++ b/examples/02_mean_reversion.py @@ -33,7 +33,7 @@ strategy = ( start, end = time_range("2021-01-01", "2026-01-01") config = mbt.BacktestConfig( - universe=[1], + universe={"binance": ["BTC-USDT:perp"]}, time_range_start=start, time_range_end=end, bar_interval=Interval.hours(12), @@ -50,9 +50,11 @@ config = mbt.BacktestConfig( # -- Run ---------------------------------------------------------------------- if __name__ == "__main__": root = os.path.join(os.path.dirname(__file__), "..") + data_root = os.path.abspath(os.path.join(root, "data")) store = mbt.DataStore( - data_root=os.path.abspath(os.path.join(root, "data")), + data_root=data_root, metadata_db=os.path.abspath(os.path.join(root, "metadata", "metadata.sqlite")), + arrow_dir=os.path.join(data_root, "mega"), ) t0 = time.perf_counter() diff --git a/examples/03_multi_asset_momentum.py b/examples/03_multi_asset_momentum.py index 5c5e61a..7fe0be7 100644 --- a/examples/03_multi_asset_momentum.py +++ b/examples/03_multi_asset_momentum.py @@ -35,7 +35,11 @@ strategy = ( start, end = time_range("2022-01-01", "2025-01-01") config = mbt.BacktestConfig( - universe=[1, 2, 3, 4, 5], + universe={ + "binance": ["BTC-USDT:perp", "ETH-USDT:perp", "LTC-USDT:perp", + "DOT-USDT:perp", "XRP-USDT:perp"], + }, + # Legacy equivalent: universe=[201, 202, 204, 206, 208] time_range_start=start, time_range_end=end, bar_interval=Interval.hours(12), @@ -53,9 +57,11 @@ config = mbt.BacktestConfig( # -- Run ---------------------------------------------------------------------- if __name__ == "__main__": root = os.path.join(os.path.dirname(__file__), "..") + data_root = os.path.abspath(os.path.join(root, "data")) store = mbt.DataStore( - data_root=os.path.abspath(os.path.join(root, "data")), + data_root=data_root, metadata_db=os.path.abspath(os.path.join(root, "metadata", "metadata.sqlite")), + arrow_dir=os.path.join(data_root, "mega"), ) t0 = time.perf_counter() diff --git a/examples/04_linear_regression.py b/examples/04_linear_regression.py index 7dacc50..7dcd099 100644 --- a/examples/04_linear_regression.py +++ b/examples/04_linear_regression.py @@ -71,7 +71,7 @@ strategy = ( start, end = time_range("2022-01-01", "2025-01-01") config = mbt.BacktestConfig( - universe=[1, 2], + universe={"binance": ["BTC-USDT:perp", "ETH-USDT:perp"]}, time_range_start=start, time_range_end=end, bar_interval=Interval.days(1), @@ -88,9 +88,11 @@ config = mbt.BacktestConfig( # -- Run ----------------------------------------------------------------------- if __name__ == "__main__": root = os.path.join(os.path.dirname(__file__), "..") + data_root = os.path.abspath(os.path.join(root, "data")) store = mbt.DataStore( - data_root=os.path.abspath(os.path.join(root, "data")), + data_root=data_root, metadata_db=os.path.abspath(os.path.join(root, "metadata", "metadata.sqlite")), + arrow_dir=os.path.join(data_root, "mega"), ) t0 = time.perf_counter() diff --git a/examples/05_stat_arb.py b/examples/05_stat_arb.py index ff7bdff..fcd7c05 100644 --- a/examples/05_stat_arb.py +++ b/examples/05_stat_arb.py @@ -15,7 +15,7 @@ from manifoldbt.indicators import close, kalman from manifoldbt.helpers import time_range, Slippage, Interval # -- Spread construction ------------------------------------------------------ -pair_close = mbt.symbol_ref("ETHUSDT", "close") +pair_close = mbt.symbol_ref("binance:ETH-USDT:perp", "close") ratio = close / (pair_close + mbt.lit(1e-12)) # -- Kalman equilibrium ------------------------------------------------------- @@ -41,7 +41,7 @@ strategy = ( start, end = time_range("2022-01-01", "2026-01-01") config = mbt.BacktestConfig( - universe=[1, 2, 5], # BTC, ETH, BNB + universe={"binance": ["BTC-USDT:perp", "ETH-USDT:perp", "BNB-USDT:perp"]}, # BTC, ETH, BNB time_range_start=start, time_range_end=end, bar_interval=Interval.hours(24), @@ -53,15 +53,16 @@ config = mbt.BacktestConfig( fees=mbt.FeeConfig.binance_perps(), slippage=Slippage.fixed_bps(2), warmup_bars=30, - symbol_names={"BTCUSDT": 1, "ETHUSDT": 2, "BNBUSDT": 5}, ) # -- Run ---------------------------------------------------------------------- if __name__ == "__main__": root = os.path.join(os.path.dirname(__file__), "..") + data_root = os.path.abspath(os.path.join(root, "data")) store = mbt.DataStore( - data_root=os.path.abspath(os.path.join(root, "data")), + data_root=data_root, metadata_db=os.path.abspath(os.path.join(root, "metadata", "metadata.sqlite")), + arrow_dir=os.path.join(data_root, "mega"), ) t0 = time.perf_counter() diff --git a/examples/06_full_visualization.py b/examples/06_full_visualization.py index 0946ddf..ba75b81 100644 --- a/examples/06_full_visualization.py +++ b/examples/06_full_visualization.py @@ -43,7 +43,11 @@ strategy = ( # -- Config ------------------------------------------------------------------- start, end = time_range("2021-01-01", "2026-01-01") -ALL_SYMBOLS = list(range(1, 23)) # 22 symbols: BTCUSDT to ARBUSDT +ALL_SYMBOLS = {"binance": [ + "BTC-USDT:perp", "ETH-USDT:perp", "LTC-USDT:perp", "BNB-USDT:perp", + "DOT-USDT:perp", "XRP-USDT:perp", "ADA-USDT:perp", "LINK-USDT:perp", + "DOGE-USDT:perp", "AVAX-USDT:perp", +]} config = mbt.BacktestConfig( universe=ALL_SYMBOLS, @@ -65,9 +69,11 @@ config = mbt.BacktestConfig( if __name__ == "__main__": root = os.path.join(os.path.dirname(__file__), "..") os.makedirs(os.path.join(root, "output"), exist_ok=True) + data_root = os.path.abspath(os.path.join(root, "data")) store = mbt.DataStore( - data_root=os.path.abspath(os.path.join(root, "data")), + data_root=data_root, metadata_db=os.path.abspath(os.path.join(root, "metadata", "metadata.sqlite")), + arrow_dir=os.path.join(data_root, "mega"), ) # -- 1. Single backtest -------------------------------------------------- @@ -82,15 +88,15 @@ if __name__ == "__main__": print("Generating tearsheet...") mbt.plot.tearsheet( result, show=True, - save=os.path.join(root, "output", "tearsheet.png"), + save=os.path.join(root, "output", "tearsheet.html"), ) # -- 3. Summary 3-panel --------------------------------------------------- mbt.plot.summary(result, show=True) - # -- 4. Candlestick chart (symbol_id=1 matches universe) ---------------- + # -- 4. Candlestick chart (first symbol in universe) -------------------- mbt.plot.chart( - result, store, symbol_id=1, + result, store, symbol_id=201, emas=[10, 25], smas=[50], n_bars=120, diff --git a/examples/07_walk_forward.py b/examples/07_walk_forward.py index 16df2d4..a2b0e21 100644 --- a/examples/07_walk_forward.py +++ b/examples/07_walk_forward.py @@ -36,7 +36,7 @@ strategy = ( start, end = time_range("2021-01-01", "2025-01-01") config = mbt.BacktestConfig( - universe=[1], + universe={"binance": ["BTC-USDT:perp"]}, time_range_start=start, time_range_end=end, bar_interval=Interval.hours(12), @@ -53,9 +53,11 @@ config = mbt.BacktestConfig( # -- Run ---------------------------------------------------------------------- if __name__ == "__main__": root = os.path.join(os.path.dirname(__file__), "..") + data_root = os.path.abspath(os.path.join(root, "data")) store = mbt.DataStore( - data_root=os.path.abspath(os.path.join(root, "data")), + data_root=data_root, metadata_db=os.path.abspath(os.path.join(root, "metadata", "metadata.sqlite")), + arrow_dir=os.path.join(data_root, "mega"), ) wf_config = { diff --git a/examples/08_sweep_2d_heatmap.py b/examples/08_sweep_2d_heatmap.py index 69c4f8e..3385f5f 100644 --- a/examples/08_sweep_2d_heatmap.py +++ b/examples/08_sweep_2d_heatmap.py @@ -31,7 +31,7 @@ strategy = ( start, end = time_range("2021-01-01", "2026-01-01") config = mbt.BacktestConfig( - universe=[1], + universe={"binance": ["BTC-USDT:perp"]}, time_range_start=start, time_range_end=end, bar_interval=Interval.hours(1), @@ -49,9 +49,11 @@ config = mbt.BacktestConfig( # -- Run ---------------------------------------------------------------------- if __name__ == "__main__": root = os.path.join(os.path.dirname(__file__), "..") + data_root = os.path.abspath(os.path.join(root, "data")) store = mbt.DataStore( - data_root=os.path.abspath(os.path.join(root, "data")), + data_root=data_root, metadata_db=os.path.abspath(os.path.join(root, "metadata", "metadata.sqlite")), + arrow_dir=os.path.join(data_root, "mega"), ) fast_values = list(range(5, 1000, 5)) diff --git a/examples/09_surface_3d.py b/examples/09_surface_3d.py index 252ba05..f52254e 100644 --- a/examples/09_surface_3d.py +++ b/examples/09_surface_3d.py @@ -31,7 +31,7 @@ strategy = ( start, end = time_range("2021-01-01", "2026-01-01") config = mbt.BacktestConfig( - universe=[1], + universe={"binance": ["BTC-USDT:perp"]}, time_range_start=start, time_range_end=end, bar_interval=Interval.hours(1), @@ -48,9 +48,11 @@ config = mbt.BacktestConfig( # -- Run ---------------------------------------------------------------------- if __name__ == "__main__": root = os.path.join(os.path.dirname(__file__), "..") + data_root = os.path.abspath(os.path.join(root, "data")) store = mbt.DataStore( - data_root=os.path.abspath(os.path.join(root, "data")), + data_root=data_root, metadata_db=os.path.abspath(os.path.join(root, "metadata", "metadata.sqlite")), + arrow_dir=os.path.join(data_root, "mega"), ) fast_values = list(range(5, 1000, 6)) diff --git a/examples/10_monte_carlo.py b/examples/10_monte_carlo.py index e053012..3da0d8c 100644 --- a/examples/10_monte_carlo.py +++ b/examples/10_monte_carlo.py @@ -34,7 +34,7 @@ strategy = ( start, end = time_range("2021-01-01", "2025-01-01") config = mbt.BacktestConfig( - universe=[1], + universe={"binance": ["BTC-USDT:perp"]}, time_range_start=start, time_range_end=end, bar_interval=Interval.hours(12), @@ -51,9 +51,11 @@ config = mbt.BacktestConfig( # -- Run ---------------------------------------------------------------------- if __name__ == "__main__": root = os.path.join(os.path.dirname(__file__), "..") + data_root = os.path.abspath(os.path.join(root, "data")) store = mbt.DataStore( - data_root=os.path.abspath(os.path.join(root, "data")), + data_root=data_root, metadata_db=os.path.abspath(os.path.join(root, "metadata", "metadata.sqlite")), + arrow_dir=os.path.join(data_root, "mega"), ) # 1. Run base backtest diff --git a/examples/11_portfolio.py b/examples/11_portfolio.py index 759f37a..120faa7 100644 --- a/examples/11_portfolio.py +++ b/examples/11_portfolio.py @@ -40,7 +40,7 @@ portfolio = ( start, end = time_range("2021-01-01", "2025-01-01") config = mbt.BacktestConfig( - universe=[1, 2], + universe={"binance": ["BTC-USDT:perp", "ETH-USDT:perp"]}, time_range_start=start, time_range_end=end, bar_interval=Interval.hours(12), @@ -57,9 +57,11 @@ config = mbt.BacktestConfig( # -- Run ---------------------------------------------------------------------- if __name__ == "__main__": root = os.path.join(os.path.dirname(__file__), "..") + data_root = os.path.abspath(os.path.join(root, "data")) store = mbt.DataStore( - data_root=os.path.abspath(os.path.join(root, "data")), + data_root=data_root, metadata_db=os.path.abspath(os.path.join(root, "metadata", "metadata.sqlite")), + arrow_dir=os.path.join(data_root, "mega"), ) print(f"Running portfolio: {portfolio}\n") diff --git a/examples/12_diagnostics.py b/examples/12_diagnostics.py index b4fab3f..3ee96ec 100644 --- a/examples/12_diagnostics.py +++ b/examples/12_diagnostics.py @@ -32,7 +32,7 @@ strategy = ( start, end = time_range("2022-01-01", "2025-01-01") config = mbt.BacktestConfig( - universe=[1], + universe={"binance": ["BTC-USDT:perp"]}, time_range_start=start, time_range_end=end, bar_interval=Interval.hours(12), @@ -49,9 +49,11 @@ config = mbt.BacktestConfig( # -- Run ---------------------------------------------------------------------- if __name__ == "__main__": root = os.path.join(os.path.dirname(__file__), "..") + data_root = os.path.abspath(os.path.join(root, "data")) store = mbt.DataStore( - data_root=os.path.abspath(os.path.join(root, "data")), + data_root=data_root, metadata_db=os.path.abspath(os.path.join(root, "metadata", "metadata.sqlite")), + arrow_dir=os.path.join(data_root, "mega"), ) # -- 1. Look-ahead bias detection ----------------------------------------- diff --git a/examples/14_multi_timeframe.py b/examples/14_multi_timeframe.py index 7427047..e6668a4 100644 --- a/examples/14_multi_timeframe.py +++ b/examples/14_multi_timeframe.py @@ -47,7 +47,7 @@ strategy = ( start, end = time_range("2022-01-01", "2025-01-01") config = mbt.BacktestConfig( - universe=[1], + universe={"binance": ["BTC-USDT:perp"]}, time_range_start=start, time_range_end=end, bar_interval=Interval.hours(1), @@ -68,9 +68,11 @@ config = mbt.BacktestConfig( # -- Run ---------------------------------------------------------------------- if __name__ == "__main__": root = os.path.join(os.path.dirname(__file__), "..") + data_root = os.path.abspath(os.path.join(root, "data")) store = mbt.DataStore( - data_root=os.path.abspath(os.path.join(root, "data")), + data_root=data_root, metadata_db=os.path.abspath(os.path.join(root, "metadata", "metadata.sqlite")), + arrow_dir=os.path.join(data_root, "mega"), ) t0 = time.perf_counter() diff --git a/examples/15_cross_exchange.py b/examples/15_cross_exchange.py new file mode 100644 index 0000000..ebe59d4 --- /dev/null +++ b/examples/15_cross_exchange.py @@ -0,0 +1,92 @@ +"""Example 15: Cross-Exchange — Signal Binance, Execution dYdX. + +Simple RSI mean-reversion: +- RSI computed on Binance BTC perp data +- Trades executed at dYdX BTC-USD prices +- Both loaded via universe dict — no special config needed + +Prerequisite: + Binance perp data (bars_1m/201.arrow) + dYdX data (dydx/1h/BTC-USD.arrow) +""" + +import time +import manifoldbt as mbt +from manifoldbt.indicators import rsi, ema +from manifoldbt.expr import col, symbol_ref, lit, when +from manifoldbt.helpers import time_range, Interval, Slippage + +# ============================================================================= +# Signal — RSI + EMA from Binance BTC, applied to dYdX BTC +# All SymbolRef expressions must be named signals (for pass 2b resolution) +# ============================================================================= +bn_btc_close = symbol_ref("binance:BTC-USDT:perp", "close") +bn_btc_rsi = rsi(bn_btc_close, 14) +bn_ema_fast = ema(bn_btc_close, 15) +bn_ema_slow = ema(bn_btc_close, 30) +trend_up = bn_ema_fast > bn_ema_slow + +# Size references named signals only (no inline SymbolRef) +signal = when( + (col("trend") > lit(0.5)) & (col("bn_rsi") > lit(70.0)), 1.0, + when((col("trend") < lit(0.5)) & (col("bn_rsi") < lit(30.0)), -1.0, + 0.0), +) + +# ============================================================================= +# Strategy +# ============================================================================= +strategy = ( + mbt.Strategy.create("cross_exchange_rsi") + .signal("bn_rsi", bn_btc_rsi) + .signal("trend", when(trend_up, 1.0, 0.0)) + .size(signal) + .describe("Signal: Binance RSI | Execution: dYdX") +) + +# ============================================================================= +# Config — everything in universe +# ============================================================================= +START, END = time_range("2024-02-01", "2026-03-01") + +config = mbt.BacktestConfig( + universe={ + "dydx": ["BTC-USD:perp"], # execution (fills here) + "binance": ["BTC-USDT:perp"], # signal source (via symbol_ref) + }, + time_range_start=START, + time_range_end=END, + bar_interval=Interval.hours(6), + initial_capital=10_000, + warmup_bars=30, + execution=mbt.ExecutionConfig(signal_delay=1), + fees=mbt.FeeConfig(maker_fee_bps=1.0, taker_fee_bps=2.5), + slippage=Slippage.fixed_bps(2), +) + +# ============================================================================= +# Run +# ============================================================================= +if __name__ == "__main__": + import os + root = os.path.dirname(os.path.abspath(__file__)) + data_root = os.path.abspath(os.path.join(root, "..", "data")) + meta_db = os.path.join(root, "..", "metadata", "metadata.sqlite") + + store = mbt.DataStore( + data_root=data_root, + metadata_db=meta_db, + arrow_dir=os.path.join(data_root, "mega"), + ) + + print("Running: cross_exchange_rsi") + print(" Signal: binance:BTC-USDT:perp (RSI + EMA)") + print(" Execution: dydx:BTC-USD:perp") + print() + + t0 = time.perf_counter() + result = mbt.run(strategy, config, store) + elapsed = time.perf_counter() - t0 + + print(result.summary()) + print(f"\nElapsed: {elapsed:.3f}s") + result.plot_equity(show=True) diff --git a/examples/16_hashrate_exogene.py b/examples/16_hashrate_exogene.py new file mode 100644 index 0000000..1a21820 --- /dev/null +++ b/examples/16_hashrate_exogene.py @@ -0,0 +1,214 @@ +"""Example 16: BTC-Hashrate Spread — Exogenous Data Strategy. + +Thesis: Bitcoin hashrate is a proxy for miner commitment and network +security. When BTC price drops but hashrate holds (or rises), miners +are still profitable and the sell-off is likely transient — buy the dip. +When price rises but hashrate lags, the rally lacks fundamental backing. + +The strategy normalizes both BTC price and hashrate via EMA ratios +(price/EMA and hashrate/EMA), then computes a spread between the two. +A rolling z-score of the spread generates the signal: negative z means +price is cheap relative to hashrate (long), positive means expensive. + +Exogenous data flow: + 1. Fetch hashrate CSV (or use sample generator below) + 2. Register via mbt.register_exo("hashrate", df) + 3. Declare in BacktestConfig(exo_data=["hashrate"]) + 4. Access with exo("hashrate") in expressions + +Prerequisite: + Binance BTC perp data + hashrate exo registered in data/mega/exo/ +""" + +import time +import numpy as np +import manifoldbt as mbt +from manifoldbt.indicators import ema, close +from manifoldbt.expr import col, exo, lit, when, hold +from manifoldbt.helpers import time_range, Interval, Slippage + + +# ============================================================================= +# Parameters +# ============================================================================= +SMOOTH = 30 # EMA period for normalization +ZSCORE_WINDOW = 90 # Rolling z-score lookback (days) +ENTRY_Z = -1.5 # Long when spread z < -1.5 (price cheap vs hashrate) +EXIT_Z = 0.0 # Exit when spread reverts to mean +SHORT_Z = 1.5 # Short when spread z > 1.5 (price expensive vs hashrate) +SIZE = 0.5 # Position size (fraction of capital) + + +# ============================================================================= +# Indicators +# ============================================================================= + +# Normalize price: ratio to its own EMA (>1 = above trend, <1 = below) +price_ratio = close / ema(close, SMOOTH) + +# Normalize hashrate the same way +hr = exo("hashrate") +hr_ratio = hr / ema(hr, SMOOTH) + +# Spread: price_ratio - hr_ratio +# Positive = price running ahead of hashrate, negative = price lagging +spread = price_ratio - hr_ratio + +# Z-score of the spread (rolling mean & std) +spread_z = spread.zscore(ZSCORE_WINDOW) + + +# ============================================================================= +# Sizing +# ============================================================================= +z = col("spread_z") + +size = when( + z < lit(ENTRY_Z), lit(SIZE), # price cheap vs hashrate -> long + when(z > lit(SHORT_Z), -lit(SIZE), # price expensive vs hashrate -> short + when((z > lit(EXIT_Z)) & (z < lit(SHORT_Z)), 0.0, # neutral zone -> flat + hold())), +) + + +# ============================================================================= +# Strategy +# ============================================================================= +strategy = ( + mbt.Strategy.create("hashrate_spread") + .signal("price_ratio", price_ratio) + .signal("hr_ratio", hr_ratio) + .signal("spread", spread) + .signal("spread_z", spread_z) + .size(size) + .describe("BTC vs Hashrate spread z-score mean-reversion") +) + + +# ============================================================================= +# Config +# ============================================================================= +START, END = time_range("2021-06-01", "2026-03-01") + +config = mbt.BacktestConfig( + universe={"binance": ["BTC-USDT:perp"]}, + time_range_start=START, + time_range_end=END, + bar_interval=Interval.days(1), + initial_capital=10_000, + warmup_bars=ZSCORE_WINDOW + SMOOTH, + exo_data=["hashrate"], + execution=mbt.ExecutionConfig(signal_delay=1, allow_short=True), + fees=mbt.FeeConfig.binance_perps(), + slippage=Slippage.fixed_bps(3), +) + + +# ============================================================================= +# Hashrate data helper +# ============================================================================= +def fetch_hashrate_csv(path: str = "hashrate.csv"): + """Load hashrate from a CSV with columns: timestamp, hashrate. + + Public sources (daily, free): + - https://api.blockchain.info/charts/hash-rate?timespan=5years&format=csv + - Glassnode, CoinMetrics (API key) + + The CSV should have: + timestamp — date or datetime (parsed automatically) + hashrate — daily avg hashrate in EH/s (float) + """ + import pandas as pd + df = pd.read_csv(path, parse_dates=["timestamp"]) + df["timestamp"] = pd.to_datetime(df["timestamp"], utc=True) + df = df.sort_values("timestamp").reset_index(drop=True) + return df + + +def generate_sample_hashrate(start="2020-01-01", end="2026-03-01"): + """Generate synthetic hashrate data for testing. + + Mimics the real BTC hashrate trajectory: + - Exponential growth trend (~50% annual) + - China ban crash (May-Jul 2021): -50% + - Recovery + continued growth + - Random noise (~5% daily vol) + """ + import pandas as pd + + dates = pd.date_range(start, end, freq="D", tz="UTC") + n = len(dates) + + # Base: exponential growth from ~120 EH/s to ~800 EH/s + t = np.arange(n) / 365.25 + base = 120 * np.exp(0.40 * t) # ~50% annual growth + + # China ban shock: May-Jul 2021 + ban_start = pd.Timestamp("2021-05-15", tz="UTC") + ban_end = pd.Timestamp("2021-07-15", tz="UTC") + recovery_end = pd.Timestamp("2022-01-01", tz="UTC") + + shock = np.ones(n) + for i, d in enumerate(dates): + if ban_start <= d <= ban_end: + # Linear drop to 50% + frac = (d - ban_start) / (ban_end - ban_start) + shock[i] = 1.0 - 0.50 * frac + elif ban_end < d < recovery_end: + # Recovery from 50% back to 100% + frac = (d - ban_end) / (recovery_end - ban_end) + shock[i] = 0.50 + 0.50 * frac + + # Random noise (geometric brownian) + rng = np.random.default_rng(42) + noise = np.exp(np.cumsum(rng.normal(0, 0.02, n))) + noise /= noise[0] + + hashrate = base * shock * noise + + return pd.DataFrame({"timestamp": dates, "hashrate": hashrate}) + + +# ============================================================================= +# Run +# ============================================================================= +if __name__ == "__main__": + import os + + root = os.path.dirname(os.path.abspath(__file__)) + data_root = os.path.abspath(os.path.join(root, "..", "data")) + meta_db = os.path.join(root, "..", "metadata", "metadata.sqlite") + + store = mbt.DataStore( + data_root=data_root, + metadata_db=meta_db, + arrow_dir=os.path.join(data_root, "mega"), + ) + + # -- Register hashrate exo data ------------------------------------------- + csv_path = os.path.join(root, "hashrate.csv") + if os.path.exists(csv_path): + print("Loading hashrate from CSV...") + hr_df = fetch_hashrate_csv(csv_path) + else: + print("No hashrate.csv found — generating synthetic data for demo...") + hr_df = generate_sample_hashrate() + + mbt.register_exo("hashrate", hr_df, store=store) + print(f" Registered {len(hr_df)} hashrate data points") + print(f" Range: {hr_df['timestamp'].iloc[0]} -> {hr_df['timestamp'].iloc[-1]}") + print() + + # -- Run backtest --------------------------------------------------------- + print("Running: hashrate_spread") + print(" Long when spread z < -1.5 (price cheap vs hashrate)") + print(" Short when spread z > +1.5 (price expensive vs hashrate)") + print() + + t0 = time.perf_counter() + result = mbt.run(strategy, config, store) + elapsed = time.perf_counter() - t0 + + print(result.summary()) + print(f"\nElapsed: {elapsed:.3f}s") + result.plot_equity(show=True) diff --git a/examples/metadata/metadata.sqlite b/examples/metadata/metadata.sqlite new file mode 100644 index 0000000000000000000000000000000000000000..339fc695e2bc323d6f9448d5e8a8002b4ee06c79 GIT binary patch literal 77824 zcmeI&U2oe|7{Kv(?UFWq+Y3`Jrh$btXz04W493`mpzE@!T1aV|N*gzpm$<1}Vz;)_ zt%#d;6XSvpz!%|)TW+{S;v4V&Oy=lsrd&avliZB$Lq zPWy`9WmKlpxX=C9m^GdFV6 zf6x3h^~?0H(?3oeOuZQYd2D_B+x)v@?~VR&_K%T&MxLJeT~uQGuL~T#eW6g=np2J* znpV?zZZ@B5`(3?ldWV{0?AeazYE9kKUBlB3497KXD^YxLb+fWkuc&LadzDA(DfTa_ zVq34NiQ~-5)0oX`YIG{iY?hDSI$tQQhyd4m0lJ5sCw5!w=zDvn)pE7AZ9nbqCBD5d z5Lco?w3`ekQ98Tl*av3QaIUG2{!9x?MPP!asd;w!Xy#m@v^uXG-RXrT8xRHWYI?&n z4~*p3=LZ5y)`)hLktK^`Ql>mj`Dk*cPu<4hhv$fLOtxgLR)G@ZIaW(^4WrkR^iDLa zmT4LCQ0d9w>0a0MEy67&Uz%M(Bc@!f6V{8)6}oi{&oLV(?eX`IN_5NIFCUBLDdcy)w&eeuI{sih9KsVpvLt56#K&SRg%X8PHpw~FZ*VNADD!(0ZEA7< z=t-*gkL|Et-@ptf?H$MNq#jq!=`yM?n7$-eU%MFi$1RJ2S$y5VOn7KZi7zHz+!`yC zE?-u@d(da?Mok${u^l!-rA%M^VS_xg@zij_T$4>rOg*~muk($z_#MCIGF|5X+sFZS){jM$Mt&=(_Rs3)@*ZfAFNnB-A6XCXPufK-Kfq^O= z-4O#(T*~ZsO-Bq`ze>mvF3wt*6;YJ263-4>7{nizh#{x@Q$$n{loZP^i}^xneqMPM zcAZ>;Dx?Nk_ z*pAv2^>uIw600?sDk;TH!)-X`p1*KKBV^%hzO;Jf4JfX*EE}0$7(LYvQqMhmbW16e z78aD_cf#p2OxBLs3Kvi}YSGCTXJU>6#nCVfocex}xa&>KSadAmAEz>jg_VVKL&H2R}^NUOMrH^kfe{%by#p}10KU}_fQ`GvZDB7jW z{FxK~*bqPf0R#|0009ILKmY**5I`U!0y865$NT?xz<>TP{^*|#0R#|0009ILKmY** z5I_I{1Trk(KmX_cKf@hN(-1%a0R#|0009ILKmY**5D)^~|C0s?Ab{00IagfB*srAb#j1-Spuejn391Q0*~0R#|0009ILKmY**{sr7@TW bool: def _require_pro(feature: str) -> None: - """Warn and raise if not Pro. Use _gate_pro for graceful skip.""" + """Print Pro warning and exit cleanly if not Pro.""" if _is_pro(): return - _warn_pro(feature) - raise LicenseError(f"{feature} -- Pro license required") + print(f"\n\033[38;5;214m[!] {feature} -- Pro feature\033[0m") + print("\033[38;5;214m -> upgrade at www.manifoldbt.com\033[0m") + raise SystemExit(0) def _gate_pro(feature: str) -> bool: @@ -151,24 +152,126 @@ def _classify_error(exc: Exception) -> Exception: # Config preparation (symbol resolution + strategy orders merge) # --------------------------------------------------------------------------- -def _prepare_config(config: BacktestConfig, strategy: Strategy, store: DataStore) -> BacktestConfig: - """Prepare config for execution: resolve symbols and merge strategy orders.""" - cfg = config +_AC_SUFFIX_MAP = { + "spot": "CryptoSpot", "perp": "CryptoPerpetual", + "future": "Future", "equity": "Equity", + "option": "EquityOption", "fx": "Forex", + "index": "Index", +} - # Resolve string symbols in universe - has_strings = any(isinstance(s, str) for s in cfg.universe) - has_strategy_orders = hasattr(strategy, '_orders') and strategy._orders +def _resolve_normalized(sym: str, provider: str, store) -> int: + """Resolve a normalized symbol name like 'BTC-USDT:perp' on a provider to SymbolId. - if not has_strings and not has_strategy_orders: - return cfg + Tries: 1) normalized parse → metadata lookup by (base, quote, asset_class, provider) + 2) fallback to raw ticker match + """ + import sqlite3, os - cfg = copy.deepcopy(cfg) + # Parse normalized name: "BTC-USDT:perp" → base=BTC, quote=USDT, ac=CryptoPerpetual + if ":" in sym: + pair, suffix = sym.rsplit(":", 1) + ac_db = _AC_SUFFIX_MAP.get(suffix) + else: + pair, ac_db = sym, None - if has_strings: - cfg.universe = resolve_universe(cfg.universe, store) + if "-" in pair: + base, quote = pair.split("-", 1) + else: + base, quote = pair, "" + + if ac_db: + # Try metadata lookup by (base, quote, asset_class, provider) + meta_db = store.metadata_db() + conn = sqlite3.connect(meta_db) + row = conn.execute( + "SELECT id FROM symbols WHERE base_currency=? COLLATE NOCASE " + "AND quote_currency=? COLLATE NOCASE AND asset_class=? " + "AND exchange=? COLLATE NOCASE ORDER BY id DESC LIMIT 1", + (base, quote, ac_db, provider.upper()), + ).fetchone() + conn.close() + if row: + return row[0] + + # Fallback: try raw ticker match + try: + return store.resolve_symbol(sym) + except Exception: + raise ValueError( + f"Symbol '{sym}' not found on provider '{provider}'. " + f"Searched: base={base}, quote={quote}, class={ac_db}" + ) + + +def _resolve_source_dict(source, store): + """Resolve a signal/execution source dict → list of (provider, norm_sym, symbol_id, raw_ticker). + + Returns the raw ticker from metadata (what the files are named on disk). + """ + if isinstance(source, dict): + import sqlite3 + conn = sqlite3.connect(store.metadata_db()) + resolved = [] + for provider, symbols in source.items(): + for sym in symbols: + sid = _resolve_normalized(sym, provider, store) + # Get raw ticker from metadata + row = conn.execute("SELECT ticker FROM symbols WHERE id=?", (sid,)).fetchone() + raw_ticker = row[0] if row else sym + resolved.append((provider, sym, sid, raw_ticker)) + conn.close() + return resolved + return None + + +def _prepare_config(config: BacktestConfig, strategy, store: DataStore) -> BacktestConfig: + """Prepare config for execution: resolve symbols, convert deprecated fields.""" + cfg = copy.deepcopy(config) + + # --- Dict universe: {"binance": ["BTC-USDT:perp"], "onchain": ["hashrate"]} --- + if isinstance(cfg.universe, dict): + # Cross-exchange (multiple providers) is a Pro feature. + if len(cfg.universe) > 1: + _require_pro("Cross-exchange backtesting") + + resolved_universe = [] + qualified_names = {} # "binance:BTC-USDT:perp" → SymbolId + + for provider, symbols in cfg.universe.items(): + for sym in symbols: + sid = _resolve_normalized(sym, provider, store) + resolved_universe.append(sid) + qualified = f"{provider}:{sym}" + qualified_names[qualified] = sid + + cfg.universe = resolved_universe + cfg.symbol_names = qualified_names + + # Clear deprecated fields + cfg.signal_source = None + cfg.execution_source = None + cfg.pair_map = {} + cfg.exo_sources = {} + cfg.provider = None + + # --- Legacy list universe: [1, 2, 3] or ["BTC-USD", "ETH-USD"] --- + elif cfg.universe: + if any(isinstance(s, str) for s in cfg.universe): + cfg.universe = resolve_universe(cfg.universe, store, cfg.symbol_names) + + # Legacy exo_sources resolution + if cfg.exo_sources and any(isinstance(k, str) for k in cfg.exo_sources): + resolved = {} + for key, val in cfg.exo_sources.items(): + sid = store.resolve_symbol(key) if isinstance(key, str) else key + resolved[sid] = val + cfg.exo_sources = resolved + + if cfg.provider and not cfg.signal_source: + cfg.signal_source = cfg.provider # Merge orders from strategy into execution config - if has_strategy_orders: + if strategy and hasattr(strategy, '_orders') and strategy._orders: if cfg.execution.orders is None: cfg.execution.orders = OrderConfig() for key, val in strategy._orders.items(): @@ -519,6 +622,7 @@ def run_batch( One :class:`Result` per strategy, in input order. """ try: + config = _prepare_config(config, None, store) config = _cap_output_resolution(config) store = _resolve_store(config, store) strategy_jsons = [strat.to_json() for strat in strategies] @@ -556,6 +660,7 @@ def run_batch_lite( One :class:`BatchResultLite` per strategy (name, metrics, equity, trade_count). """ try: + config = _prepare_config(config, None, store) config = _cap_output_resolution(config) store = _resolve_store(config, store) strategy_jsons = [strat.to_json() for strat in strategies] @@ -640,6 +745,7 @@ def run_walk_forward( """ if not _gate_pro("Walk-forward optimization"): return {"folds": [], "best_params_per_fold": []} + config = _prepare_config(config, strategy, store) wf_json = json.dumps(_convert_param_grid_in_config(wf_config)) return _run_walk_forward_native(strategy.to_json(), wf_json, config.to_json(), store) @@ -667,6 +773,7 @@ def run_sweep_2d( Returns: Dict with ``metric_grid`` (2D list), ``x_values``, ``y_values``, etc. """ + config = _prepare_config(config, strategy, store) sweep_json = json.dumps(_convert_scalar_values_in_sweep(sweep_config)) return _run_sweep_2d_native(strategy.to_json(), sweep_json, config.to_json(), store) @@ -692,6 +799,7 @@ def run_stability( Returns: Dict with ``stability_score``, ``metric_values``, ``mean_metric``, ``std_metric``. """ + config = _prepare_config(config, strategy, store) stab_json = json.dumps(_convert_scalar_values_in_stability(stability_config)) return _run_stability_native(strategy.to_json(), stab_json, config.to_json(), store) @@ -835,6 +943,7 @@ def run_portfolio( breakdown via ``result.per_strategy``. """ try: + config = _prepare_config(config, None, store) raw_combined, per_strategy_info = _run_portfolio_native( portfolio.to_json(), config.to_json(), @@ -892,6 +1001,105 @@ def _convert_scalar_values_in_stability(stability_config: Dict[str, Any]) -> Dic return result +# --------------------------------------------------------------------------- +# Exogenous data registration +# --------------------------------------------------------------------------- + +def register_exo( + name: str, + data, + store: Optional["DataStore"] = None, + data_root: str = "data", + provider: Optional[str] = None, + timeframe: str = "1d", +): + """Register an exogenous data series for use in strategies. + + Without ``provider``: writes to ``{root}/exo/{name}.arrow`` (legacy layout). + With ``provider``: writes to ``{root}/{provider}/{timeframe}/{name}.arrow`` + (unified layout, used for cross-exchange data). + + Args: + name: Series identifier (e.g. ``"hashrate"``, ``"BTCUSDT"``). + data: A pandas/polars DataFrame or dict with a ``"timestamp"`` column + and one or more float value columns. + store: Optional DataStore to infer ``data_root`` from. + data_root: Root data directory (default ``"data"``). + provider: Provider name for unified layout (e.g. ``"binance"``). + timeframe: Timeframe label (e.g. ``"1d"``, ``"1h"``). Default ``"1d"``. + + Example:: + + # Legacy (non-symbol exo like hashrate) + bt.register_exo("hashrate", df) + + # Unified layout (cross-exchange) + bt.register_exo("BTCUSDT", df, provider="binance", timeframe="1h") + """ + import pyarrow as pa + from pathlib import Path + + # Resolve data root + if store is not None: + root = Path(store.data_root()) / "mega" + else: + root = Path(data_root) / "mega" + + if provider: + # Unified layout: {root}/{provider}/{timeframe}/{name}.arrow + target_dir = root / provider / timeframe + else: + # Legacy layout: {root}/exo/{name}.arrow + target_dir = root / "exo" + target_dir.mkdir(parents=True, exist_ok=True) + + # Convert to Arrow Table + if hasattr(data, "to_arrow"): + # Polars DataFrame + table = data.to_arrow() + elif hasattr(data, "columns"): + # Pandas DataFrame + import pandas as pd + table = pa.Table.from_pandas(data) + elif isinstance(data, dict): + table = pa.table(data) + else: + raise TypeError(f"Unsupported data type: {type(data)}. Use a pandas/polars DataFrame or dict.") + + # Ensure timestamp is TimestampNanosecond(UTC) + ts_idx = table.schema.get_field_index("timestamp") + if ts_idx < 0: + raise ValueError("Data must have a 'timestamp' column") + + ts_type = table.schema.field(ts_idx).type + if not pa.types.is_timestamp(ts_type): + raise ValueError(f"'timestamp' column must be a timestamp type, got {ts_type}") + + # Cast to nanos UTC if needed + target_type = pa.timestamp("ns", tz="UTC") + if ts_type != target_type: + ts_col = table.column(ts_idx).cast(target_type) + table = table.set_column(ts_idx, pa.field("timestamp", target_type), ts_col) + + # Cast value columns to float64 + for i, field in enumerate(table.schema): + if field.name == "timestamp": + continue + if field.type != pa.float64(): + table = table.set_column( + i, pa.field(field.name, pa.float64()), table.column(i).cast(pa.float64()) + ) + + # Write Arrow IPC + path = target_dir / f"{name}.arrow" + writer = pa.ipc.new_file(str(path), table.schema) + writer.write_table(table) + writer.close() + + print(f"Registered exo '{name}': {table.num_rows} rows, " + f"columns={[f.name for f in table.schema if f.name != 'timestamp']} -> {path}") + + # --------------------------------------------------------------------------- # Public API # --------------------------------------------------------------------------- @@ -919,6 +1127,7 @@ __all__ = [ "TimeframeRef", "asset", "col", + "exo", "lit", "param", "s", @@ -956,6 +1165,8 @@ __all__ = [ # Portfolio "Portfolio", "run_portfolio", + # Exogenous data + "register_exo", # Version "__version__", # Indicators (submodule) diff --git a/python/manifoldbt/config.py b/python/manifoldbt/config.py index 623b912..9722824 100644 --- a/python/manifoldbt/config.py +++ b/python/manifoldbt/config.py @@ -187,8 +187,8 @@ class BacktestConfig: Allows indicators (EMA, SMA, etc.) to stabilise. During warmup, equity tracking runs but no trades are generated. Set to at least the longest indicator window (e.g. 25 for EMA(25)).""" - precise: bool = False - """When True, always load finest resolution (1m) regardless of bar_interval. + accuracy: bool = False + """When True, simulation runs on 1-minute bars regardless of bar_interval. Signals are still evaluated at bar_interval resolution (hybrid mode). Use for precise SL/TP fills and intraday drawdown tracking. Slower.""" extra_timeframes: Dict[str, Any] = field(default_factory=dict) @@ -196,6 +196,27 @@ class BacktestConfig: Maps labels to Interval dicts. The engine resamples native bars and injects prefixed columns (e.g. "1h.close", "4h.high"). Example: ``{"1h": Interval.hours(1), "4h": Interval.hours(4)}``""" + exo_data: List[str] = field(default_factory=list) + """Exogenous data series names to inject into signal evaluation. + Each name corresponds to an ``exo/{name}/`` directory in the data store + (written via ``bt.register_exo()``). Columns are ASOF-joined onto + bar timestamps and accessible as ``col("exo.{name}.{column}")``. + Example: ``["hashrate", "fear_greed"]``""" + signal_source: Any = None + """Signal data source. Dict mapping provider → list of normalized symbols. + Example: ``{"binance": ["BTC-USDT:perp", "ETH-USDT:perp"]}`` + Also accepts a string (single provider for all symbols) for backward compat.""" + execution_source: Any = None + """Execution data source. Same format as ``signal_source``. + Fill prices come from this source. When absent, same as ``signal_source``. + Example: ``{"dydx": ["BTC-USD:perp", "ETH-USD:perp"]}``""" + pair_map: Dict[str, str] = field(default_factory=dict) + """Explicit mapping from signal symbol to execution symbol. + Required when signal and execution have different tickers. + Example: ``{"BTC-USDT:perp": "BTC-USD:perp"}``""" + # Deprecated — kept for backward compat + provider: Optional[str] = None + exo_sources: Dict = field(default_factory=dict) def to_json_dict(self) -> dict: d: dict = { @@ -224,10 +245,23 @@ class BacktestConfig: d["symbol_names"] = self.symbol_names if self.warmup_bars > 0: d["warmup_bars"] = self.warmup_bars - if self.precise: + if self.accuracy: d["precise"] = True if self.extra_timeframes: d["extra_timeframes"] = self.extra_timeframes + if self.exo_data: + d["exo_data"] = self.exo_data + if self.signal_source: + d["signal_source"] = self.signal_source + if self.execution_source: + d["execution_source"] = self.execution_source + # Deprecated fields (backward compat) + if self.provider: + d["provider"] = self.provider + if self.exo_sources: + d["exo_sources"] = { + str(sid): list(src) for sid, src in self.exo_sources.items() + } return d def to_json(self) -> str: @@ -237,12 +271,14 @@ class BacktestConfig: def resolve_universe( universe: List[Union[int, str]], store: Any, + symbol_names: Optional[Dict[str, int]] = None, ) -> List[int]: """Resolve a mixed list of symbol IDs and ticker names to integer IDs. Args: universe: List of integer IDs or string ticker names. store: A ``DataStore`` instance (must have ``resolve_symbol()``). + symbol_names: Optional name-to-ID mapping (checked before store). Returns: List of integer symbol IDs. @@ -256,12 +292,15 @@ def resolve_universe( if isinstance(item, int): result.append(item) elif isinstance(item, str): - if store is None: + if symbol_names and item in symbol_names: + result.append(symbol_names[item]) + elif store is None: raise TypeError( f"DataStore required to resolve symbol name {item!r}. " f"Pass integer IDs or provide a store." ) - result.append(store.resolve_symbol(item)) + else: + result.append(store.resolve_symbol(item)) else: result.append(int(item)) return result diff --git a/python/manifoldbt/expr.py b/python/manifoldbt/expr.py index b714469..db0203b 100644 --- a/python/manifoldbt/expr.py +++ b/python/manifoldbt/expr.py @@ -5,7 +5,7 @@ Builds an expression tree that serializes to JSON matching the Rust """ from __future__ import annotations -from typing import Any, Union +from typing import Any, Optional, Union from manifoldbt._serde import scalar_value_to_json @@ -518,6 +518,33 @@ def when(condition: Expr, true_value: Any = 1.0, false_value: Any = float("nan") return Expr("IfElse", condition, _coerce(true_value), _coerce(false_value)) +def exo(name: str, column: Optional[str] = None) -> Expr: + """Reference an exogenous data column. + + Exogenous data is registered via ``bt.register_exo()`` and declared + in ``BacktestConfig(exo_data=[...])``. + + Args: + name: Exo series name (e.g. ``"hashrate"``). + column: Column name within the exo series. If ``None``, defaults + to ``name`` (convenient when the series has a single value column + with the same name as the series). + + Returns: + An ``Expr`` referencing ``col("exo.{name}.{column}")``. + + Example:: + + # Single-column shorthand + signal = rsi(exo("hashrate"), 14) > 70 + + # Multi-column explicit + signal = exo("onchain", "active_addresses") > 1_000_000 + """ + col_name = column if column is not None else name + return col(f"exo.{name}.{col_name}") + + def symbol_ref(symbol: str, column: str) -> Expr: """Reference a column from a specific symbol's data. diff --git a/python/manifoldbt/indicators.py b/python/manifoldbt/indicators.py index e3defd3..e648514 100644 --- a/python/manifoldbt/indicators.py +++ b/python/manifoldbt/indicators.py @@ -128,9 +128,14 @@ def rsi(source: Expr, period=14) -> Expr: return source.rsi(period) -def stoch_k(period: int = 14) -> Expr: - """Stochastic %K oscillator (native Rust, uses high/low/close).""" - return Expr("StochK", high, low, close, period) +def stoch_k(period: int = 14, *, h: Expr = None, l: Expr = None, c: Expr = None) -> Expr: + """Stochastic %K oscillator (native Rust). + + Args: + h, l, c: Custom high/low/close columns (e.g. exo columns). + Defaults to native bar columns. + """ + return Expr("StochK", h or high, l or low, c or close, period) def stochastic_k(period: int = 14, source: Expr = None) -> Expr: @@ -144,19 +149,32 @@ def stochastic_k(period: int = 14, source: Expr = None) -> Expr: return (c - lowest) / (highest - lowest + lit(1e-12)) * lit(100.0) -def williams_r(period: int = 14) -> Expr: - """Williams %R oscillator (native Rust, uses high/low/close).""" - return Expr("WilliamsR", high, low, close, period) +def williams_r(period: int = 14, *, h: Expr = None, l: Expr = None, c: Expr = None) -> Expr: + """Williams %R oscillator (native Rust). + + Args: + h, l, c: Custom high/low/close columns. Defaults to native bar columns. + """ + return Expr("WilliamsR", h or high, l or low, c or close, period) -def cci(period: int = 20) -> Expr: - """Commodity Channel Index (native Rust, uses high/low/close).""" - return Expr("Cci", high, low, close, period) +def cci(period: int = 20, *, h: Expr = None, l: Expr = None, c: Expr = None) -> Expr: + """Commodity Channel Index (native Rust). + + Args: + h, l, c: Custom high/low/close columns. Defaults to native bar columns. + """ + return Expr("Cci", h or high, l or low, c or close, period) -def adx(period: int = 14) -> Expr: - """Average Directional Index (native Rust, uses high/low/close).""" - return Expr("Adx", high, low, close, period) +def adx(period: int = 14, *, h: Expr = None, l: Expr = None, c: Expr = None) -> Expr: + """Average Directional Index (native Rust). + + Args: + h, l, c: Custom high/low/close columns (e.g. exo columns). + Defaults to native bar columns. + """ + return Expr("Adx", h or high, l or low, c or close, period) # --------------------------------------------------------------------------- @@ -183,39 +201,59 @@ def bollinger_width(source: Expr, period: int = 20, num_std: float = 2.0) -> Exp return source.bollinger_width(period, num_std) -def atr(period: int = 14) -> Expr: +def atr(period: int = 14, *, h: Expr = None, l: Expr = None, c: Expr = None) -> Expr: """Average True Range (native Rust, Wilder's smoothing, single-pass O(n)). - Uses ``high``, ``low``, ``close`` columns from the bar data. + Args: + h, l, c: Custom high/low/close columns. Defaults to native bar columns. """ - return Expr("Atr", high, low, close, period) + return Expr("Atr", h or high, l or low, c or close, period) -def true_range() -> Expr: - """True Range (native Rust, uses high/low/close).""" - return Expr("TrueRange", high, low, close) +def true_range(*, h: Expr = None, l: Expr = None, c: Expr = None) -> Expr: + """True Range (native Rust). - -def natr(period: int = 14) -> Expr: - """Normalized ATR (native Rust, uses high/low/close).""" - return Expr("Natr", high, low, close, period) - - -def keltner_channels(period: int = 20, multiplier: float = 1.5) -> Tuple[Expr, Expr, Expr]: - """Keltner Channels (native Rust, uses high/low/close). - - Returns: - ``(upper, middle, lower)`` — three ``Expr`` objects. + Args: + h, l, c: Custom high/low/close columns. Defaults to native bar columns. """ - upper = Expr("KeltnerUpper", high, low, close, period, multiplier) - middle = close.ewm_mean(float(period)) - lower = Expr("KeltnerLower", high, low, close, period, multiplier) + return Expr("TrueRange", h or high, l or low, c or close) + + +def natr(period: int = 14, *, h: Expr = None, l: Expr = None, c: Expr = None) -> Expr: + """Normalized ATR (native Rust). + + Args: + h, l, c: Custom high/low/close columns. Defaults to native bar columns. + """ + return Expr("Natr", h or high, l or low, c or close, period) + + +def keltner_channels( + period: int = 20, multiplier: float = 1.5, + *, h: Expr = None, l: Expr = None, c: Expr = None, +) -> Tuple[Expr, Expr, Expr]: + """Keltner Channels (native Rust). + + Args: + h, l, c: Custom high/low/close columns. Defaults to native bar columns. + """ + _h, _l, _c = h or high, l or low, c or close + upper = Expr("KeltnerUpper", _h, _l, _c, period, multiplier) + middle = _c.ewm_mean(float(period)) + lower = Expr("KeltnerLower", _h, _l, _c, period, multiplier) return upper, middle, lower -def supertrend(period: int = 10, multiplier: float = 3.0) -> Expr: - """SuperTrend indicator (native Rust, uses high/low/close).""" - return Expr("SuperTrend", high, low, close, period, multiplier) +def supertrend( + period: int = 10, multiplier: float = 3.0, + *, h: Expr = None, l: Expr = None, c: Expr = None, +) -> Expr: + """SuperTrend indicator (native Rust). + + Args: + h, l, c: Custom high/low/close columns. Defaults to native bar columns. + """ + return Expr("SuperTrend", h or high, l or low, c or close, period, multiplier) # --------------------------------------------------------------------------- @@ -271,19 +309,31 @@ def obv(source: Expr = None, vol: Expr = None) -> Expr: vol if vol is not None else volume) -def vwap() -> Expr: - """Volume Weighted Average Price (native Rust, uses high/low/close/volume).""" - return Expr("Vwap", high, low, close, volume) +def vwap(*, h: Expr = None, l: Expr = None, c: Expr = None, v: Expr = None) -> Expr: + """Volume Weighted Average Price (native Rust). + + Args: + h, l, c, v: Custom high/low/close/volume columns. Defaults to native bar columns. + """ + return Expr("Vwap", h or high, l or low, c or close, v or volume) -def ad_line() -> Expr: - """Accumulation/Distribution Line (native Rust, uses high/low/close/volume).""" - return Expr("AdLine", high, low, close, volume) +def ad_line(*, h: Expr = None, l: Expr = None, c: Expr = None, v: Expr = None) -> Expr: + """Accumulation/Distribution Line (native Rust). + + Args: + h, l, c, v: Custom high/low/close/volume columns. Defaults to native bar columns. + """ + return Expr("AdLine", h or high, l or low, c or close, v or volume) -def mfi(period: int = 14) -> Expr: - """Money Flow Index (native Rust, uses high/low/close/volume).""" - return Expr("Mfi", high, low, close, volume, period) +def mfi(period: int = 14, *, h: Expr = None, l: Expr = None, c: Expr = None, v: Expr = None) -> Expr: + """Money Flow Index (native Rust). + + Args: + h, l, c, v: Custom high/low/close/volume columns. Defaults to native bar columns. + """ + return Expr("Mfi", h or high, l or low, c or close, v or volume, period) # --------------------------------------------------------------------------- diff --git a/python/manifoldbt/plot/chart.py b/python/manifoldbt/plot/chart.py index 913eb79..8e13c4e 100644 --- a/python/manifoldbt/plot/chart.py +++ b/python/manifoldbt/plot/chart.py @@ -66,26 +66,32 @@ def _load_bars( start_dt = datetime.fromtimestamp(start_ns / 1e9, tz=timezone.utc) end_dt = datetime.fromtimestamp(end_ns / 1e9, tz=timezone.utc) - tables = [] - day = start_dt.date() - end_day = end_dt.date() - while day <= end_day: - path = ( - data_root - / "bars_1m" - / str(symbol_id) - / str(day.year) - / f"{day.month:02d}" - / f"{day.day:02d}.parquet" - ) - if path.exists(): - tables.append(pq.read_table(str(path))) - day += timedelta(days=1) + # Try Arrow IPC file first (new layout), then Parquet partitions (legacy) + arrow_dir = Path(store.data_root()) / "mega" if not str(data_root).endswith("mega") else data_root + ipc_path = arrow_dir / "bars_1m" / f"{symbol_id}.arrow" + if ipc_path.exists(): + table = pa.ipc.open_file(str(ipc_path)).read_all() + else: + tables = [] + day = start_dt.date() + end_day = end_dt.date() + while day <= end_day: + path = ( + data_root + / "bars_1m" + / str(symbol_id) + / str(day.year) + / f"{day.month:02d}" + / f"{day.day:02d}.parquet" + ) + if path.exists(): + tables.append(pq.read_table(str(path))) + day += timedelta(days=1) - if not tables: - return {} + if not tables: + return {} - table = pa.concat_tables(tables) + table = pa.concat_tables(tables) # Filter to time range ts_col = table.column("timestamp").cast(pa.int64()).to_numpy(zero_copy_only=False)