mirror of
https://github.com/manifoldbt/manifoldbt.git
synced 2026-11-02 07:56:11 +00:00
3098 lines
108 KiB
JSON
3098 lines
108 KiB
JSON
{
|
|
"repo": "manifoldbt/manifoldbt",
|
|
"run_id": "40957995034",
|
|
"run_url": "https://github.com/manifoldbt/manifoldbt/actions/runs/40957995034",
|
|
"commit": "e53c0011378df87ae7e7e8ccd6ff0d8860b60fcd",
|
|
"run_started_at": "2026-10-29T21:27:46Z",
|
|
"conclusion": "success",
|
|
"synced_at": "2026-10-29T22:11:57.518906+00:00",
|
|
"backtests": {
|
|
"schema_version": 2,
|
|
"generated_at": "2026-10-29T22:11:44.969409+00:00",
|
|
"environment": {
|
|
"os": "Linux",
|
|
"os_release": "7.0.0-1017-azure",
|
|
"arch": "x86_64",
|
|
"cpu": "AMD EPYC 9V74 80-Core Processor",
|
|
"logical_cores": 4,
|
|
"ram_gb": 16.8,
|
|
"python": "3.12.15",
|
|
"versions": {
|
|
"manifoldbt": "0.31.1",
|
|
"vectorbt": "0.28.4",
|
|
"raptorbt": "0.9.0",
|
|
"nautilus_trader": "1.231.0",
|
|
"numpy": "2.4.3",
|
|
"numba": "0.64.0",
|
|
"pandas": "2.3.3"
|
|
},
|
|
"runner": "GitHub Actions 1000001317",
|
|
"ci": true,
|
|
"run_url": "https://github.com/manifoldbt/manifoldbt/actions/runs/40957995034",
|
|
"commit": "e53c0011378df87ae7e7e8ccd6ff0d8860b60fcd",
|
|
"pinned_cores": null,
|
|
"engines": [
|
|
"manifoldbt",
|
|
"vectorbt",
|
|
"raptorbt",
|
|
"nautilus_trader"
|
|
],
|
|
"engine_max_bars": {
|
|
"nautilus_trader": 100000
|
|
},
|
|
"engine_max_sweep_combos": {
|
|
"nautilus_trader": 64
|
|
}
|
|
},
|
|
"reference": "manifoldbt",
|
|
"engines": [
|
|
"manifoldbt",
|
|
"vectorbt",
|
|
"raptorbt",
|
|
"nautilus_trader"
|
|
],
|
|
"reps": 7,
|
|
"results": [
|
|
{
|
|
"workload": "sma_cross",
|
|
"title": "SMA 30/150 crossover, long-only, no cost",
|
|
"bars": 100000,
|
|
"data_digest": "f6e413e413e61a53",
|
|
"engines": [
|
|
"manifoldbt",
|
|
"vectorbt",
|
|
"raptorbt",
|
|
"nautilus_trader"
|
|
],
|
|
"parity": {
|
|
"vectorbt": {
|
|
"status": "exact",
|
|
"expected": "exact",
|
|
"publishable": true,
|
|
"diffs": {
|
|
"final_equity_vs_capital": 2.037268131971359e-15,
|
|
"round_trips_delta": 0,
|
|
"total_fees_vs_capital": 0.0
|
|
},
|
|
"metrics": {
|
|
"manifoldbt": {
|
|
"total_return": 0.06552457977030235,
|
|
"final_equity": 106552.45797703024,
|
|
"round_trips": 451,
|
|
"fills": 902,
|
|
"total_fees": 0.0
|
|
},
|
|
"vectorbt": {
|
|
"total_return": 0.06552457977030426,
|
|
"final_equity": 106552.45797703044,
|
|
"round_trips": 451,
|
|
"fills": null,
|
|
"total_fees": 0.0
|
|
}
|
|
},
|
|
"note": ""
|
|
},
|
|
"raptorbt": {
|
|
"status": "exact",
|
|
"expected": "exact",
|
|
"publishable": true,
|
|
"diffs": {
|
|
"final_equity_vs_capital": 0.0,
|
|
"round_trips_delta": 0,
|
|
"total_fees_vs_capital": 0.0
|
|
},
|
|
"metrics": {
|
|
"manifoldbt": {
|
|
"total_return": 0.06552457977030235,
|
|
"final_equity": 106552.45797703024,
|
|
"round_trips": 451,
|
|
"fills": 902,
|
|
"total_fees": 0.0
|
|
},
|
|
"raptorbt": {
|
|
"total_return": 0.06552457977030238,
|
|
"final_equity": 106552.45797703024,
|
|
"round_trips": 451,
|
|
"fills": null,
|
|
"total_fees": 0.0
|
|
}
|
|
},
|
|
"note": ""
|
|
},
|
|
"nautilus_trader": {
|
|
"status": "exact",
|
|
"expected": "exact",
|
|
"publishable": true,
|
|
"diffs": {
|
|
"final_equity_vs_capital": 4.330242518335581e-11,
|
|
"round_trips_delta": 0,
|
|
"total_fees_vs_capital": 0.0
|
|
},
|
|
"metrics": {
|
|
"manifoldbt": {
|
|
"total_return": 0.06552457977030235,
|
|
"final_equity": 106552.45797703024,
|
|
"round_trips": 451,
|
|
"fills": 902,
|
|
"total_fees": 0.0
|
|
},
|
|
"nautilus_trader": {
|
|
"total_return": 0.06552457972699988,
|
|
"final_equity": 106552.4579727,
|
|
"round_trips": 451,
|
|
"fills": 902,
|
|
"total_fees": 0.0
|
|
}
|
|
},
|
|
"note": ""
|
|
}
|
|
},
|
|
"status": "exact",
|
|
"paired_with": [
|
|
"sma_cross_metrics"
|
|
],
|
|
"timings": {
|
|
"manifoldbt": {
|
|
"samples_s": [
|
|
0.005003755999979376,
|
|
0.005694140999992214,
|
|
0.005790310000008958,
|
|
0.005752534000009746,
|
|
0.005784524000006286,
|
|
0.005812262000006285,
|
|
0.005868112999962705
|
|
],
|
|
"min_s": 0.005003755999979376,
|
|
"median_s": 0.005784524000006286,
|
|
"max_s": 0.005868112999962705,
|
|
"iqr_s": 0.00011812100001407089,
|
|
"iqr_over_median": 0.020420176321153224
|
|
},
|
|
"vectorbt": {
|
|
"samples_s": [
|
|
0.18963530800002104,
|
|
0.18369645900000364,
|
|
0.18261824599997567,
|
|
0.18764876099999128,
|
|
0.18897347399999376,
|
|
0.18172108699997125,
|
|
0.18642155999998522
|
|
],
|
|
"min_s": 0.18172108699997125,
|
|
"median_s": 0.18642155999998522,
|
|
"max_s": 0.18963530800002104,
|
|
"iqr_s": 0.006355228000018087,
|
|
"iqr_over_median": 0.034090627715048574
|
|
},
|
|
"raptorbt": {
|
|
"samples_s": [
|
|
0.01170275699999479,
|
|
0.009227943999974286,
|
|
0.011434043000008387,
|
|
0.010768489999975372,
|
|
0.01125242999998477,
|
|
0.008672665000005964,
|
|
0.010662802000013016
|
|
],
|
|
"min_s": 0.008672665000005964,
|
|
"median_s": 0.010768489999975372,
|
|
"max_s": 0.01170275699999479,
|
|
"iqr_s": 0.0022060990000341008,
|
|
"iqr_over_median": 0.20486614186753632
|
|
},
|
|
"nautilus_trader": {
|
|
"samples_s": [
|
|
5.385194051999974,
|
|
5.413725835000008,
|
|
5.582772408000011,
|
|
5.376771009999999,
|
|
5.389220787999989,
|
|
5.3655716350000375,
|
|
5.576347401000021
|
|
],
|
|
"min_s": 5.3655716350000375,
|
|
"median_s": 5.389220787999989,
|
|
"max_s": 5.582772408000011,
|
|
"iqr_s": 0.1995763910000221,
|
|
"iqr_over_median": 0.03703251339125179
|
|
}
|
|
},
|
|
"speedup": {
|
|
"vectorbt": {
|
|
"median_of_ratios": 32.260609458082406,
|
|
"min": 31.2651231138195,
|
|
"max": 37.89859217771664
|
|
},
|
|
"raptorbt": {
|
|
"median_of_ratios": 1.871955906728605,
|
|
"min": 1.4921324950589951,
|
|
"max": 2.3387944975820214
|
|
},
|
|
"nautilus_trader": {
|
|
"median_of_ratios": 950.2794852511296,
|
|
"min": 923.1468978848915,
|
|
"max": 1076.2303461683925
|
|
}
|
|
},
|
|
"noisy": true,
|
|
"cpu_over_wall": {
|
|
"manifoldbt": null,
|
|
"vectorbt": 1.014826542752928,
|
|
"raptorbt": null,
|
|
"nautilus_trader": 0.9997478805707817
|
|
}
|
|
},
|
|
{
|
|
"workload": "sma_cross_metrics",
|
|
"title": "SMA 30/150 crossover, with a performance summary",
|
|
"bars": 100000,
|
|
"data_digest": "f6e413e413e61a53",
|
|
"engines": [
|
|
"manifoldbt",
|
|
"vectorbt",
|
|
"raptorbt",
|
|
"nautilus_trader"
|
|
],
|
|
"parity": {
|
|
"vectorbt": {
|
|
"status": "exact",
|
|
"expected": "exact",
|
|
"publishable": true,
|
|
"diffs": {
|
|
"final_equity_vs_capital": 2.037268131971359e-15,
|
|
"round_trips_delta": 0,
|
|
"total_fees_vs_capital": 0.0,
|
|
"max_drawdown_rel": 1.4217536369471578e-15,
|
|
"advisory_ratio_rel": {
|
|
"sharpe": 0.019738981745476065,
|
|
"sortino": 0.028733597705797927,
|
|
"volatility": 0.006628159308143959
|
|
}
|
|
},
|
|
"metrics": {
|
|
"manifoldbt": {
|
|
"total_return": 0.06552457977030235,
|
|
"final_equity": 106552.45797703024,
|
|
"round_trips": 451,
|
|
"fills": 902,
|
|
"total_fees": 0.0,
|
|
"max_drawdown": -0.07808828448007843,
|
|
"sharpe": 2.046447206847637,
|
|
"sortino": 3.9171646026603506,
|
|
"volatility": 0.16872364573866735
|
|
},
|
|
"vectorbt": {
|
|
"total_return": 0.06552457977030426,
|
|
"final_equity": 106552.45797703044,
|
|
"round_trips": 451,
|
|
"fills": null,
|
|
"total_fees": 0.0,
|
|
"max_drawdown": -0.07808828448007832,
|
|
"sharpe": 2.087655398652483,
|
|
"sortino": 4.03304859862106,
|
|
"volatility": 0.16984943485125972
|
|
}
|
|
},
|
|
"note": ""
|
|
},
|
|
"raptorbt": {
|
|
"status": "exact",
|
|
"expected": "exact",
|
|
"publishable": true,
|
|
"diffs": {
|
|
"final_equity_vs_capital": 0.0,
|
|
"round_trips_delta": 0,
|
|
"total_fees_vs_capital": 0.0,
|
|
"max_drawdown_rel": 3.554384092367891e-16,
|
|
"ratio_basis": "native"
|
|
},
|
|
"metrics": {
|
|
"manifoldbt": {
|
|
"total_return": 0.06552457977030235,
|
|
"final_equity": 106552.45797703024,
|
|
"round_trips": 451,
|
|
"fills": 902,
|
|
"total_fees": 0.0,
|
|
"max_drawdown": -0.07808828448007843,
|
|
"sharpe": 2.046447206847637,
|
|
"sortino": 3.9171646026603506,
|
|
"volatility": 0.16872364573866735
|
|
},
|
|
"raptorbt": {
|
|
"total_return": 0.06552457977030238,
|
|
"final_equity": 106552.45797703024,
|
|
"round_trips": 451,
|
|
"fills": null,
|
|
"total_fees": 0.0,
|
|
"max_drawdown": -0.0780882844800784,
|
|
"sharpe": 0.9530483357635315,
|
|
"sortino": 1.3508000365056292,
|
|
"volatility": null
|
|
}
|
|
},
|
|
"note": ""
|
|
},
|
|
"nautilus_trader": {
|
|
"status": "exact",
|
|
"expected": "exact",
|
|
"publishable": true,
|
|
"diffs": {
|
|
"final_equity_vs_capital": 4.330242518335581e-11,
|
|
"round_trips_delta": 0,
|
|
"total_fees_vs_capital": 0.0,
|
|
"max_drawdown_absent": true,
|
|
"ratio_basis": "native"
|
|
},
|
|
"metrics": {
|
|
"manifoldbt": {
|
|
"total_return": 0.06552457977030235,
|
|
"final_equity": 106552.45797703024,
|
|
"round_trips": 451,
|
|
"fills": 902,
|
|
"total_fees": 0.0,
|
|
"max_drawdown": -0.07808828448007843,
|
|
"sharpe": 2.046447206847637,
|
|
"sortino": 3.9171646026603506,
|
|
"volatility": 0.16872364573866735
|
|
},
|
|
"nautilus_trader": {
|
|
"total_return": 0.06552457972699988,
|
|
"final_equity": 106552.4579727,
|
|
"round_trips": 451,
|
|
"fills": 902,
|
|
"total_fees": 0.0,
|
|
"max_drawdown": null,
|
|
"sharpe": 1.7883767980329837,
|
|
"sortino": 3.1735725831042556,
|
|
"volatility": 0.14751974319351674
|
|
}
|
|
},
|
|
"note": ""
|
|
}
|
|
},
|
|
"status": "exact",
|
|
"paired_with": [
|
|
"sma_cross"
|
|
],
|
|
"timings": {
|
|
"manifoldbt": {
|
|
"samples_s": [
|
|
0.005578705999994327,
|
|
0.005590863999998419,
|
|
0.0058770770000080574,
|
|
0.0058720909999863125,
|
|
0.005823872000007668,
|
|
0.005392368000002534,
|
|
0.005575446999955602
|
|
],
|
|
"min_s": 0.005392368000002534,
|
|
"median_s": 0.005590863999998419,
|
|
"max_s": 0.0058770770000080574,
|
|
"iqr_s": 0.00029664400003071023,
|
|
"iqr_over_median": 0.05305870434887955
|
|
},
|
|
"vectorbt": {
|
|
"samples_s": [
|
|
0.8975099569999827,
|
|
0.9007609230000071,
|
|
0.6978552140000147,
|
|
0.9109456930000022,
|
|
0.8980337310000266,
|
|
0.8951000849999673,
|
|
0.7060965200000169
|
|
],
|
|
"min_s": 0.6978552140000147,
|
|
"median_s": 0.8975099569999827,
|
|
"max_s": 0.9109456930000022,
|
|
"iqr_s": 0.19466440299999022,
|
|
"iqr_over_median": 0.21689386449892495
|
|
},
|
|
"raptorbt": {
|
|
"samples_s": [
|
|
0.00953471599999034,
|
|
0.011175519000005352,
|
|
0.011960388999995075,
|
|
0.011404220999992276,
|
|
0.00861020699997539,
|
|
0.010282331999974303,
|
|
0.01205954599998904
|
|
],
|
|
"min_s": 0.00861020699997539,
|
|
"median_s": 0.011175519000005352,
|
|
"max_s": 0.01205954599998904,
|
|
"iqr_s": 0.0024256730000047355,
|
|
"iqr_over_median": 0.21705238029693064
|
|
},
|
|
"nautilus_trader": {
|
|
"samples_s": [
|
|
5.364124597,
|
|
5.404177465999993,
|
|
5.557452631000018,
|
|
5.3959936489999905,
|
|
5.392923852000024,
|
|
5.386600496000028,
|
|
5.56775154799999
|
|
],
|
|
"min_s": 5.364124597,
|
|
"median_s": 5.3959936489999905,
|
|
"max_s": 5.56775154799999,
|
|
"iqr_s": 0.17085213499998986,
|
|
"iqr_over_median": 0.03166277540590748
|
|
}
|
|
},
|
|
"speedup": {
|
|
"vectorbt": {
|
|
"median_of_ratios": 155.1313991901906,
|
|
"min": 118.74188716585778,
|
|
"max": 165.99387968320164
|
|
},
|
|
"raptorbt": {
|
|
"median_of_ratios": 1.9421056315405976,
|
|
"min": 1.4784334202338327,
|
|
"max": 2.162973838704784
|
|
},
|
|
"nautilus_trader": {
|
|
"median_of_ratios": 961.535631561415,
|
|
"min": 918.9220073416043,
|
|
"max": 998.9304320472002
|
|
}
|
|
},
|
|
"noisy": true,
|
|
"cpu_over_wall": {
|
|
"manifoldbt": null,
|
|
"vectorbt": 0.9972398765487223,
|
|
"raptorbt": null,
|
|
"nautilus_trader": 1.00081367362624
|
|
}
|
|
},
|
|
{
|
|
"workload": "sma_cross",
|
|
"title": "SMA 30/150 crossover, long-only, no cost",
|
|
"bars": 1000000,
|
|
"data_digest": "ac3e442a94f3cd06",
|
|
"engines": [
|
|
"manifoldbt",
|
|
"vectorbt",
|
|
"raptorbt"
|
|
],
|
|
"parity": {
|
|
"vectorbt": {
|
|
"status": "exact",
|
|
"expected": "exact",
|
|
"publishable": true,
|
|
"diffs": {
|
|
"final_equity_vs_capital": 6.257323548197746e-15,
|
|
"round_trips_delta": 0,
|
|
"total_fees_vs_capital": 0.0
|
|
},
|
|
"metrics": {
|
|
"manifoldbt": {
|
|
"total_return": 0.1746603062593124,
|
|
"final_equity": 117466.03062593123,
|
|
"round_trips": 4539,
|
|
"fills": 9079,
|
|
"total_fees": 0.0
|
|
},
|
|
"vectorbt": {
|
|
"total_return": 0.1746603062593186,
|
|
"final_equity": 117466.03062593186,
|
|
"round_trips": 4539,
|
|
"fills": null,
|
|
"total_fees": 0.0
|
|
}
|
|
},
|
|
"note": ""
|
|
},
|
|
"raptorbt": {
|
|
"status": "exact",
|
|
"expected": "exact",
|
|
"publishable": true,
|
|
"diffs": {
|
|
"final_equity_vs_capital": 1.4551915228366853e-16,
|
|
"round_trips_delta": 0,
|
|
"total_fees_vs_capital": 0.0
|
|
},
|
|
"metrics": {
|
|
"manifoldbt": {
|
|
"total_return": 0.1746603062593124,
|
|
"final_equity": 117466.03062593123,
|
|
"round_trips": 4539,
|
|
"fills": 9079,
|
|
"total_fees": 0.0
|
|
},
|
|
"raptorbt": {
|
|
"total_return": 0.1746603062593125,
|
|
"final_equity": 117466.03062593125,
|
|
"round_trips": 4539,
|
|
"fills": null,
|
|
"total_fees": 0.0
|
|
}
|
|
},
|
|
"note": ""
|
|
}
|
|
},
|
|
"status": "exact",
|
|
"unsupported": {
|
|
"nautilus_trader": "Not run above 100,000 bars. NautilusTrader is event-driven: every bar goes through a simulated exchange and a Python strategy callback, so one call costs seconds per 100,000 bars and grows linearly with the series. Repeated and interleaved, a longer series does not fit the time budget of the job. The measured time at the ceiling is in the table, and `bench.py --engine-max-bars nautilus_trader=N` lifts the ceiling for a local run."
|
|
},
|
|
"beyond_ceiling": {
|
|
"nautilus_trader": 100000
|
|
},
|
|
"paired_with": [
|
|
"sma_cross_metrics"
|
|
],
|
|
"timings": {
|
|
"manifoldbt": {
|
|
"samples_s": [
|
|
0.03465895200002933,
|
|
0.03271708900001613,
|
|
0.031551861999957964,
|
|
0.03358030100002907,
|
|
0.031368079999992915,
|
|
0.031333500000016556,
|
|
0.031780814999990525
|
|
],
|
|
"min_s": 0.031333500000016556,
|
|
"median_s": 0.031780814999990525,
|
|
"max_s": 0.03465895200002933,
|
|
"iqr_s": 0.002212221000036152,
|
|
"iqr_over_median": 0.06960869317029193
|
|
},
|
|
"vectorbt": {
|
|
"samples_s": [
|
|
2.3673678570000334,
|
|
2.0574110539999992,
|
|
2.3647311710000167,
|
|
2.0408310270000243,
|
|
2.350951108000004,
|
|
2.043063800000027,
|
|
2.322806449999973
|
|
],
|
|
"min_s": 2.0408310270000243,
|
|
"median_s": 2.322806449999973,
|
|
"max_s": 2.3673678570000334,
|
|
"iqr_s": 0.32166737099998954,
|
|
"iqr_over_median": 0.13848221017295403
|
|
},
|
|
"raptorbt": {
|
|
"samples_s": [
|
|
0.110731573999999,
|
|
0.08843513599998687,
|
|
0.11148736500001633,
|
|
0.08842377800004897,
|
|
0.11201640299998417,
|
|
0.08779423499998984,
|
|
0.11281501500002378
|
|
],
|
|
"min_s": 0.08779423499998984,
|
|
"median_s": 0.110731573999999,
|
|
"max_s": 0.11281501500002378,
|
|
"iqr_s": 0.023592624999935197,
|
|
"iqr_over_median": 0.2130614073988997
|
|
}
|
|
},
|
|
"speedup": {
|
|
"vectorbt": {
|
|
"median_of_ratios": 68.30465782687341,
|
|
"min": 60.77464960776432,
|
|
"max": 74.94743641447111
|
|
},
|
|
"raptorbt": {
|
|
"median_of_ratios": 3.1948910053571526,
|
|
"min": 2.6332038536513545,
|
|
"max": 3.571031539068042
|
|
}
|
|
},
|
|
"noisy": true,
|
|
"cpu_over_wall": {
|
|
"manifoldbt": null,
|
|
"vectorbt": 0.9988961029360517,
|
|
"raptorbt": 1.0116572335669969
|
|
}
|
|
},
|
|
{
|
|
"workload": "sma_cross_metrics",
|
|
"title": "SMA 30/150 crossover, with a performance summary",
|
|
"bars": 1000000,
|
|
"data_digest": "ac3e442a94f3cd06",
|
|
"engines": [
|
|
"manifoldbt",
|
|
"vectorbt",
|
|
"raptorbt"
|
|
],
|
|
"parity": {
|
|
"vectorbt": {
|
|
"status": "exact",
|
|
"expected": "exact",
|
|
"publishable": true,
|
|
"diffs": {
|
|
"final_equity_vs_capital": 6.257323548197746e-15,
|
|
"round_trips_delta": 0,
|
|
"total_fees_vs_capital": 0.0,
|
|
"max_drawdown_rel": 1.1589115435726748e-14,
|
|
"advisory_ratio_rel": {
|
|
"sharpe": 0.0049912091455338835,
|
|
"sortino": 0.006381194164092784,
|
|
"volatility": 0.00036567995436717903
|
|
}
|
|
},
|
|
"metrics": {
|
|
"manifoldbt": {
|
|
"total_return": 0.1746603062593124,
|
|
"final_equity": 117466.03062593123,
|
|
"round_trips": 4539,
|
|
"fills": 9079,
|
|
"total_fees": 0.0,
|
|
"max_drawdown": -0.1532780348294871,
|
|
"sharpe": 0.6060293940216315,
|
|
"sortino": 1.1883562254598439,
|
|
"volatility": 0.1608771511259405
|
|
},
|
|
"vectorbt": {
|
|
"total_return": 0.1746603062593186,
|
|
"final_equity": 117466.03062593186,
|
|
"round_trips": 4539,
|
|
"fills": null,
|
|
"total_fees": 0.0,
|
|
"max_drawdown": -0.15327803482948532,
|
|
"sharpe": 0.6090693867148674,
|
|
"sortino": 1.1959880574725121,
|
|
"volatility": 0.16093600219587953
|
|
}
|
|
},
|
|
"note": ""
|
|
},
|
|
"raptorbt": {
|
|
"status": "exact",
|
|
"expected": "exact",
|
|
"publishable": true,
|
|
"diffs": {
|
|
"final_equity_vs_capital": 1.4551915228366853e-16,
|
|
"round_trips_delta": 0,
|
|
"total_fees_vs_capital": 0.0,
|
|
"max_drawdown_rel": 1.810799286832283e-16,
|
|
"ratio_basis": "native"
|
|
},
|
|
"metrics": {
|
|
"manifoldbt": {
|
|
"total_return": 0.1746603062593124,
|
|
"final_equity": 117466.03062593123,
|
|
"round_trips": 4539,
|
|
"fills": 9079,
|
|
"total_fees": 0.0,
|
|
"max_drawdown": -0.1532780348294871,
|
|
"sharpe": 0.6060293940216315,
|
|
"sortino": 1.1883562254598439,
|
|
"volatility": 0.1608771511259405
|
|
},
|
|
"raptorbt": {
|
|
"total_return": 0.1746603062593125,
|
|
"final_equity": 117466.03062593125,
|
|
"round_trips": 4539,
|
|
"fills": null,
|
|
"total_fees": 0.0,
|
|
"max_drawdown": -0.15327803482948713,
|
|
"sharpe": 0.2661390460797749,
|
|
"sortino": 0.37666881597859997,
|
|
"volatility": null
|
|
}
|
|
},
|
|
"note": ""
|
|
}
|
|
},
|
|
"status": "exact",
|
|
"unsupported": {
|
|
"nautilus_trader": "Not run above 100,000 bars. NautilusTrader is event-driven: every bar goes through a simulated exchange and a Python strategy callback, so one call costs seconds per 100,000 bars and grows linearly with the series. Repeated and interleaved, a longer series does not fit the time budget of the job. The measured time at the ceiling is in the table, and `bench.py --engine-max-bars nautilus_trader=N` lifts the ceiling for a local run."
|
|
},
|
|
"beyond_ceiling": {
|
|
"nautilus_trader": 100000
|
|
},
|
|
"paired_with": [
|
|
"sma_cross"
|
|
],
|
|
"timings": {
|
|
"manifoldbt": {
|
|
"samples_s": [
|
|
0.03331567599997243,
|
|
0.03211887499998056,
|
|
0.03344631199996684,
|
|
0.03196384199998192,
|
|
0.032142564000025686,
|
|
0.033810189999996965,
|
|
0.032156108999970456
|
|
],
|
|
"min_s": 0.03196384199998192,
|
|
"median_s": 0.032156108999970456,
|
|
"max_s": 0.033810189999996965,
|
|
"iqr_s": 0.0013274369999862756,
|
|
"iqr_over_median": 0.04128102066041309
|
|
},
|
|
"vectorbt": {
|
|
"samples_s": [
|
|
8.631613638000033,
|
|
8.61205582599996,
|
|
8.589484955999978,
|
|
8.548467796000011,
|
|
8.579728159000013,
|
|
8.707297460000007,
|
|
8.698635926999998
|
|
],
|
|
"min_s": 8.548467796000011,
|
|
"median_s": 8.61205582599996,
|
|
"max_s": 8.707297460000007,
|
|
"iqr_s": 0.11890776799998548,
|
|
"iqr_over_median": 0.013807129261865753
|
|
},
|
|
"raptorbt": {
|
|
"samples_s": [
|
|
0.12047529400001622,
|
|
0.12149541399998043,
|
|
0.12105698999999959,
|
|
0.12132746499997893,
|
|
0.12275237500000458,
|
|
0.12201715800000557,
|
|
0.12147080200003302
|
|
],
|
|
"min_s": 0.12047529400001622,
|
|
"median_s": 0.12147080200003302,
|
|
"max_s": 0.12275237500000458,
|
|
"iqr_s": 0.0009601680000059787,
|
|
"iqr_over_median": 0.0079045168402339
|
|
}
|
|
},
|
|
"speedup": {
|
|
"vectorbt": {
|
|
"median_of_ratios": 266.927310434636,
|
|
"min": 256.8141131975476,
|
|
"max": 270.5127018635243
|
|
},
|
|
"raptorbt": {
|
|
"median_of_ratios": 3.777534215975715,
|
|
"min": 3.6088870840423115,
|
|
"max": 3.8189976070330443
|
|
}
|
|
},
|
|
"noisy": false,
|
|
"cpu_over_wall": {
|
|
"manifoldbt": null,
|
|
"vectorbt": 1.0002106478411412,
|
|
"raptorbt": 0.9875434351287348
|
|
}
|
|
},
|
|
{
|
|
"workload": "sma_cross",
|
|
"title": "SMA 30/150 crossover, long-only, no cost",
|
|
"bars": 10000000,
|
|
"data_digest": "73123cc435bd1913",
|
|
"engines": [
|
|
"manifoldbt",
|
|
"vectorbt",
|
|
"raptorbt"
|
|
],
|
|
"parity": {
|
|
"vectorbt": {
|
|
"status": "exact",
|
|
"expected": "exact",
|
|
"publishable": true,
|
|
"diffs": {
|
|
"final_equity_vs_capital": 1.367880031466484e-14,
|
|
"round_trips_delta": 0,
|
|
"total_fees_vs_capital": 0.0
|
|
},
|
|
"metrics": {
|
|
"manifoldbt": {
|
|
"total_return": 0.03788615047806765,
|
|
"final_equity": 103788.61504780677,
|
|
"round_trips": 46112,
|
|
"fills": 92224,
|
|
"total_fees": 0.0
|
|
},
|
|
"vectorbt": {
|
|
"total_return": 0.03788615047805419,
|
|
"final_equity": 103788.6150478054,
|
|
"round_trips": 46112,
|
|
"fills": null,
|
|
"total_fees": 0.0
|
|
}
|
|
},
|
|
"note": ""
|
|
},
|
|
"raptorbt": {
|
|
"status": "exact",
|
|
"expected": "exact",
|
|
"publishable": true,
|
|
"diffs": {
|
|
"final_equity_vs_capital": 1.4551915228366853e-16,
|
|
"round_trips_delta": 0,
|
|
"total_fees_vs_capital": 0.0
|
|
},
|
|
"metrics": {
|
|
"manifoldbt": {
|
|
"total_return": 0.03788615047806765,
|
|
"final_equity": 103788.61504780677,
|
|
"round_trips": 46112,
|
|
"fills": 92224,
|
|
"total_fees": 0.0
|
|
},
|
|
"raptorbt": {
|
|
"total_return": 0.03788615047806757,
|
|
"final_equity": 103788.61504780676,
|
|
"round_trips": 46112,
|
|
"fills": null,
|
|
"total_fees": 0.0
|
|
}
|
|
},
|
|
"note": ""
|
|
}
|
|
},
|
|
"status": "exact",
|
|
"unsupported": {
|
|
"nautilus_trader": "Not run above 100,000 bars. NautilusTrader is event-driven: every bar goes through a simulated exchange and a Python strategy callback, so one call costs seconds per 100,000 bars and grows linearly with the series. Repeated and interleaved, a longer series does not fit the time budget of the job. The measured time at the ceiling is in the table, and `bench.py --engine-max-bars nautilus_trader=N` lifts the ceiling for a local run."
|
|
},
|
|
"beyond_ceiling": {
|
|
"nautilus_trader": 100000
|
|
},
|
|
"paired_with": [
|
|
"sma_cross_metrics"
|
|
],
|
|
"timings": {
|
|
"manifoldbt": {
|
|
"samples_s": [
|
|
0.25625206699999126,
|
|
0.2521691389999887,
|
|
0.2541436660000045,
|
|
0.2495417400000406,
|
|
0.24472355600005358,
|
|
0.24591852999992625,
|
|
0.24932512500004123
|
|
],
|
|
"min_s": 0.24472355600005358,
|
|
"median_s": 0.2495417400000406,
|
|
"max_s": 0.25625206699999126,
|
|
"iqr_s": 0.008225136000078237,
|
|
"iqr_over_median": 0.032960962763491586
|
|
},
|
|
"vectorbt": {
|
|
"samples_s": [
|
|
26.66456360799998,
|
|
25.92168389599999,
|
|
26.682199731999958,
|
|
26.088747322000017,
|
|
26.758494357000018,
|
|
25.891162434999842,
|
|
26.777559189000158
|
|
],
|
|
"min_s": 25.891162434999842,
|
|
"median_s": 26.66456360799998,
|
|
"max_s": 26.777559189000158,
|
|
"iqr_s": 0.8368104610000273,
|
|
"iqr_over_median": 0.03138286728791485
|
|
},
|
|
"raptorbt": {
|
|
"samples_s": [
|
|
1.6040951090000135,
|
|
1.6015829319999284,
|
|
1.6075852010000062,
|
|
1.6096403850000343,
|
|
1.6126825730000292,
|
|
1.6159527730001173,
|
|
1.6120409280001695
|
|
],
|
|
"min_s": 1.6015829319999284,
|
|
"median_s": 1.6096403850000343,
|
|
"max_s": 1.6159527730001173,
|
|
"iqr_s": 0.008587464000015643,
|
|
"iqr_over_median": 0.005335020219448247
|
|
}
|
|
},
|
|
"speedup": {
|
|
"vectorbt": {
|
|
"median_of_ratios": 104.98864737396008,
|
|
"min": 102.7948304808272,
|
|
"max": 109.34171926217907
|
|
},
|
|
"raptorbt": {
|
|
"median_of_ratios": 6.450385354369063,
|
|
"min": 6.25983285824605,
|
|
"max": 6.589813417878236
|
|
}
|
|
},
|
|
"noisy": false,
|
|
"cpu_over_wall": {
|
|
"manifoldbt": 1.1300893683860138,
|
|
"vectorbt": 0.9998678971947417,
|
|
"raptorbt": 1.0014578046362215
|
|
}
|
|
},
|
|
{
|
|
"workload": "sma_cross_metrics",
|
|
"title": "SMA 30/150 crossover, with a performance summary",
|
|
"bars": 10000000,
|
|
"data_digest": "73123cc435bd1913",
|
|
"engines": [
|
|
"manifoldbt",
|
|
"vectorbt",
|
|
"raptorbt"
|
|
],
|
|
"parity": {
|
|
"vectorbt": {
|
|
"status": "exact",
|
|
"expected": "exact",
|
|
"publishable": true,
|
|
"diffs": {
|
|
"final_equity_vs_capital": 1.367880031466484e-14,
|
|
"round_trips_delta": 0,
|
|
"total_fees_vs_capital": 0.0,
|
|
"max_drawdown_rel": 5.4536763289837e-15,
|
|
"advisory_ratio_rel": {
|
|
"sharpe": 0.0029322134925119526,
|
|
"sortino": 0.0030751996753319386,
|
|
"volatility": 0.00027124700216687384
|
|
}
|
|
},
|
|
"metrics": {
|
|
"manifoldbt": {
|
|
"total_return": 0.03788615047806765,
|
|
"final_equity": 103788.61504780677,
|
|
"round_trips": 46112,
|
|
"fills": 92224,
|
|
"total_fees": 0.0,
|
|
"max_drawdown": -0.6717919730204706,
|
|
"sharpe": 0.09022600140504972,
|
|
"sortino": 0.17523697483973677,
|
|
"volatility": 0.15574452123910787
|
|
},
|
|
"vectorbt": {
|
|
"total_return": 0.03788615047805419,
|
|
"final_equity": 103788.6150478054,
|
|
"round_trips": 46112,
|
|
"fills": null,
|
|
"total_fees": 0.0,
|
|
"max_drawdown": -0.6717919730204742,
|
|
"sharpe": 0.09049134133707379,
|
|
"sortino": 0.17577752583010015,
|
|
"volatility": 0.1557022874604037
|
|
}
|
|
},
|
|
"note": ""
|
|
},
|
|
"raptorbt": {
|
|
"status": "exact",
|
|
"expected": "exact",
|
|
"publishable": true,
|
|
"diffs": {
|
|
"final_equity_vs_capital": 1.4551915228366853e-16,
|
|
"round_trips_delta": 0,
|
|
"total_fees_vs_capital": 0.0,
|
|
"max_drawdown_rel": 0.0,
|
|
"ratio_basis": "native"
|
|
},
|
|
"metrics": {
|
|
"manifoldbt": {
|
|
"total_return": 0.03788615047806765,
|
|
"final_equity": 103788.61504780677,
|
|
"round_trips": 46112,
|
|
"fills": 92224,
|
|
"total_fees": 0.0,
|
|
"max_drawdown": -0.6717919730204706,
|
|
"sharpe": 0.09022600140504972,
|
|
"sortino": 0.17523697483973677,
|
|
"volatility": 0.15574452123910787
|
|
},
|
|
"raptorbt": {
|
|
"total_return": 0.03788615047806757,
|
|
"final_equity": 103788.61504780676,
|
|
"round_trips": 46112,
|
|
"fills": null,
|
|
"total_fees": 0.0,
|
|
"max_drawdown": -0.6717919730204706,
|
|
"sharpe": 0.037977702535686,
|
|
"sortino": 0.0537122315884283,
|
|
"volatility": null
|
|
}
|
|
},
|
|
"note": ""
|
|
}
|
|
},
|
|
"status": "exact",
|
|
"unsupported": {
|
|
"nautilus_trader": "Not run above 100,000 bars. NautilusTrader is event-driven: every bar goes through a simulated exchange and a Python strategy callback, so one call costs seconds per 100,000 bars and grows linearly with the series. Repeated and interleaved, a longer series does not fit the time budget of the job. The measured time at the ceiling is in the table, and `bench.py --engine-max-bars nautilus_trader=N` lifts the ceiling for a local run."
|
|
},
|
|
"beyond_ceiling": {
|
|
"nautilus_trader": 100000
|
|
},
|
|
"paired_with": [
|
|
"sma_cross"
|
|
],
|
|
"timings": {
|
|
"manifoldbt": {
|
|
"samples_s": [
|
|
0.25170789000003424,
|
|
0.25158246099999815,
|
|
0.24940107800000533,
|
|
0.24961206899990884,
|
|
0.2560705490000146,
|
|
0.2538790969999809,
|
|
0.25119612399998914
|
|
],
|
|
"min_s": 0.24940107800000533,
|
|
"median_s": 0.25158246099999815,
|
|
"max_s": 0.2560705490000146,
|
|
"iqr_s": 0.0042670280000720595,
|
|
"iqr_over_median": 0.01696075307917467
|
|
},
|
|
"vectorbt": {
|
|
"samples_s": [
|
|
102.34788693400003,
|
|
101.68348374000004,
|
|
102.11958061000007,
|
|
101.466101084,
|
|
102.57090565199996,
|
|
101.63180017700006,
|
|
102.86496043399984
|
|
],
|
|
"min_s": 101.466101084,
|
|
"median_s": 102.11958061000007,
|
|
"max_s": 102.86496043399984,
|
|
"iqr_s": 0.9391054749999057,
|
|
"iqr_over_median": 0.009196135250362981
|
|
},
|
|
"raptorbt": {
|
|
"samples_s": [
|
|
1.608975332,
|
|
1.608635364999941,
|
|
1.6003084800000806,
|
|
1.6032221919999756,
|
|
1.6079026459999568,
|
|
1.603872722999995,
|
|
1.599719481999955
|
|
],
|
|
"min_s": 1.599719481999955,
|
|
"median_s": 1.603872722999995,
|
|
"max_s": 1.608975332,
|
|
"iqr_s": 0.00832688499986034,
|
|
"iqr_over_median": 0.005191736775898998
|
|
}
|
|
},
|
|
"speedup": {
|
|
"vectorbt": {
|
|
"median_of_ratios": 406.4951726514356,
|
|
"min": 400.315746266451,
|
|
"max": 409.5005878116347
|
|
},
|
|
"raptorbt": {
|
|
"median_of_ratios": 6.392232408764704,
|
|
"min": 6.279139292976113,
|
|
"max": 6.422855266668059
|
|
}
|
|
},
|
|
"noisy": false,
|
|
"cpu_over_wall": {
|
|
"manifoldbt": 1.122799524732398,
|
|
"vectorbt": 0.99988146013369,
|
|
"raptorbt": 1.001545833022658
|
|
}
|
|
},
|
|
{
|
|
"workload": "ema_rsi_fees",
|
|
"title": "EMA 12/26 crossover + RSI(14) filter, 5 bps taker fee",
|
|
"bars": 100000,
|
|
"data_digest": "f6e413e413e61a53",
|
|
"engines": [
|
|
"manifoldbt",
|
|
"vectorbt",
|
|
"nautilus_trader"
|
|
],
|
|
"parity": {
|
|
"vectorbt": {
|
|
"status": "exact",
|
|
"expected": "exact",
|
|
"publishable": true,
|
|
"diffs": {
|
|
"final_equity_vs_capital": 0.0,
|
|
"round_trips_delta": 0,
|
|
"total_fees_vs_capital": 1.1368683772161604e-17
|
|
},
|
|
"metrics": {
|
|
"manifoldbt": {
|
|
"total_return": -0.01566350876078959,
|
|
"final_equity": 98433.64912392104,
|
|
"round_trips": 3176,
|
|
"fills": 6352,
|
|
"total_fees": 1539.0168535292935
|
|
},
|
|
"vectorbt": {
|
|
"total_return": -0.015663508760789594,
|
|
"final_equity": 98433.64912392104,
|
|
"round_trips": 3176,
|
|
"fills": null,
|
|
"total_fees": 1539.0168535292923
|
|
}
|
|
},
|
|
"note": ""
|
|
},
|
|
"nautilus_trader": {
|
|
"status": "exact",
|
|
"expected": "exact",
|
|
"publishable": true,
|
|
"diffs": {
|
|
"final_equity_vs_capital": 3.71044734492898e-12,
|
|
"round_trips_delta": 0,
|
|
"total_fees_vs_capital": 4.071057446708437e-13
|
|
},
|
|
"metrics": {
|
|
"manifoldbt": {
|
|
"total_return": -0.01566350876078959,
|
|
"final_equity": 98433.64912392104,
|
|
"round_trips": 3176,
|
|
"fills": 6352,
|
|
"total_fees": 1539.0168535292935
|
|
},
|
|
"nautilus_trader": {
|
|
"total_return": -0.015663508764499956,
|
|
"final_equity": 98433.64912355,
|
|
"round_trips": 3176,
|
|
"fills": 6352,
|
|
"total_fees": 1539.016853570004
|
|
}
|
|
},
|
|
"note": ""
|
|
}
|
|
},
|
|
"status": "exact",
|
|
"unsupported": {
|
|
"raptorbt": "Two blockers, either one sufficient. Sizing: raptorbt has no fixed-quantity mode. `position_sizes` is a fraction of equity (measured: 0.5 buys exactly half the equity of the bar before the entry), `lot_size` rounds a computed size down to a multiple, and `alloted_capital` fixes the notional, not the quantity. Reproducing `units=5` would mean feeding a fraction derived from an equity curve that does not exist until the run is over. Indicator: `raptorbt.ema` seeds on a simple mean of the first `period` bars and emits from bar `period-1`, while manifoldbt seeds on the first observation and emits from bar 0 (measured: 11 leading NaN for span 12, and ema[11] equal to sma(12)[11] to the last bit). The two are the same recursion with a different warmup, so the signal differs early and the round-trip count with it. Its `sma` and `rsi` do match, which is why the other three workloads run."
|
|
},
|
|
"timings": {
|
|
"manifoldbt": {
|
|
"samples_s": [
|
|
0.007998699999916425,
|
|
0.007399608000014268,
|
|
0.007736864999969839,
|
|
0.006715642000017397,
|
|
0.006709364000016649,
|
|
0.007591843999989578,
|
|
0.007412468999973498
|
|
],
|
|
"min_s": 0.006709364000016649,
|
|
"median_s": 0.007412468999973498,
|
|
"max_s": 0.007998699999916425,
|
|
"iqr_s": 0.0010212229999524425,
|
|
"iqr_over_median": 0.137770964027788
|
|
},
|
|
"vectorbt": {
|
|
"samples_s": [
|
|
0.19333602799997607,
|
|
0.19119976199999655,
|
|
0.19926081600010548,
|
|
0.19566461400017943,
|
|
0.19351609899990763,
|
|
0.18901162299994212,
|
|
0.19730058200002532
|
|
],
|
|
"min_s": 0.18901162299994212,
|
|
"median_s": 0.19351609899990763,
|
|
"max_s": 0.19926081600010548,
|
|
"iqr_s": 0.006100820000028762,
|
|
"iqr_over_median": 0.0315261625857375
|
|
},
|
|
"nautilus_trader": {
|
|
"samples_s": [
|
|
11.840473814000006,
|
|
11.77435276999995,
|
|
11.79245008099997,
|
|
11.814377443000012,
|
|
11.802522227000054,
|
|
11.799825139999939,
|
|
11.787171551000029
|
|
],
|
|
"min_s": 11.77435276999995,
|
|
"median_s": 11.799825139999939,
|
|
"max_s": 11.840473814000006,
|
|
"iqr_s": 0.027205891999983578,
|
|
"iqr_over_median": 0.0023056182339311953
|
|
}
|
|
},
|
|
"speedup": {
|
|
"vectorbt": {
|
|
"median_of_ratios": 25.83917445351536,
|
|
"min": 24.170931276581964,
|
|
"max": 29.135652853393996
|
|
},
|
|
"nautilus_trader": {
|
|
"median_of_ratios": 1590.1815644749638,
|
|
"min": 1480.2997754789806,
|
|
"max": 1759.2327647854677
|
|
}
|
|
},
|
|
"noisy": false,
|
|
"cpu_over_wall": {
|
|
"manifoldbt": null,
|
|
"vectorbt": 1.0005226818768154,
|
|
"nautilus_trader": 0.9999858006374122
|
|
}
|
|
},
|
|
{
|
|
"workload": "ema_rsi_fees",
|
|
"title": "EMA 12/26 crossover + RSI(14) filter, 5 bps taker fee",
|
|
"bars": 1000000,
|
|
"data_digest": "ac3e442a94f3cd06",
|
|
"engines": [
|
|
"manifoldbt",
|
|
"vectorbt"
|
|
],
|
|
"parity": {
|
|
"vectorbt": {
|
|
"status": "exact",
|
|
"expected": "exact",
|
|
"publishable": true,
|
|
"diffs": {
|
|
"final_equity_vs_capital": 6.548361852765083e-15,
|
|
"round_trips_delta": 0,
|
|
"total_fees_vs_capital": 1.0913936421275139e-16
|
|
},
|
|
"metrics": {
|
|
"manifoldbt": {
|
|
"total_return": -0.14995537166228978,
|
|
"final_equity": 85004.46283377102,
|
|
"round_trips": 31383,
|
|
"fills": 62767,
|
|
"total_fees": 14870.28066032091
|
|
},
|
|
"vectorbt": {
|
|
"total_return": -0.14995537166229642,
|
|
"final_equity": 85004.46283377036,
|
|
"round_trips": 31383,
|
|
"fills": null,
|
|
"total_fees": 14870.2806603209
|
|
}
|
|
},
|
|
"note": ""
|
|
}
|
|
},
|
|
"status": "exact",
|
|
"unsupported": {
|
|
"raptorbt": "Two blockers, either one sufficient. Sizing: raptorbt has no fixed-quantity mode. `position_sizes` is a fraction of equity (measured: 0.5 buys exactly half the equity of the bar before the entry), `lot_size` rounds a computed size down to a multiple, and `alloted_capital` fixes the notional, not the quantity. Reproducing `units=5` would mean feeding a fraction derived from an equity curve that does not exist until the run is over. Indicator: `raptorbt.ema` seeds on a simple mean of the first `period` bars and emits from bar `period-1`, while manifoldbt seeds on the first observation and emits from bar 0 (measured: 11 leading NaN for span 12, and ema[11] equal to sma(12)[11] to the last bit). The two are the same recursion with a different warmup, so the signal differs early and the round-trip count with it. Its `sma` and `rsi` do match, which is why the other three workloads run.",
|
|
"nautilus_trader": "Not run above 100,000 bars. NautilusTrader is event-driven: every bar goes through a simulated exchange and a Python strategy callback, so one call costs seconds per 100,000 bars and grows linearly with the series. Repeated and interleaved, a longer series does not fit the time budget of the job. The measured time at the ceiling is in the table, and `bench.py --engine-max-bars nautilus_trader=N` lifts the ceiling for a local run."
|
|
},
|
|
"beyond_ceiling": {
|
|
"nautilus_trader": 100000
|
|
},
|
|
"timings": {
|
|
"manifoldbt": {
|
|
"samples_s": [
|
|
0.05033930500007955,
|
|
0.046616243000016766,
|
|
0.047160177000023396,
|
|
0.0457901459999448,
|
|
0.04675854900006016,
|
|
0.04688864500008094,
|
|
0.04841835900015212
|
|
],
|
|
"min_s": 0.0457901459999448,
|
|
"median_s": 0.04688864500008094,
|
|
"max_s": 0.05033930500007955,
|
|
"iqr_s": 0.0018021160001353564,
|
|
"iqr_over_median": 0.03843395346852624
|
|
},
|
|
"vectorbt": {
|
|
"samples_s": [
|
|
2.160681984000121,
|
|
2.529209142999889,
|
|
2.17315201800011,
|
|
2.5243838760000017,
|
|
2.1602348619999248,
|
|
2.5232705769999484,
|
|
2.1554850449999776
|
|
],
|
|
"min_s": 2.1554850449999776,
|
|
"median_s": 2.17315201800011,
|
|
"max_s": 2.529209142999889,
|
|
"iqr_s": 0.36414901400007693,
|
|
"iqr_over_median": 0.16756720698039015
|
|
}
|
|
},
|
|
"speedup": {
|
|
"vectorbt": {
|
|
"median_of_ratios": 46.19978395816229,
|
|
"min": 42.92236422407314,
|
|
"max": 55.12941312751099
|
|
}
|
|
},
|
|
"noisy": true,
|
|
"cpu_over_wall": {
|
|
"manifoldbt": null,
|
|
"vectorbt": 0.9989882236793732
|
|
}
|
|
},
|
|
{
|
|
"workload": "sma_cross_costs",
|
|
"title": "SMA 30/150 with a 5 bps fee and 2 bps of slippage",
|
|
"bars": 100000,
|
|
"data_digest": "f6e413e413e61a53",
|
|
"engines": [
|
|
"manifoldbt",
|
|
"vectorbt",
|
|
"nautilus_trader"
|
|
],
|
|
"parity": {
|
|
"vectorbt": {
|
|
"status": "exact",
|
|
"expected": "exact",
|
|
"publishable": true,
|
|
"diffs": {
|
|
"final_equity_vs_capital": 2.037268131971359e-15,
|
|
"round_trips_delta": 0,
|
|
"total_fees_vs_capital": 8.526512829121202e-19
|
|
},
|
|
"metrics": {
|
|
"manifoldbt": {
|
|
"total_return": -0.0027521446989833276,
|
|
"final_equity": 99724.78553010167,
|
|
"round_trips": 451,
|
|
"fills": 902,
|
|
"total_fees": 218.4336288899406
|
|
},
|
|
"vectorbt": {
|
|
"total_return": -0.0027521446989813188,
|
|
"final_equity": 99724.78553010187,
|
|
"round_trips": 451,
|
|
"fills": null,
|
|
"total_fees": 218.43362888994068
|
|
}
|
|
},
|
|
"note": ""
|
|
},
|
|
"nautilus_trader": {
|
|
"status": "exact",
|
|
"expected": "exact",
|
|
"publishable": true,
|
|
"diffs": {
|
|
"final_equity_vs_capital": 1.658918336033821e-14,
|
|
"round_trips_delta": 0,
|
|
"total_fees_vs_capital": 4.0059433104033813e-13
|
|
},
|
|
"metrics": {
|
|
"manifoldbt": {
|
|
"total_return": -0.0027521446989833276,
|
|
"final_equity": 99724.78553010167,
|
|
"round_trips": 451,
|
|
"fills": 902,
|
|
"total_fees": 218.4336288899406
|
|
},
|
|
"nautilus_trader": {
|
|
"total_return": -0.00275214469899987,
|
|
"final_equity": 99724.78553010001,
|
|
"round_trips": 451,
|
|
"fills": 902,
|
|
"total_fees": 218.43362893000003
|
|
}
|
|
},
|
|
"note": ""
|
|
}
|
|
},
|
|
"status": "exact",
|
|
"unsupported": {
|
|
"raptorbt": "Same blocker as the other fixed-quantity workload: raptorbt has no units sizing, and a cost model on top of a fraction of equity compares policy rather than arithmetic. See `ema_rsi_fees` for the measurements behind that."
|
|
},
|
|
"timings": {
|
|
"manifoldbt": {
|
|
"samples_s": [
|
|
0.004976890000079948,
|
|
0.004827699999850665,
|
|
0.004812094999806504,
|
|
0.004866396000124951,
|
|
0.004853177000086362,
|
|
0.004894616000001406,
|
|
0.004828318000136278
|
|
],
|
|
"min_s": 0.004812094999806504,
|
|
"median_s": 0.004853177000086362,
|
|
"max_s": 0.004976890000079948,
|
|
"iqr_s": 6.691600015074073e-05,
|
|
"iqr_over_median": 0.013788081528769704
|
|
},
|
|
"vectorbt": {
|
|
"samples_s": [
|
|
0.4627003489999879,
|
|
0.1859375799999725,
|
|
0.18592269700002362,
|
|
0.18748765700001968,
|
|
0.18843548700010615,
|
|
0.18558119699991948,
|
|
0.18438420000006772
|
|
],
|
|
"min_s": 0.18438420000006772,
|
|
"median_s": 0.1859375799999725,
|
|
"max_s": 0.4627003489999879,
|
|
"iqr_s": 0.0028542900001866656,
|
|
"iqr_over_median": 0.015350796757638167
|
|
},
|
|
"nautilus_trader": {
|
|
"samples_s": [
|
|
5.34309964199997,
|
|
5.390984937999974,
|
|
5.620415864000051,
|
|
5.393419793000021,
|
|
5.3850978639998175,
|
|
5.603234077000025,
|
|
5.353652335999868
|
|
],
|
|
"min_s": 5.34309964199997,
|
|
"median_s": 5.390984937999974,
|
|
"max_s": 5.620415864000051,
|
|
"iqr_s": 0.2495817410001564,
|
|
"iqr_over_median": 0.04629613027499013
|
|
}
|
|
},
|
|
"speedup": {
|
|
"vectorbt": {
|
|
"median_of_ratios": 38.52700376114186,
|
|
"min": 37.91537415802714,
|
|
"max": 92.96977610366217
|
|
},
|
|
"nautilus_trader": {
|
|
"median_of_ratios": 1109.6026095697705,
|
|
"min": 1073.5820245000673,
|
|
"max": 1167.9769132209672
|
|
}
|
|
},
|
|
"noisy": false,
|
|
"cpu_over_wall": {
|
|
"manifoldbt": null,
|
|
"vectorbt": 1.0123704282353791,
|
|
"nautilus_trader": 1.0002650436153357
|
|
}
|
|
},
|
|
{
|
|
"workload": "sma_cross_costs",
|
|
"title": "SMA 30/150 with a 5 bps fee and 2 bps of slippage",
|
|
"bars": 1000000,
|
|
"data_digest": "ac3e442a94f3cd06",
|
|
"engines": [
|
|
"manifoldbt",
|
|
"vectorbt"
|
|
],
|
|
"parity": {
|
|
"vectorbt": {
|
|
"status": "exact",
|
|
"expected": "exact",
|
|
"publishable": true,
|
|
"diffs": {
|
|
"final_equity_vs_capital": 5.820766091346741e-15,
|
|
"round_trips_delta": 0,
|
|
"total_fees_vs_capital": 3.1832314562052486e-17
|
|
},
|
|
"metrics": {
|
|
"manifoldbt": {
|
|
"total_return": -0.029348513206911897,
|
|
"final_equity": 97065.14867930881,
|
|
"round_trips": 4539,
|
|
"fills": 9079,
|
|
"total_fees": 2150.1249884272365
|
|
},
|
|
"vectorbt": {
|
|
"total_return": -0.029348513206906096,
|
|
"final_equity": 97065.1486793094,
|
|
"round_trips": 4539,
|
|
"fills": null,
|
|
"total_fees": 2150.1249884272397
|
|
}
|
|
},
|
|
"note": ""
|
|
}
|
|
},
|
|
"status": "exact",
|
|
"unsupported": {
|
|
"raptorbt": "Same blocker as the other fixed-quantity workload: raptorbt has no units sizing, and a cost model on top of a fraction of equity compares policy rather than arithmetic. See `ema_rsi_fees` for the measurements behind that.",
|
|
"nautilus_trader": "Not run above 100,000 bars. NautilusTrader is event-driven: every bar goes through a simulated exchange and a Python strategy callback, so one call costs seconds per 100,000 bars and grows linearly with the series. Repeated and interleaved, a longer series does not fit the time budget of the job. The measured time at the ceiling is in the table, and `bench.py --engine-max-bars nautilus_trader=N` lifts the ceiling for a local run."
|
|
},
|
|
"beyond_ceiling": {
|
|
"nautilus_trader": 100000
|
|
},
|
|
"timings": {
|
|
"manifoldbt": {
|
|
"samples_s": [
|
|
0.032394836000094074,
|
|
0.02900139600001239,
|
|
0.03024253800003862,
|
|
0.029261875999964104,
|
|
0.030234973000005994,
|
|
0.03191694700012704,
|
|
0.03155918100014787
|
|
],
|
|
"min_s": 0.02900139600001239,
|
|
"median_s": 0.03024253800003862,
|
|
"max_s": 0.032394836000094074,
|
|
"iqr_s": 0.002655071000162934,
|
|
"iqr_over_median": 0.08779259862910789
|
|
},
|
|
"vectorbt": {
|
|
"samples_s": [
|
|
2.382637167999974,
|
|
2.150568961999852,
|
|
2.3837831329999517,
|
|
2.517840238999952,
|
|
2.1268394609999177,
|
|
2.5001764660000845,
|
|
2.127410725000118
|
|
],
|
|
"min_s": 2.1268394609999177,
|
|
"median_s": 2.382637167999974,
|
|
"max_s": 2.517840238999952,
|
|
"iqr_s": 0.3727657409999665,
|
|
"iqr_over_median": 0.1564509049075535
|
|
}
|
|
},
|
|
"speedup": {
|
|
"vectorbt": {
|
|
"median_of_ratios": 74.15398079454289,
|
|
"min": 67.41020069532698,
|
|
"max": 86.04507240079346
|
|
}
|
|
},
|
|
"noisy": true,
|
|
"cpu_over_wall": {
|
|
"manifoldbt": null,
|
|
"vectorbt": 0.999428253286531
|
|
}
|
|
},
|
|
{
|
|
"workload": "sma_cross_costs",
|
|
"title": "SMA 30/150 with a 5 bps fee and 2 bps of slippage",
|
|
"bars": 10000000,
|
|
"data_digest": "73123cc435bd1913",
|
|
"engines": [
|
|
"manifoldbt",
|
|
"vectorbt"
|
|
],
|
|
"parity": {
|
|
"vectorbt": {
|
|
"status": "exact",
|
|
"expected": "exact",
|
|
"publishable": true,
|
|
"diffs": {
|
|
"final_equity_vs_capital": 7.275957614183426e-16,
|
|
"round_trips_delta": 0,
|
|
"total_fees_vs_capital": 4.729372449219226e-16
|
|
},
|
|
"metrics": {
|
|
"manifoldbt": {
|
|
"total_return": -0.24245832953095314,
|
|
"final_equity": 75754.16704690468,
|
|
"round_trips": 46112,
|
|
"fills": 92224,
|
|
"total_fees": 17373.010435196742
|
|
},
|
|
"vectorbt": {
|
|
"total_return": -0.2424583295309525,
|
|
"final_equity": 75754.16704690475,
|
|
"round_trips": 46112,
|
|
"fills": null,
|
|
"total_fees": 17373.01043519679
|
|
}
|
|
},
|
|
"note": ""
|
|
}
|
|
},
|
|
"status": "exact",
|
|
"unsupported": {
|
|
"raptorbt": "Same blocker as the other fixed-quantity workload: raptorbt has no units sizing, and a cost model on top of a fraction of equity compares policy rather than arithmetic. See `ema_rsi_fees` for the measurements behind that.",
|
|
"nautilus_trader": "Not run above 100,000 bars. NautilusTrader is event-driven: every bar goes through a simulated exchange and a Python strategy callback, so one call costs seconds per 100,000 bars and grows linearly with the series. Repeated and interleaved, a longer series does not fit the time budget of the job. The measured time at the ceiling is in the table, and `bench.py --engine-max-bars nautilus_trader=N` lifts the ceiling for a local run."
|
|
},
|
|
"beyond_ceiling": {
|
|
"nautilus_trader": 100000
|
|
},
|
|
"timings": {
|
|
"manifoldbt": {
|
|
"samples_s": [
|
|
0.2504298819999349,
|
|
0.24986269600003652,
|
|
0.2522004449999713,
|
|
0.2513720869999361,
|
|
0.2519064430000526,
|
|
0.2523443480001788,
|
|
0.25183230200013895
|
|
],
|
|
"min_s": 0.24986269600003652,
|
|
"median_s": 0.25183230200013895,
|
|
"max_s": 0.2523443480001788,
|
|
"iqr_s": 0.001770563000036418,
|
|
"iqr_over_median": 0.007030722373476303
|
|
},
|
|
"vectorbt": {
|
|
"samples_s": [
|
|
26.61114352200002,
|
|
26.621539975000132,
|
|
27.867324426000096,
|
|
26.775263908999932,
|
|
26.545150439999816,
|
|
26.650210641000058,
|
|
27.39442427400013
|
|
],
|
|
"min_s": 26.545150439999816,
|
|
"median_s": 26.650210641000058,
|
|
"max_s": 27.867324426000096,
|
|
"iqr_s": 0.7832807520001097,
|
|
"iqr_over_median": 0.0293911655165333
|
|
}
|
|
},
|
|
"speedup": {
|
|
"vectorbt": {
|
|
"median_of_ratios": 106.51645625636526,
|
|
"min": 105.37702062663904,
|
|
"max": 110.49672979761502
|
|
}
|
|
},
|
|
"noisy": false,
|
|
"cpu_over_wall": {
|
|
"manifoldbt": 1.1193511244488303,
|
|
"vectorbt": 0.9998670459799067
|
|
}
|
|
},
|
|
{
|
|
"workload": "multi_asset",
|
|
"title": "Five assets in one book, SMA 30/150, 5 bps fee",
|
|
"bars": 100000,
|
|
"data_digest": "f6e413e413e61a53",
|
|
"engines": [
|
|
"manifoldbt",
|
|
"vectorbt",
|
|
"nautilus_trader"
|
|
],
|
|
"parity": {
|
|
"vectorbt": {
|
|
"status": "exact",
|
|
"expected": "exact",
|
|
"publishable": true,
|
|
"diffs": {
|
|
"final_equity_vs_capital": 2.473825588822365e-15,
|
|
"round_trips_delta": 0,
|
|
"total_fees_vs_capital": 1.1368683772161604e-18
|
|
},
|
|
"metrics": {
|
|
"manifoldbt": {
|
|
"total_return": -0.004781171860802846,
|
|
"final_equity": 99521.88281391971,
|
|
"round_trips": 2353,
|
|
"fills": 4706,
|
|
"total_fees": 463.6046247237474
|
|
},
|
|
"vectorbt": {
|
|
"total_return": -0.0047811718608004065,
|
|
"final_equity": 99521.88281391996,
|
|
"round_trips": 2353,
|
|
"fills": null,
|
|
"total_fees": 463.6046247237473
|
|
}
|
|
},
|
|
"note": ""
|
|
},
|
|
"nautilus_trader": {
|
|
"status": "exact",
|
|
"expected": "exact",
|
|
"publishable": true,
|
|
"diffs": {
|
|
"final_equity_vs_capital": 2.1028972696512937e-12,
|
|
"round_trips_delta": 0,
|
|
"total_fees_vs_capital": 2.6374698336439905e-12
|
|
},
|
|
"metrics": {
|
|
"manifoldbt": {
|
|
"total_return": -0.004781171860802846,
|
|
"final_equity": 99521.88281391971,
|
|
"round_trips": 2353,
|
|
"fills": 4706,
|
|
"total_fees": 463.6046247237474
|
|
},
|
|
"nautilus_trader": {
|
|
"total_return": -0.004781171858699973,
|
|
"final_equity": 99521.88281413,
|
|
"round_trips": 2353,
|
|
"fills": 4706,
|
|
"total_fees": 463.6046244600004
|
|
}
|
|
},
|
|
"note": ""
|
|
}
|
|
},
|
|
"status": "exact",
|
|
"unsupported": {
|
|
"raptorbt": "No multi-instrument portfolio on the entry point this harness drives: `run_single_backtest` is one instrument, and `run_multi_backtest` broadcasts over strategies rather than over assets. 0.9.0 does add `run_portfolio_backtest`, but its allocation model is a different one and would need its own parity work before any timing from it could be published. The units blocker applies here too."
|
|
},
|
|
"timings": {
|
|
"manifoldbt": {
|
|
"samples_s": [
|
|
0.02059622199999467,
|
|
0.02018197000006694,
|
|
0.020343180999816468,
|
|
0.020426287999953274,
|
|
0.02039440899989131,
|
|
0.020182485000077577,
|
|
0.021703507999973226
|
|
],
|
|
"min_s": 0.02018197000006694,
|
|
"median_s": 0.02039440899989131,
|
|
"max_s": 0.021703507999973226,
|
|
"iqr_s": 0.00041373699991709145,
|
|
"iqr_over_median": 0.0202867854576858
|
|
},
|
|
"vectorbt": {
|
|
"samples_s": [
|
|
0.22611408099987784,
|
|
0.22364048100007494,
|
|
0.22635658900003364,
|
|
0.22192519899999752,
|
|
0.2200141450002775,
|
|
0.21853300200018566,
|
|
0.22443047800015847
|
|
],
|
|
"min_s": 0.21853300200018566,
|
|
"median_s": 0.22364048100007494,
|
|
"max_s": 0.22635658900003364,
|
|
"iqr_s": 0.006099935999600348,
|
|
"iqr_over_median": 0.02727563441252974
|
|
},
|
|
"nautilus_trader": {
|
|
"samples_s": [
|
|
46.67444978000003,
|
|
46.700505641999825,
|
|
46.727046771000005,
|
|
46.69212941400019,
|
|
46.68636811100032,
|
|
46.53433783699984,
|
|
46.654291887
|
|
],
|
|
"min_s": 46.53433783699984,
|
|
"median_s": 46.68636811100032,
|
|
"max_s": 46.727046771000005,
|
|
"iqr_s": 0.04621375499982605,
|
|
"iqr_over_median": 0.0009898768499179333
|
|
}
|
|
},
|
|
"speedup": {
|
|
"vectorbt": {
|
|
"median_of_ratios": 10.86468569328432,
|
|
"min": 10.340746666411501,
|
|
"max": 11.12690237589076
|
|
},
|
|
"nautilus_trader": {
|
|
"median_of_ratios": 2289.1748474422147,
|
|
"min": 2149.6198626995024,
|
|
"max": 2313.9716113860504
|
|
}
|
|
},
|
|
"noisy": false,
|
|
"cpu_over_wall": {
|
|
"manifoldbt": null,
|
|
"vectorbt": 0.9993504382302503,
|
|
"nautilus_trader": 0.9999108289110435
|
|
}
|
|
},
|
|
{
|
|
"workload": "multi_asset",
|
|
"title": "Five assets in one book, SMA 30/150, 5 bps fee",
|
|
"bars": 1000000,
|
|
"data_digest": "ac3e442a94f3cd06",
|
|
"engines": [
|
|
"manifoldbt",
|
|
"vectorbt"
|
|
],
|
|
"parity": {
|
|
"vectorbt": {
|
|
"status": "exact",
|
|
"expected": "exact",
|
|
"publishable": true,
|
|
"diffs": {
|
|
"final_equity_vs_capital": 7.8580342233181e-15,
|
|
"round_trips_delta": 0,
|
|
"total_fees_vs_capital": 4.456524038687348e-16
|
|
},
|
|
"metrics": {
|
|
"manifoldbt": {
|
|
"total_return": -0.04398719435120224,
|
|
"final_equity": 95601.28056487978,
|
|
"round_trips": 22915,
|
|
"fills": 45832,
|
|
"total_fees": 4462.300363471209
|
|
},
|
|
"vectorbt": {
|
|
"total_return": -0.043987194351210075,
|
|
"final_equity": 95601.280564879,
|
|
"round_trips": 22915,
|
|
"fills": null,
|
|
"total_fees": 4462.300363471253
|
|
}
|
|
},
|
|
"note": ""
|
|
}
|
|
},
|
|
"status": "exact",
|
|
"unsupported": {
|
|
"raptorbt": "No multi-instrument portfolio on the entry point this harness drives: `run_single_backtest` is one instrument, and `run_multi_backtest` broadcasts over strategies rather than over assets. 0.9.0 does add `run_portfolio_backtest`, but its allocation model is a different one and would need its own parity work before any timing from it could be published. The units blocker applies here too.",
|
|
"nautilus_trader": "Not run above 100,000 bars. NautilusTrader is event-driven: every bar goes through a simulated exchange and a Python strategy callback, so one call costs seconds per 100,000 bars and grows linearly with the series. Repeated and interleaved, a longer series does not fit the time budget of the job. The measured time at the ceiling is in the table, and `bench.py --engine-max-bars nautilus_trader=N` lifts the ceiling for a local run."
|
|
},
|
|
"beyond_ceiling": {
|
|
"nautilus_trader": 100000
|
|
},
|
|
"timings": {
|
|
"manifoldbt": {
|
|
"samples_s": [
|
|
0.16223774199988839,
|
|
0.16147274300010395,
|
|
0.16152757000008933,
|
|
0.1612045130000297,
|
|
0.16097107800032973,
|
|
0.16162710400021751,
|
|
0.16194812799994907
|
|
],
|
|
"min_s": 0.16097107800032973,
|
|
"median_s": 0.16152757000008933,
|
|
"max_s": 0.16223774199988839,
|
|
"iqr_s": 0.0007436149999193731,
|
|
"iqr_over_median": 0.004603641346916578
|
|
},
|
|
"vectorbt": {
|
|
"samples_s": [
|
|
2.8480030569999144,
|
|
2.547931556000094,
|
|
2.842490375000125,
|
|
2.5276471260003746,
|
|
2.839487858999746,
|
|
2.5107581349998327,
|
|
2.8327051839996784
|
|
],
|
|
"min_s": 2.5107581349998327,
|
|
"median_s": 2.8327051839996784,
|
|
"max_s": 2.8480030569999144,
|
|
"iqr_s": 0.31484324899975036,
|
|
"iqr_over_median": 0.11114578770078817
|
|
}
|
|
},
|
|
"speedup": {
|
|
"vectorbt": {
|
|
"median_of_ratios": 17.491435183496343,
|
|
"min": 15.534264197398809,
|
|
"max": 17.639739351151825
|
|
}
|
|
},
|
|
"noisy": false,
|
|
"cpu_over_wall": {
|
|
"manifoldbt": 1.237854789938384,
|
|
"vectorbt": 0.9995238122904215
|
|
}
|
|
},
|
|
{
|
|
"workload": "bracket_sl_tp",
|
|
"title": "SMA 10/50 entry with a 15 bps stop / 30 bps target bracket",
|
|
"bars": 100000,
|
|
"data_digest": "f6e413e413e61a53",
|
|
"engines": [
|
|
"manifoldbt",
|
|
"vectorbt",
|
|
"raptorbt",
|
|
"nautilus_trader"
|
|
],
|
|
"parity": {
|
|
"vectorbt": {
|
|
"status": "documented",
|
|
"expected": "documented",
|
|
"publishable": false,
|
|
"diffs": {
|
|
"final_equity_vs_capital": 0.01064905871993862,
|
|
"round_trips_delta": 95,
|
|
"total_fees_vs_capital": 0.0
|
|
},
|
|
"metrics": {
|
|
"manifoldbt": {
|
|
"total_return": -0.05655656604998993,
|
|
"final_equity": 94344.34339500101,
|
|
"round_trips": 2303,
|
|
"fills": 4606,
|
|
"total_fees": 0.0
|
|
},
|
|
"vectorbt": {
|
|
"total_return": -0.06720562476992854,
|
|
"final_equity": 93279.43752300715,
|
|
"round_trips": 2208,
|
|
"fills": null,
|
|
"total_fees": 0.0
|
|
}
|
|
},
|
|
"note": "Re-entry on the exit bar. When a bracket fires intrabar and the entry condition still holds at that bar's close, manifoldbt books two orders on that bar (the stop or target exit, then a fresh entry at the close); vectorbt processes one order per bar and re-enters on the next bar instead. Neither is wrong, and on controlled bars the bracket fills themselves match exactly (see the cross-engine parity suite shipped with the library). The harness counts the affected round-trips so the size of the divergence is measured, not asserted."
|
|
},
|
|
"raptorbt": {
|
|
"status": "documented",
|
|
"expected": "documented",
|
|
"publishable": false,
|
|
"diffs": {
|
|
"final_equity_vs_capital": 0.00984269552616519,
|
|
"round_trips_delta": 922,
|
|
"total_fees_vs_capital": 0.0
|
|
},
|
|
"metrics": {
|
|
"manifoldbt": {
|
|
"total_return": -0.05655656604998993,
|
|
"final_equity": 94344.34339500101,
|
|
"round_trips": 2303,
|
|
"fills": 4606,
|
|
"total_fees": 0.0
|
|
},
|
|
"raptorbt": {
|
|
"total_return": -0.06639926157615511,
|
|
"final_equity": 93360.07384238449,
|
|
"round_trips": 1381,
|
|
"fills": null,
|
|
"total_fees": 0.0
|
|
}
|
|
},
|
|
"note": "Same fork in the road, taken further: raptorbt does not re-arm at all. Once a bracket closes a position, the entry level being still true is not enough to open another one; it waits for the level to go false and true again. So on the same bars the three engines book a different number of round-trips from the same signal, manifoldbt re-entering on the exit bar, vectorbt on the next bar, raptorbt not until the next crossing. The count of affected round-trips is measured rather than asserted, and it is the same population for both challengers."
|
|
},
|
|
"nautilus_trader": {
|
|
"status": "documented",
|
|
"expected": "documented",
|
|
"publishable": false,
|
|
"diffs": {
|
|
"final_equity_vs_capital": 1.0002018289960689e-05,
|
|
"round_trips_delta": 0,
|
|
"total_fees_vs_capital": 0.0
|
|
},
|
|
"metrics": {
|
|
"manifoldbt": {
|
|
"total_return": -0.05655656604998993,
|
|
"final_equity": 94344.34339500101,
|
|
"round_trips": 2303,
|
|
"fills": 4606,
|
|
"total_fees": 0.0
|
|
},
|
|
"nautilus_trader": {
|
|
"total_return": -0.0565465640317,
|
|
"final_equity": 94345.34359683,
|
|
"round_trips": 2303,
|
|
"fills": 4986,
|
|
"total_fees": 0.0
|
|
}
|
|
},
|
|
"note": "Re-entry sizing. Entries, exits and fill prices are the same trades as the reference's, round-trip for round-trip: the stop and the target are native contingent orders, they fill at their own levels, and because the simulated exchange runs before the strategy sees the bar, a position stopped out intrabar is re-entered at that bar's close just as the reference does. What differs is the quantity of that re-entry. The reference sizes it from the equity at the close, counting the position it has just closed at the close price, so it buys back the quantity it held; Nautilus sizes it from the cash balance after the stop or target has been booked at its own price. Neither is wrong, the two quantities differ by the gap between the exit price and the close, and that gap is carried into every later trade. The harness measures it (re-entries on the exit bar, and how far each one's quantity sits from the position closed on that bar) instead of asserting it."
|
|
}
|
|
},
|
|
"status": "documented",
|
|
"divergence_scale": {
|
|
"raptorbt": {
|
|
"round_trips": 1381,
|
|
"final_equity": 93360.07384238449
|
|
},
|
|
"nautilus_trader": {
|
|
"round_trips": 2303,
|
|
"final_equity": 94345.34359683,
|
|
"reentries_on_exit_bar": 922,
|
|
"reentry_quantity_gap_median": 0.0002688238851963217,
|
|
"reentry_quantity_gap_max": 0.001083501638118605
|
|
},
|
|
"manifoldbt": {
|
|
"reentries_on_exit_bar": 922,
|
|
"round_trips": 2303,
|
|
"share_of_round_trips": 0.4003473729917499,
|
|
"sl_exits": 630,
|
|
"tp_exits": 387
|
|
}
|
|
},
|
|
"timings": {
|
|
"manifoldbt": {
|
|
"samples_s": [
|
|
0.006350284999825817,
|
|
0.008997652999823913,
|
|
0.008885545999874012,
|
|
0.00901196400036497,
|
|
0.00913446999993539,
|
|
0.009062393000021984,
|
|
0.009105551000175183
|
|
],
|
|
"min_s": 0.006350284999825817,
|
|
"median_s": 0.00901196400036497,
|
|
"max_s": 0.00913446999993539,
|
|
"iqr_s": 0.0002200050003011711,
|
|
"iqr_over_median": 0.02441254761917172
|
|
},
|
|
"vectorbt": {
|
|
"samples_s": [
|
|
0.18698212700019212,
|
|
0.7245795230001022,
|
|
0.7294917330000317,
|
|
0.1876639269999032,
|
|
0.1863206039997749,
|
|
0.18440360999966288,
|
|
0.18600110199986375
|
|
],
|
|
"min_s": 0.18440360999966288,
|
|
"median_s": 0.18698212700019212,
|
|
"max_s": 0.7294917330000317,
|
|
"iqr_s": 0.5385784210002384,
|
|
"iqr_over_median": 2.8803738070627736
|
|
},
|
|
"raptorbt": {
|
|
"samples_s": [
|
|
0.008234757000082027,
|
|
0.008315796999795566,
|
|
0.0082745050003723,
|
|
0.008318501000303513,
|
|
0.00821186699977261,
|
|
0.008236663999923621,
|
|
0.0082547660003911
|
|
],
|
|
"min_s": 0.00821186699977261,
|
|
"median_s": 0.0082547660003911,
|
|
"max_s": 0.008318501000303513,
|
|
"iqr_s": 8.103999971353915e-05,
|
|
"iqr_over_median": 0.00981735880940775
|
|
},
|
|
"nautilus_trader": {
|
|
"samples_s": [
|
|
12.028105260999837,
|
|
11.50112996100006,
|
|
11.55629219399998,
|
|
12.03835626,
|
|
12.03721223399998,
|
|
12.16667993100009,
|
|
12.123024049999913
|
|
],
|
|
"min_s": 11.50112996100006,
|
|
"median_s": 12.03721223399998,
|
|
"max_s": 12.16667993100009,
|
|
"iqr_s": 0.566731855999933,
|
|
"iqr_over_median": 0.04708165354093847
|
|
}
|
|
},
|
|
"speedup": {
|
|
"vectorbt": {
|
|
"median_of_ratios": 20.823865584938325,
|
|
"min": 20.348224801022813,
|
|
"max": 82.09869523047601
|
|
},
|
|
"raptorbt": {
|
|
"median_of_ratios": 0.9230508466263989,
|
|
"min": 0.8989976429755304,
|
|
"max": 1.2967539252660154
|
|
},
|
|
"nautilus_trader": {
|
|
"median_of_ratios": 1331.388298167423,
|
|
"min": 1278.2366647419212,
|
|
"max": 1894.1047939312577
|
|
}
|
|
},
|
|
"noisy": true,
|
|
"cpu_over_wall": {
|
|
"manifoldbt": null,
|
|
"vectorbt": 0.9935263058388717,
|
|
"raptorbt": null,
|
|
"nautilus_trader": 1.0001102459055125
|
|
}
|
|
},
|
|
{
|
|
"workload": "bracket_sl_tp",
|
|
"title": "SMA 10/50 entry with a 15 bps stop / 30 bps target bracket",
|
|
"bars": 1000000,
|
|
"data_digest": "ac3e442a94f3cd06",
|
|
"engines": [
|
|
"manifoldbt",
|
|
"vectorbt",
|
|
"raptorbt"
|
|
],
|
|
"parity": {
|
|
"vectorbt": {
|
|
"status": "documented",
|
|
"expected": "documented",
|
|
"publishable": false,
|
|
"diffs": {
|
|
"final_equity_vs_capital": 0.0036541513284635584,
|
|
"round_trips_delta": 1090,
|
|
"total_fees_vs_capital": 0.0
|
|
},
|
|
"metrics": {
|
|
"manifoldbt": {
|
|
"total_return": -0.4672281492275321,
|
|
"final_equity": 53277.18507724679,
|
|
"round_trips": 23073,
|
|
"fills": 46147,
|
|
"total_fees": 0.0
|
|
},
|
|
"vectorbt": {
|
|
"total_return": -0.47088230055599567,
|
|
"final_equity": 52911.76994440043,
|
|
"round_trips": 21983,
|
|
"fills": null,
|
|
"total_fees": 0.0
|
|
}
|
|
},
|
|
"note": "Re-entry on the exit bar. When a bracket fires intrabar and the entry condition still holds at that bar's close, manifoldbt books two orders on that bar (the stop or target exit, then a fresh entry at the close); vectorbt processes one order per bar and re-enters on the next bar instead. Neither is wrong, and on controlled bars the bracket fills themselves match exactly (see the cross-engine parity suite shipped with the library). The harness counts the affected round-trips so the size of the divergence is measured, not asserted."
|
|
},
|
|
"raptorbt": {
|
|
"status": "documented",
|
|
"expected": "documented",
|
|
"publishable": false,
|
|
"diffs": {
|
|
"final_equity_vs_capital": 0.10667808181878907,
|
|
"round_trips_delta": 9321,
|
|
"total_fees_vs_capital": 0.0
|
|
},
|
|
"metrics": {
|
|
"manifoldbt": {
|
|
"total_return": -0.4672281492275321,
|
|
"final_equity": 53277.18507724679,
|
|
"round_trips": 23073,
|
|
"fills": 46147,
|
|
"total_fees": 0.0
|
|
},
|
|
"raptorbt": {
|
|
"total_return": -0.36055006740874307,
|
|
"final_equity": 63944.993259125695,
|
|
"round_trips": 13752,
|
|
"fills": null,
|
|
"total_fees": 0.0
|
|
}
|
|
},
|
|
"note": "Same fork in the road, taken further: raptorbt does not re-arm at all. Once a bracket closes a position, the entry level being still true is not enough to open another one; it waits for the level to go false and true again. So on the same bars the three engines book a different number of round-trips from the same signal, manifoldbt re-entering on the exit bar, vectorbt on the next bar, raptorbt not until the next crossing. The count of affected round-trips is measured rather than asserted, and it is the same population for both challengers."
|
|
}
|
|
},
|
|
"status": "documented",
|
|
"unsupported": {
|
|
"nautilus_trader": "Not run above 100,000 bars. NautilusTrader is event-driven: every bar goes through a simulated exchange and a Python strategy callback, so one call costs seconds per 100,000 bars and grows linearly with the series. Repeated and interleaved, a longer series does not fit the time budget of the job. The measured time at the ceiling is in the table, and `bench.py --engine-max-bars nautilus_trader=N` lifts the ceiling for a local run."
|
|
},
|
|
"beyond_ceiling": {
|
|
"nautilus_trader": 100000
|
|
},
|
|
"divergence_scale": {
|
|
"raptorbt": {
|
|
"round_trips": 13752,
|
|
"final_equity": 63944.993259125695
|
|
},
|
|
"manifoldbt": {
|
|
"reentries_on_exit_bar": 9321,
|
|
"round_trips": 23073,
|
|
"share_of_round_trips": 0.40397867637498375,
|
|
"sl_exits": 6357,
|
|
"tp_exits": 3896
|
|
}
|
|
},
|
|
"timings": {
|
|
"manifoldbt": {
|
|
"samples_s": [
|
|
0.05274533000010706,
|
|
0.06768290400032129,
|
|
0.0642738989999998,
|
|
0.06436788600012733,
|
|
0.06496422700001858,
|
|
0.065159795999989,
|
|
0.06473150700003316
|
|
],
|
|
"min_s": 0.05274533000010706,
|
|
"median_s": 0.06473150700003316,
|
|
"max_s": 0.06768290400032129,
|
|
"iqr_s": 0.0008858969999891997,
|
|
"iqr_over_median": 0.01368571567457476
|
|
},
|
|
"vectorbt": {
|
|
"samples_s": [
|
|
2.4788003610001397,
|
|
3.0176480000000083,
|
|
2.4670809270000973,
|
|
2.9988701140000558,
|
|
2.4413719180001863,
|
|
3.0018531919999987,
|
|
2.4668596380001873
|
|
],
|
|
"min_s": 2.4413719180001863,
|
|
"median_s": 2.4788003610001397,
|
|
"max_s": 3.0176480000000083,
|
|
"iqr_s": 0.5349935539998114,
|
|
"iqr_over_median": 0.2158276085549518
|
|
},
|
|
"raptorbt": {
|
|
"samples_s": [
|
|
0.10213059699981386,
|
|
0.07959314600020662,
|
|
0.0797496919999503,
|
|
0.07983418699996037,
|
|
0.0804967150002085,
|
|
0.0796335810000528,
|
|
0.0796369859999686
|
|
],
|
|
"min_s": 0.07959314600020662,
|
|
"median_s": 0.0797496919999503,
|
|
"max_s": 0.10213059699981386,
|
|
"iqr_s": 0.0008631340001556964,
|
|
"iqr_over_median": 0.010823038666484559
|
|
}
|
|
},
|
|
"speedup": {
|
|
"vectorbt": {
|
|
"median_of_ratios": 44.58508458776656,
|
|
"min": 37.580250404572475,
|
|
"max": 46.99563659939388
|
|
},
|
|
"raptorbt": {
|
|
"median_of_ratios": 1.2390929395678867,
|
|
"min": 1.1759712024151445,
|
|
"max": 1.9362964834916487
|
|
}
|
|
},
|
|
"noisy": true,
|
|
"cpu_over_wall": {
|
|
"manifoldbt": null,
|
|
"vectorbt": 0.999338499908002,
|
|
"raptorbt": 1.015359630812729
|
|
}
|
|
},
|
|
{
|
|
"workload": "bracket_sl_tp",
|
|
"title": "SMA 10/50 entry with a 15 bps stop / 30 bps target bracket",
|
|
"bars": 10000000,
|
|
"data_digest": "73123cc435bd1913",
|
|
"engines": [
|
|
"manifoldbt",
|
|
"vectorbt",
|
|
"raptorbt"
|
|
],
|
|
"parity": {
|
|
"vectorbt": {
|
|
"status": "documented",
|
|
"expected": "documented",
|
|
"publishable": false,
|
|
"diffs": {
|
|
"final_equity_vs_capital": 0.00053833908042622,
|
|
"round_trips_delta": 10184,
|
|
"total_fees_vs_capital": 0.0
|
|
},
|
|
"metrics": {
|
|
"manifoldbt": {
|
|
"total_return": -0.9959910122076127,
|
|
"final_equity": 400.89877923873416,
|
|
"round_trips": 231002,
|
|
"fills": 462004,
|
|
"total_fees": 0.0
|
|
},
|
|
"vectorbt": {
|
|
"total_return": -0.9965293512880389,
|
|
"final_equity": 347.06487119611216,
|
|
"round_trips": 220818,
|
|
"fills": null,
|
|
"total_fees": 0.0
|
|
}
|
|
},
|
|
"note": "Re-entry on the exit bar. When a bracket fires intrabar and the entry condition still holds at that bar's close, manifoldbt books two orders on that bar (the stop or target exit, then a fresh entry at the close); vectorbt processes one order per bar and re-enters on the next bar instead. Neither is wrong, and on controlled bars the bracket fills themselves match exactly (see the cross-engine parity suite shipped with the library). The harness counts the affected round-trips so the size of the divergence is measured, not asserted."
|
|
},
|
|
"raptorbt": {
|
|
"status": "documented",
|
|
"expected": "documented",
|
|
"publishable": false,
|
|
"diffs": {
|
|
"final_equity_vs_capital": 0.01997283196010167,
|
|
"round_trips_delta": 93238,
|
|
"total_fees_vs_capital": 0.0
|
|
},
|
|
"metrics": {
|
|
"manifoldbt": {
|
|
"total_return": -0.9959910122076127,
|
|
"final_equity": 400.89877923873416,
|
|
"round_trips": 231002,
|
|
"fills": 462004,
|
|
"total_fees": 0.0
|
|
},
|
|
"raptorbt": {
|
|
"total_return": -0.9760181802475111,
|
|
"final_equity": 2398.1819752489014,
|
|
"round_trips": 137764,
|
|
"fills": null,
|
|
"total_fees": 0.0
|
|
}
|
|
},
|
|
"note": "Same fork in the road, taken further: raptorbt does not re-arm at all. Once a bracket closes a position, the entry level being still true is not enough to open another one; it waits for the level to go false and true again. So on the same bars the three engines book a different number of round-trips from the same signal, manifoldbt re-entering on the exit bar, vectorbt on the next bar, raptorbt not until the next crossing. The count of affected round-trips is measured rather than asserted, and it is the same population for both challengers."
|
|
}
|
|
},
|
|
"status": "documented",
|
|
"unsupported": {
|
|
"nautilus_trader": "Not run above 100,000 bars. NautilusTrader is event-driven: every bar goes through a simulated exchange and a Python strategy callback, so one call costs seconds per 100,000 bars and grows linearly with the series. Repeated and interleaved, a longer series does not fit the time budget of the job. The measured time at the ceiling is in the table, and `bench.py --engine-max-bars nautilus_trader=N` lifts the ceiling for a local run."
|
|
},
|
|
"beyond_ceiling": {
|
|
"nautilus_trader": 100000
|
|
},
|
|
"divergence_scale": {
|
|
"raptorbt": {
|
|
"round_trips": 137764,
|
|
"final_equity": 2398.1819752489014
|
|
},
|
|
"manifoldbt": {
|
|
"reentries_on_exit_bar": 93238,
|
|
"round_trips": 231002,
|
|
"share_of_round_trips": 0.4036242110457918,
|
|
"sl_exits": 63242,
|
|
"tp_exits": 39409
|
|
}
|
|
},
|
|
"timings": {
|
|
"manifoldbt": {
|
|
"samples_s": [
|
|
0.5657713939999667,
|
|
0.5608834879999449,
|
|
0.5625231619997066,
|
|
0.5580769169996529,
|
|
0.5603999030004161,
|
|
0.5544574270002158,
|
|
0.5546165169998858
|
|
],
|
|
"min_s": 0.5544574270002158,
|
|
"median_s": 0.5603999030004161,
|
|
"max_s": 0.5657713939999667,
|
|
"iqr_s": 0.007906644999820855,
|
|
"iqr_over_median": 0.01410893356242245
|
|
},
|
|
"vectorbt": {
|
|
"samples_s": [
|
|
29.67863446399997,
|
|
30.817361803999574,
|
|
29.325717661999988,
|
|
29.97844646600015,
|
|
30.663778313999956,
|
|
29.90734812799974,
|
|
29.785703636000108
|
|
],
|
|
"min_s": 29.325717661999988,
|
|
"median_s": 29.90734812799974,
|
|
"max_s": 30.817361803999574,
|
|
"iqr_s": 0.9851438499999858,
|
|
"iqr_over_median": 0.032939859655349325
|
|
},
|
|
"raptorbt": {
|
|
"samples_s": [
|
|
1.6218705189999127,
|
|
1.630273969999962,
|
|
1.6301002649997827,
|
|
1.6473132979999718,
|
|
1.6229110079998463,
|
|
1.6232319149999057,
|
|
1.6228542539997761
|
|
],
|
|
"min_s": 1.6218705189999127,
|
|
"median_s": 1.6232319149999057,
|
|
"max_s": 1.6473132979999718,
|
|
"iqr_s": 0.007419716000185872,
|
|
"iqr_over_median": 0.004570952512467267
|
|
}
|
|
},
|
|
"speedup": {
|
|
"vectorbt": {
|
|
"median_of_ratios": 53.71740982796964,
|
|
"min": 52.132462524299186,
|
|
"max": 54.94431992264782
|
|
},
|
|
"raptorbt": {
|
|
"median_of_ratios": 2.906617871410973,
|
|
"min": 2.8666534508459227,
|
|
"max": 2.951767485486228
|
|
}
|
|
},
|
|
"noisy": false,
|
|
"cpu_over_wall": {
|
|
"manifoldbt": 1.044238751390307,
|
|
"vectorbt": 0.999966736895194,
|
|
"raptorbt": 1.0001267503619855
|
|
}
|
|
}
|
|
],
|
|
"cold_start": {
|
|
"workload": "sma_cross",
|
|
"bars": 20000,
|
|
"engines": [
|
|
"manifoldbt",
|
|
"vectorbt",
|
|
"raptorbt",
|
|
"nautilus_trader"
|
|
],
|
|
"samples_s": {
|
|
"manifoldbt": [
|
|
0.49620019399981175,
|
|
0.4234076390002883,
|
|
0.4168319129998963
|
|
],
|
|
"vectorbt": [
|
|
7.356408870999985,
|
|
7.019544604999737,
|
|
7.11201858000004
|
|
],
|
|
"raptorbt": [
|
|
0.3939954120000948,
|
|
0.3873526700003822,
|
|
0.389611665999837
|
|
],
|
|
"nautilus_trader": [
|
|
1.9270602299998245,
|
|
1.922550893999869,
|
|
1.9328080519999276
|
|
],
|
|
"baseline": [
|
|
0.35246750499982227,
|
|
0.3546913170002881,
|
|
0.35192168499997933
|
|
]
|
|
},
|
|
"median_s": {
|
|
"manifoldbt": 0.4234076390002883,
|
|
"vectorbt": 7.11201858000004,
|
|
"raptorbt": 0.389611665999837,
|
|
"nautilus_trader": 1.9270602299998245,
|
|
"baseline": 0.35246750499982227
|
|
},
|
|
"engine_share_s": {
|
|
"manifoldbt": 0.07094013400046606,
|
|
"vectorbt": 6.759551075000218,
|
|
"raptorbt": 0.03714416100001472,
|
|
"nautilus_trader": 1.5745927250000022
|
|
},
|
|
"ratio": {
|
|
"vectorbt": 16.7970955762449,
|
|
"raptorbt": 0.9201810031575072,
|
|
"nautilus_trader": 4.551311909605183
|
|
}
|
|
},
|
|
"memory": {
|
|
"workload": "sma_cross",
|
|
"bars": 2000000,
|
|
"engines": [
|
|
"manifoldbt",
|
|
"vectorbt",
|
|
"raptorbt",
|
|
"nautilus_trader"
|
|
],
|
|
"manifoldbt": {
|
|
"before_mb": 511.848448,
|
|
"peak_mb": 511.868928,
|
|
"added_mb": 0.02047999999996364,
|
|
"added_mb_per_million_bars": 0.01023999999998182,
|
|
"seconds": 0.05178048600009788,
|
|
"mode": "memory",
|
|
"engine": "mbt",
|
|
"workload": "sma_cross",
|
|
"bars": 2000000
|
|
},
|
|
"vectorbt": {
|
|
"before_mb": 665.018368,
|
|
"peak_mb": 926.306304,
|
|
"added_mb": 261.28793599999995,
|
|
"added_mb_per_million_bars": 130.64396799999997,
|
|
"seconds": 4.326520187999904,
|
|
"mode": "memory",
|
|
"engine": "vbt",
|
|
"workload": "sma_cross",
|
|
"bars": 2000000
|
|
},
|
|
"raptorbt": {
|
|
"before_mb": 350.834688,
|
|
"peak_mb": 391.376896,
|
|
"added_mb": 40.54220799999996,
|
|
"added_mb_per_million_bars": 20.27110399999998,
|
|
"seconds": 0.22201791599991338,
|
|
"mode": "memory",
|
|
"engine": "rbt",
|
|
"workload": "sma_cross",
|
|
"bars": 2000000
|
|
},
|
|
"nautilus_trader": {
|
|
"before_mb": 598.224896,
|
|
"peak_mb": 598.913024,
|
|
"added_mb": 0.6881280000000061,
|
|
"added_mb_per_million_bars": 6.881280000000061,
|
|
"seconds": 6.714505022000139,
|
|
"mode": "memory",
|
|
"engine": "nt",
|
|
"workload": "sma_cross",
|
|
"bars": 100000
|
|
}
|
|
},
|
|
"sweeps": null,
|
|
"sweep_reps": null
|
|
},
|
|
"sweeps": {
|
|
"schema_version": 2,
|
|
"generated_at": "2026-10-29T21:56:56.045472+00:00",
|
|
"environment": {
|
|
"os": "Linux",
|
|
"os_release": "7.0.0-1017-azure",
|
|
"arch": "x86_64",
|
|
"cpu": "INTEL(R) XEON(R) PLATINUM 8573C",
|
|
"logical_cores": 4,
|
|
"ram_gb": 16.8,
|
|
"python": "3.12.15",
|
|
"versions": {
|
|
"manifoldbt": "0.31.1",
|
|
"vectorbt": "0.28.4",
|
|
"raptorbt": "0.9.0",
|
|
"nautilus_trader": "1.231.0",
|
|
"numpy": "2.4.3",
|
|
"numba": "0.64.0",
|
|
"pandas": "2.3.3"
|
|
},
|
|
"runner": "GitHub Actions 1000001319",
|
|
"ci": true,
|
|
"run_url": "https://github.com/manifoldbt/manifoldbt/actions/runs/40957995034",
|
|
"commit": "e53c0011378df87ae7e7e8ccd6ff0d8860b60fcd",
|
|
"pinned_cores": null,
|
|
"engines": [
|
|
"manifoldbt",
|
|
"vectorbt",
|
|
"raptorbt",
|
|
"nautilus_trader"
|
|
],
|
|
"engine_max_bars": {
|
|
"nautilus_trader": 100000
|
|
},
|
|
"engine_max_sweep_combos": {
|
|
"nautilus_trader": 64
|
|
}
|
|
},
|
|
"reference": "manifoldbt",
|
|
"engines": [
|
|
"manifoldbt",
|
|
"vectorbt",
|
|
"raptorbt",
|
|
"nautilus_trader"
|
|
],
|
|
"reps": 1,
|
|
"results": [
|
|
{
|
|
"workload": "sma_cross",
|
|
"title": "SMA 30/150 crossover, long-only, no cost",
|
|
"bars": 100000,
|
|
"data_digest": "f6e413e413e61a53",
|
|
"engines": [
|
|
"manifoldbt",
|
|
"vectorbt",
|
|
"raptorbt",
|
|
"nautilus_trader"
|
|
],
|
|
"parity": {
|
|
"vectorbt": {
|
|
"status": "exact",
|
|
"expected": "exact",
|
|
"publishable": true,
|
|
"diffs": {
|
|
"final_equity_vs_capital": 2.037268131971359e-15,
|
|
"round_trips_delta": 0,
|
|
"total_fees_vs_capital": 0.0
|
|
},
|
|
"metrics": {
|
|
"manifoldbt": {
|
|
"total_return": 0.06552457977030235,
|
|
"final_equity": 106552.45797703024,
|
|
"round_trips": 451,
|
|
"fills": 902,
|
|
"total_fees": 0.0
|
|
},
|
|
"vectorbt": {
|
|
"total_return": 0.06552457977030426,
|
|
"final_equity": 106552.45797703044,
|
|
"round_trips": 451,
|
|
"fills": null,
|
|
"total_fees": 0.0
|
|
}
|
|
},
|
|
"note": ""
|
|
},
|
|
"raptorbt": {
|
|
"status": "exact",
|
|
"expected": "exact",
|
|
"publishable": true,
|
|
"diffs": {
|
|
"final_equity_vs_capital": 0.0,
|
|
"round_trips_delta": 0,
|
|
"total_fees_vs_capital": 0.0
|
|
},
|
|
"metrics": {
|
|
"manifoldbt": {
|
|
"total_return": 0.06552457977030235,
|
|
"final_equity": 106552.45797703024,
|
|
"round_trips": 451,
|
|
"fills": 902,
|
|
"total_fees": 0.0
|
|
},
|
|
"raptorbt": {
|
|
"total_return": 0.06552457977030238,
|
|
"final_equity": 106552.45797703024,
|
|
"round_trips": 451,
|
|
"fills": null,
|
|
"total_fees": 0.0
|
|
}
|
|
},
|
|
"note": ""
|
|
},
|
|
"nautilus_trader": {
|
|
"status": "exact",
|
|
"expected": "exact",
|
|
"publishable": true,
|
|
"diffs": {
|
|
"final_equity_vs_capital": 4.330242518335581e-11,
|
|
"round_trips_delta": 0,
|
|
"total_fees_vs_capital": 0.0
|
|
},
|
|
"metrics": {
|
|
"manifoldbt": {
|
|
"total_return": 0.06552457977030235,
|
|
"final_equity": 106552.45797703024,
|
|
"round_trips": 451,
|
|
"fills": 902,
|
|
"total_fees": 0.0
|
|
},
|
|
"nautilus_trader": {
|
|
"total_return": 0.06552457972699988,
|
|
"final_equity": 106552.4579727,
|
|
"round_trips": 451,
|
|
"fills": 902,
|
|
"total_fees": 0.0
|
|
}
|
|
},
|
|
"note": ""
|
|
}
|
|
},
|
|
"status": "exact",
|
|
"timings": {
|
|
"manifoldbt": {
|
|
"samples_s": [
|
|
0.005553190000000541
|
|
],
|
|
"min_s": 0.005553190000000541,
|
|
"median_s": 0.005553190000000541,
|
|
"max_s": 0.005553190000000541,
|
|
"iqr_s": 0.0,
|
|
"iqr_over_median": 0.0
|
|
},
|
|
"vectorbt": {
|
|
"samples_s": [
|
|
0.36855029700001296
|
|
],
|
|
"min_s": 0.36855029700001296,
|
|
"median_s": 0.36855029700001296,
|
|
"max_s": 0.36855029700001296,
|
|
"iqr_s": 0.0,
|
|
"iqr_over_median": 0.0
|
|
},
|
|
"raptorbt": {
|
|
"samples_s": [
|
|
0.009104550000017753
|
|
],
|
|
"min_s": 0.009104550000017753,
|
|
"median_s": 0.009104550000017753,
|
|
"max_s": 0.009104550000017753,
|
|
"iqr_s": 0.0,
|
|
"iqr_over_median": 0.0
|
|
},
|
|
"nautilus_trader": {
|
|
"samples_s": [
|
|
4.878609755000014
|
|
],
|
|
"min_s": 4.878609755000014,
|
|
"median_s": 4.878609755000014,
|
|
"max_s": 4.878609755000014,
|
|
"iqr_s": 0.0,
|
|
"iqr_over_median": 0.0
|
|
}
|
|
},
|
|
"speedup": {
|
|
"vectorbt": {
|
|
"median_of_ratios": 66.36731266172724,
|
|
"min": 66.36731266172724,
|
|
"max": 66.36731266172724
|
|
},
|
|
"raptorbt": {
|
|
"median_of_ratios": 1.6395171063869354,
|
|
"min": 1.6395171063869354,
|
|
"max": 1.6395171063869354
|
|
},
|
|
"nautilus_trader": {
|
|
"median_of_ratios": 878.5238313473046,
|
|
"min": 878.5238313473046,
|
|
"max": 878.5238313473046
|
|
}
|
|
},
|
|
"noisy": false,
|
|
"cpu_over_wall": {
|
|
"manifoldbt": null,
|
|
"vectorbt": null,
|
|
"raptorbt": null,
|
|
"nautilus_trader": 1.000284967453804
|
|
}
|
|
}
|
|
],
|
|
"cold_start": null,
|
|
"memory": null,
|
|
"sweeps": [
|
|
{
|
|
"bars": 20000,
|
|
"combos": 5000,
|
|
"requested_combos": 5000,
|
|
"grid": {
|
|
"fast": 50,
|
|
"slow": 100
|
|
},
|
|
"data_digest": "0d360424244ea1ee",
|
|
"engines": [
|
|
"manifoldbt",
|
|
"vectorbt",
|
|
"raptorbt"
|
|
],
|
|
"metrics": false,
|
|
"tier_before": {
|
|
"pro": true
|
|
},
|
|
"out_of_scope": {
|
|
"nautilus_trader": {
|
|
"status": "out of scope",
|
|
"reason": "NautilusTrader has no parameter-grid API, so a grid is a Python loop of full backtests, each one replaying every bar through the simulated exchange. Measured: 1.06 s per backtest at 20,000 bars (mean over 16 cells spread across this grid), which puts this grid of 5,000 combinations at about 88 minutes per call. Looped grids are held to 64 combinations, so that a point stays inside the time budget of the job. It is not run.",
|
|
"parity_scope": "anchor grid",
|
|
"anchor_seconds": 16.897
|
|
}
|
|
},
|
|
"parity": {
|
|
"vectorbt": {
|
|
"status": "exact",
|
|
"combos_compared": 5000,
|
|
"worst_abs_return_delta": 1.065987575987748e-15,
|
|
"worst_component": "return",
|
|
"worst_pair": [
|
|
23,
|
|
113
|
|
]
|
|
},
|
|
"raptorbt": {
|
|
"status": "exact",
|
|
"combos_compared": 5000,
|
|
"worst_abs_return_delta": 1.1102230246251565e-16,
|
|
"worst_component": "return",
|
|
"worst_pair": [
|
|
17,
|
|
288
|
|
]
|
|
},
|
|
"nautilus_trader": {
|
|
"status": "exact",
|
|
"combos_compared": 16,
|
|
"worst_abs_return_delta": 1.22973742300303e-10,
|
|
"worst_component": "return",
|
|
"worst_pair": [
|
|
5,
|
|
443
|
|
],
|
|
"checked_at_combos": 16,
|
|
"scope": "anchor grid, not this grid"
|
|
}
|
|
},
|
|
"status": "exact",
|
|
"timings": {
|
|
"seconds": {
|
|
"manifoldbt": 0.28112943400003587,
|
|
"vectorbt": 9.578583200999958,
|
|
"raptorbt": 5.863541193999993,
|
|
"nautilus_trader": null
|
|
},
|
|
"ratio": {
|
|
"vectorbt": 34.071790579561785,
|
|
"raptorbt": 20.857087465267846
|
|
},
|
|
"reps": 2,
|
|
"per_combo_us": 56.225886800007174
|
|
},
|
|
"memory": {
|
|
"manifoldbt": 33.6,
|
|
"vectorbt": 2531.4,
|
|
"raptorbt": 39.0,
|
|
"nautilus_trader": null
|
|
},
|
|
"memory_baseline_mb": 1005.7,
|
|
"tier_after": {
|
|
"pro": true
|
|
}
|
|
},
|
|
{
|
|
"bars": 20000,
|
|
"combos": 20000,
|
|
"requested_combos": 20000,
|
|
"grid": {
|
|
"fast": 125,
|
|
"slow": 160
|
|
},
|
|
"data_digest": "0d360424244ea1ee",
|
|
"engines": [
|
|
"manifoldbt",
|
|
"raptorbt"
|
|
],
|
|
"metrics": false,
|
|
"tier_before": {
|
|
"pro": true
|
|
},
|
|
"out_of_scope": {
|
|
"vectorbt": {
|
|
"status": "out of scope",
|
|
"reason": "vectorbt materialises the simulation per combination (measured: 1.57 MB per combination at 20k bars), so this grid would need tens of gigabytes. Running it would measure the swap file.",
|
|
"parity_scope": "anchor grid",
|
|
"anchor_seconds": 4.262
|
|
},
|
|
"nautilus_trader": {
|
|
"status": "out of scope",
|
|
"reason": "NautilusTrader has no parameter-grid API, so a grid is a Python loop of full backtests, each one replaying every bar through the simulated exchange. Measured: 1.14 s per backtest at 20,000 bars (mean over 16 cells spread across this grid), which puts this grid of 20,000 combinations at about 6.3 hours per call. Looped grids are held to 64 combinations, so that a point stays inside the time budget of the job. It is not run.",
|
|
"parity_scope": "anchor grid",
|
|
"anchor_seconds": 18.272
|
|
}
|
|
},
|
|
"parity": {
|
|
"raptorbt": {
|
|
"status": "exact",
|
|
"combos_compared": 20000,
|
|
"worst_abs_return_delta": 1.1102230246251565e-16,
|
|
"worst_component": "return",
|
|
"worst_pair": [
|
|
17,
|
|
288
|
|
]
|
|
},
|
|
"vectorbt": {
|
|
"status": "exact",
|
|
"combos_compared": 250,
|
|
"worst_abs_return_delta": 2.9733160378242474e-15,
|
|
"worst_component": "return",
|
|
"worst_pair": [
|
|
11,
|
|
98
|
|
],
|
|
"checked_at_combos": 250,
|
|
"scope": "anchor grid, not this grid"
|
|
},
|
|
"nautilus_trader": {
|
|
"status": "exact",
|
|
"combos_compared": 16,
|
|
"worst_abs_return_delta": 9.083811480792292e-11,
|
|
"worst_component": "return",
|
|
"worst_pair": [
|
|
5,
|
|
263
|
|
],
|
|
"checked_at_combos": 16,
|
|
"scope": "anchor grid, not this grid"
|
|
}
|
|
},
|
|
"status": "exact",
|
|
"timings": {
|
|
"seconds": {
|
|
"manifoldbt": 1.088042373999997,
|
|
"raptorbt": 22.632095171000003,
|
|
"vectorbt": null,
|
|
"nautilus_trader": null
|
|
},
|
|
"ratio": {
|
|
"raptorbt": 20.800747941274636
|
|
},
|
|
"reps": 2,
|
|
"per_combo_us": 54.402118699999846
|
|
},
|
|
"memory": {
|
|
"manifoldbt": 95.7,
|
|
"vectorbt": null,
|
|
"raptorbt": 59.9,
|
|
"nautilus_trader": null
|
|
},
|
|
"memory_baseline_mb": 942.9,
|
|
"tier_after": {
|
|
"pro": true
|
|
}
|
|
},
|
|
{
|
|
"bars": 200000,
|
|
"combos": 10000,
|
|
"requested_combos": 10000,
|
|
"grid": {
|
|
"fast": 100,
|
|
"slow": 100
|
|
},
|
|
"data_digest": "41728ac4d4fb6c04",
|
|
"engines": [
|
|
"manifoldbt",
|
|
"raptorbt"
|
|
],
|
|
"metrics": false,
|
|
"tier_before": {
|
|
"pro": true
|
|
},
|
|
"out_of_scope": {
|
|
"vectorbt": {
|
|
"status": "out of scope",
|
|
"reason": "vectorbt materialises the simulation per combination (measured: 1.57 MB per combination at 20k bars), so this grid would need tens of gigabytes. Running it would measure the swap file.",
|
|
"parity_scope": "anchor grid",
|
|
"anchor_seconds": 9.61
|
|
},
|
|
"nautilus_trader": {
|
|
"status": "out of scope",
|
|
"reason": "NautilusTrader has no parameter-grid API, so a grid is a Python loop of full backtests, each one replaying every bar through the simulated exchange. Measured: 10.97 s per backtest at 200,000 bars (mean over 16 cells spread across this grid), which puts this grid of 10,000 combinations at about 30.5 hours per call. Looped grids are held to 64 combinations, so that a point stays inside the time budget of the job. It is not run.",
|
|
"parity_scope": "anchor grid",
|
|
"anchor_seconds": 175.567
|
|
}
|
|
},
|
|
"parity": {
|
|
"raptorbt": {
|
|
"status": "exact",
|
|
"combos_compared": 10000,
|
|
"worst_abs_return_delta": 1.1796119636642288e-16,
|
|
"worst_component": "return",
|
|
"worst_pair": [
|
|
31,
|
|
708
|
|
]
|
|
},
|
|
"vectorbt": {
|
|
"status": "exact",
|
|
"combos_compared": 250,
|
|
"worst_abs_return_delta": 1.2051123987610879e-14,
|
|
"worst_component": "return",
|
|
"worst_pair": [
|
|
13,
|
|
103
|
|
],
|
|
"checked_at_combos": 250,
|
|
"scope": "anchor grid, not this grid"
|
|
},
|
|
"nautilus_trader": {
|
|
"status": "exact",
|
|
"combos_compared": 16,
|
|
"worst_abs_return_delta": 3.4502534163038945e-10,
|
|
"worst_component": "return",
|
|
"worst_pair": [
|
|
5,
|
|
708
|
|
],
|
|
"checked_at_combos": 16,
|
|
"scope": "anchor grid, not this grid"
|
|
}
|
|
},
|
|
"status": "exact",
|
|
"timings": {
|
|
"seconds": {
|
|
"manifoldbt": 4.506411168,
|
|
"raptorbt": 137.06195497099998,
|
|
"vectorbt": null,
|
|
"nautilus_trader": null
|
|
},
|
|
"ratio": {
|
|
"raptorbt": 30.414880014561508
|
|
},
|
|
"reps": 2,
|
|
"per_combo_us": 450.64111679999996
|
|
},
|
|
"memory": {
|
|
"manifoldbt": 342.6,
|
|
"vectorbt": null,
|
|
"raptorbt": 359.2,
|
|
"nautilus_trader": null
|
|
},
|
|
"memory_baseline_mb": 1291.4,
|
|
"tier_after": {
|
|
"pro": true
|
|
}
|
|
},
|
|
{
|
|
"bars": 200000,
|
|
"combos": 150000,
|
|
"requested_combos": 150000,
|
|
"grid": {
|
|
"fast": 375,
|
|
"slow": 400
|
|
},
|
|
"data_digest": "41728ac4d4fb6c04",
|
|
"engines": [
|
|
"manifoldbt"
|
|
],
|
|
"metrics": false,
|
|
"tier_before": {
|
|
"pro": true
|
|
},
|
|
"out_of_scope": {
|
|
"vectorbt": {
|
|
"status": "out of scope",
|
|
"reason": "vectorbt materialises the simulation per combination (measured: 1.57 MB per combination at 20k bars), so this grid would need tens of gigabytes. Running it would measure the swap file.",
|
|
"parity_scope": "anchor grid",
|
|
"anchor_seconds": 10.253
|
|
},
|
|
"raptorbt": {
|
|
"status": "out of scope",
|
|
"reason": "raptorbt has no fan-out API for a parameter grid on one instrument, so a grid is a Python loop of full backtests. Measured: 13.0 ms per backtest at 200,000 bars (mean over 16 cells spread across this grid), which puts this grid of 150,000 combinations at about 32 minutes per call. That is past the time budget of the job. It is not run.",
|
|
"parity_scope": "anchor grid",
|
|
"anchor_seconds": 0.208
|
|
},
|
|
"nautilus_trader": {
|
|
"status": "out of scope",
|
|
"reason": "NautilusTrader has no parameter-grid API, so a grid is a Python loop of full backtests, each one replaying every bar through the simulated exchange. Measured: 17.80 s per backtest at 200,000 bars (mean over 16 cells spread across this grid), which puts this grid of 150,000 combinations at about 741.7 hours per call. Looped grids are held to 64 combinations, so that a point stays inside the time budget of the job. It is not run.",
|
|
"parity_scope": "anchor grid",
|
|
"anchor_seconds": 284.824
|
|
}
|
|
},
|
|
"parity": {
|
|
"vectorbt": {
|
|
"status": "exact",
|
|
"combos_compared": 250,
|
|
"worst_abs_return_delta": 1.2051123987610879e-14,
|
|
"worst_component": "return",
|
|
"worst_pair": [
|
|
13,
|
|
103
|
|
],
|
|
"checked_at_combos": 250,
|
|
"scope": "anchor grid, not this grid"
|
|
},
|
|
"raptorbt": {
|
|
"status": "exact",
|
|
"combos_compared": 16,
|
|
"worst_abs_return_delta": 1.0234868508263162e-16,
|
|
"worst_component": "return",
|
|
"worst_pair": [
|
|
255,
|
|
1428
|
|
],
|
|
"checked_at_combos": 16,
|
|
"scope": "anchor grid, not this grid"
|
|
},
|
|
"nautilus_trader": {
|
|
"status": "exact",
|
|
"combos_compared": 16,
|
|
"worst_abs_return_delta": 2.2606871930008765e-10,
|
|
"worst_component": "return",
|
|
"worst_pair": [
|
|
5,
|
|
1428
|
|
],
|
|
"checked_at_combos": 16,
|
|
"scope": "anchor grid, not this grid"
|
|
}
|
|
},
|
|
"status": "exact",
|
|
"timings": {
|
|
"seconds": {
|
|
"manifoldbt": 94.52575937300003,
|
|
"vectorbt": null,
|
|
"raptorbt": null,
|
|
"nautilus_trader": null
|
|
},
|
|
"ratio": {},
|
|
"reps": 2,
|
|
"per_combo_us": 630.1717291533336
|
|
},
|
|
"memory": {
|
|
"manifoldbt": 824.1,
|
|
"vectorbt": null,
|
|
"raptorbt": null,
|
|
"nautilus_trader": null
|
|
},
|
|
"memory_baseline_mb": 1095.9,
|
|
"tier_after": {
|
|
"pro": true
|
|
}
|
|
}
|
|
],
|
|
"sweep_reps": 2
|
|
}
|
|
}
|