From 160ed2f94800f3ef5f7bb80c1b303f89129326ed Mon Sep 17 00:00:00 2001 From: danielmjrr <218780195+danielmjrr@users.noreply.github.com> Date: Mon, 3 Aug 2026 19:43:21 +0200 Subject: [PATCH] tickreplay 0.1.0: tick-accurate MT5 backtesting engine + research toolkit Memory-mapped tickstores, a nogil numba replay kernel validated trade-for-trade against an independent reference implementation, purged walk-forward CV, an append-only FDR registry, tick-exact labels, and online changepoint detection. 129 tests, 4 runnable examples on synthetic data, CI for linux/macos. --- .github/workflows/ci.yml | 21 +++ .gitignore | 13 ++ LICENSE | 21 +++ README.md | 140 ++++++++++++++ examples/benchmark.py | 81 ++++++++ examples/breakout.py | 46 +++++ examples/ma_cross.py | 41 ++++ examples/sweep_and_validate.py | 59 ++++++ pyproject.toml | 38 ++++ src/tickreplay/__init__.py | 27 +++ src/tickreplay/engine.py | 181 ++++++++++++++++++ src/tickreplay/ingest.py | 123 ++++++++++++ src/tickreplay/research/__init__.py | 11 ++ src/tickreplay/research/bocpd.py | 50 +++++ src/tickreplay/research/labels.py | 102 ++++++++++ src/tickreplay/research/registry.py | 34 ++++ src/tickreplay/research/stats.py | 46 +++++ src/tickreplay/research/validation.py | 32 ++++ src/tickreplay/store.py | 134 +++++++++++++ src/tickreplay/sweep.py | 32 ++++ src/tickreplay/synth.py | 44 +++++ tests/reference.py | 106 +++++++++++ tests/test_bocpd.py | 67 +++++++ tests/test_engine.py | 261 ++++++++++++++++++++++++++ tests/test_ingest.py | 154 +++++++++++++++ tests/test_labels.py | 178 ++++++++++++++++++ tests/test_registry.py | 90 +++++++++ tests/test_stats.py | 85 +++++++++ tests/test_store.py | 194 +++++++++++++++++++ tests/test_sweep.py | 74 ++++++++ tests/test_synth.py | 60 ++++++ tests/test_validation.py | 104 ++++++++++ 32 files changed, 2649 insertions(+) create mode 100644 .github/workflows/ci.yml create mode 100644 .gitignore create mode 100644 LICENSE create mode 100644 README.md create mode 100644 examples/benchmark.py create mode 100644 examples/breakout.py create mode 100644 examples/ma_cross.py create mode 100644 examples/sweep_and_validate.py create mode 100644 pyproject.toml create mode 100644 src/tickreplay/__init__.py create mode 100644 src/tickreplay/engine.py create mode 100644 src/tickreplay/ingest.py create mode 100644 src/tickreplay/research/__init__.py create mode 100644 src/tickreplay/research/bocpd.py create mode 100644 src/tickreplay/research/labels.py create mode 100644 src/tickreplay/research/registry.py create mode 100644 src/tickreplay/research/stats.py create mode 100644 src/tickreplay/research/validation.py create mode 100644 src/tickreplay/store.py create mode 100644 src/tickreplay/sweep.py create mode 100644 src/tickreplay/synth.py create mode 100644 tests/reference.py create mode 100644 tests/test_bocpd.py create mode 100644 tests/test_engine.py create mode 100644 tests/test_ingest.py create mode 100644 tests/test_labels.py create mode 100644 tests/test_registry.py create mode 100644 tests/test_stats.py create mode 100644 tests/test_store.py create mode 100644 tests/test_sweep.py create mode 100644 tests/test_synth.py create mode 100644 tests/test_validation.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..de50e19 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,21 @@ +name: ci + +on: [push, pull_request] + +jobs: + test: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest] + python-version: ["3.11", "3.13"] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + cache: pip + cache-dependency-path: pyproject.toml + - run: pip install -e ".[dev]" + - run: pytest -q diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..80d4a52 --- /dev/null +++ b/.gitignore @@ -0,0 +1,13 @@ +__pycache__/ +*.py[cod] +*.egg-info/ +.eggs/ +build/ +dist/ +.venv/ +venv/ +.pytest_cache/ +.coverage +/data/ +/*.npy +.DS_Store diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..17f1236 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 danielmjrr + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..26a1ec2 --- /dev/null +++ b/README.md @@ -0,0 +1,140 @@ +# tickreplay + +[![ci](https://github.com/danielmjrr/tickreplay/actions/workflows/ci.yml/badge.svg)](https://github.com/danielmjrr/tickreplay/actions/workflows/ci.yml) + +Tick-accurate backtesting for MetaTrader 5 data, built around two ideas: + +1. **Bar backtests lie.** Inside one OHLC bar you cannot know whether the high or the + low printed first, so any bar-level simulation of a stop loss and take profit is + guessing about path. tickreplay replays every tick, so barrier ordering, spread + costs, and fill prices are exact. +2. **Most backtest profits are statistical artifacts.** The library ships the + machinery to catch that: purged walk-forward splits, overlap-corrected + significance tests, and an append-only evaluation registry with cumulative + false-discovery control. + +It is fast enough that neither idea costs you iteration speed: the replay kernel +streams a year of ticks in well under a second on a laptop. + +## Install + +``` +pip install git+https://github.com/danielmjrr/tickreplay +``` + +Python ≥ 3.11. Dependencies: numpy, pandas, numba, pyarrow, scipy. + +## Quickstart + +```python +import numpy as np +import tickreplay as tr + +# synthetic data so this runs without any broker export +time_ns, bid, ask = tr.synth.generate_ticks(days=30, seed=7) +store = tr.TickStore(time_ns, bid, ask) +bars = store.bars("M5") + +# a signal is one int per bar: +1 buy, -1 sell, 0 nothing +fast = bars["close"].rolling(10).mean() +slow = bars["close"].rolling(50).mean() +above = (fast > slow).astype("int8") +signal = np.where(above.diff() > 0, 1, np.where(above.diff() < 0, -1, 0)) + +result = tr.replay(store, bars, signal, + tr.Config(sl=3.0, tp=6.0, max_hold="4h", cost=0.05)) +print(result.summary()) +print(result.trades.head()) +``` + +Entries happen at the first tick after the signal bar closes (buys at ask, sells at +bid); exits are checked tick-by-tick in the order stop → target → time → opposite +signal. One position at a time — the result is the *tradable* expectancy of the +strategy, not an overlapped ideal. Full semantics are documented in +`tickreplay/engine.py`. + +## Using your own data + +Export ticks from MT5 (View → Symbols → select symbol → Ticks tab → request a range → +Export). That produces a tab-separated CSV. Then: + +```python +import tickreplay as tr + +tr.ingest.ticks_to_monthly_parquet("XAUUSD_ticks.csv", "data/xauusd/parquet") +store = tr.TickStore.build("data/xauusd/parquet", "data/xauusd/store") +# later sessions: +store = tr.TickStore.open("data/xauusd/store") +``` + +The store is a set of memory-mapped flat arrays: opening it is instant, the OS page +cache shares it across processes, and a years-long tick history costs no RAM until +touched. `tr.ingest.validate_against_broker_bars` cross-checks the tick export +against the broker's own M1 bars — recommended before trusting any new export +(mismatches usually mean the tick and bar files came from different servers). + +## Why it's fast + +- **Storage**: ticks live in memory-mapped `int64/float64` arrays; bars carry their + tick index spans, so mapping a bar to its ticks is O(1) and derived data never + duplicates the tick stream. +- **Kernel**: the replay loop is a single numba-JIT function, `nogil` and + branch-light, that streams the arrays once. It is memory-bandwidth-bound — on an + Apple M4 it processes roughly a billion ticks per second. +- **Sweeps**: because the kernel releases the GIL and the data is shared, parameter + grids fan out over threads with zero copies (`tr.sweep_exits`). A 100-config grid + over years of ticks takes seconds. + +Run `python examples/benchmark.py` for numbers on your machine. + +## Why you can trust it + +- **Two independent engines.** The numba kernel is tested trade-for-trade, + byte-for-byte against a deliberately naive pure-Python reference implementation + (`tests/reference.py`) across randomized fixtures and config combinations. Any + semantic drift between the two fails CI. +- **Look-ahead is tested, not assumed.** Truncation tests verify that removing + future data leaves all earlier trades byte-identical; bar spans make the + no-look-ahead boundary explicit (a bar exists only after its last tick). +- **The research module assumes you will fool yourself** and gets in the way: + - `research.validation.purged_expanding_folds` — walk-forward splits that purge + training samples whose label window leaks into the test period (plus embargo); + - `research.stats.thin_by_gap` — overlapping trade windows inflate t-statistics; + test a non-overlapping subset instead; + - `research.registry` — an append-only log of *every* evaluation you ever ran, + with Benjamini-Hochberg q-values recomputed over the full history on every + read. Run a thousand configs and the bar for "significant" rises accordingly; + - `research.labels` — tick-exact excursion and first-passage labels for + measurement (where bar data cannot even order target-vs-stop correctly); + - `research.bocpd` — online Bayesian changepoint detection (Adams & MacKay), + strictly filtered, with causality enforced by test. + +`examples/sweep_and_validate.py` shows the intended workflow end to end — including +what an honest negative result looks like. + +## Examples + +All examples run on synthetic data in about a second, no broker export needed: + +- [`examples/ma_cross.py`](examples/ma_cross.py) — moving-average cross, fixed stops +- [`examples/breakout.py`](examples/breakout.py) — N-bar breakout with + opposite-signal exits and an exit-reason breakdown +- [`examples/sweep_and_validate.py`](examples/sweep_and_validate.py) — the intended + research loop: sweep a grid, then test the winner with thinning and a + selection-adjusted threshold +- [`examples/benchmark.py`](examples/benchmark.py) — throughput measurement on your + hardware (on an M4 MacBook Air: ~1,100M ticks/s scanned inside positions, 20M-tick + store replayed in under 4 ms) + +## Scope and limitations + +- Single position, netting-style: no pyramiding, no hedged legs, no partial fills. +- Quote data only (bid/ask). Spot FX/metals CFDs have no real volume feed; if your + strategy needs volume, you need futures data instead. +- Costs = spread (paid via bid/ask fills) + a fixed round-trip add. No market-impact + model — realistic for retail sizes, not for size. +- Timestamps are broker time; the library never converts time zones. + +## License + +MIT diff --git a/examples/benchmark.py b/examples/benchmark.py new file mode 100644 index 0000000..604a63a --- /dev/null +++ b/examples/benchmark.py @@ -0,0 +1,81 @@ +"""Replay throughput on synthetic ticks. + +Run: python examples/benchmark.py [--ticks 20000000] +""" +import argparse +import statistics +import time + +import numpy as np + +import tickreplay as tr + +TICKS_PER_DAY = 200_000 +WARMUP_TICKS = 100_000 + + +def build_store(n_ticks, seed): + # generate_ticks skips weekends, so ask for enough calendar days to cover it. + days = int(np.ceil(n_ticks / TICKS_PER_DAY * 7 / 5)) + 3 + time_ns, bid, ask = tr.synth.generate_ticks( + days=days, ticks_per_day=TICKS_PER_DAY, seed=seed) + return tr.TickStore(time_ns[:n_ticks], bid[:n_ticks], ask[:n_ticks]) + + +def ma_cross_signal(bars): + fast = bars["close"].rolling(10).mean() + slow = bars["close"].rolling(50).mean() + regime = np.sign(fast - slow).fillna(0.0).to_numpy() + return np.where(regime != np.r_[0.0, regime[:-1]], regime, 0).astype("int8") + + +def main(): + parser = argparse.ArgumentParser(description="Replay throughput on synthetic ticks.") + parser.add_argument("--ticks", type=int, default=20_000_000, + help="store size; 20M ticks needs about 1 GB of RAM") + parser.add_argument("--timeframe", default="M5", choices=sorted(tr.TIMEFRAMES), + help="bar timeframe the signal is built on") + parser.add_argument("--seed", type=int, default=1, help="synthetic data seed") + args = parser.parse_args() + + t0 = time.perf_counter() + store = build_store(args.ticks, args.seed) + gen_ms = (time.perf_counter() - t0) * 1e3 + + config = tr.Config(sl=3.0, tp=6.0, max_hold="4h", cost=0.05) + + # Compile the kernel on a small slice so JIT time stays out of the measurement. + warm = tr.TickStore(store.time_ns[:WARMUP_TICKS], store.bid[:WARMUP_TICKS], + store.ask[:WARMUP_TICKS]) + warm_bars = warm.bars(args.timeframe) + tr.replay(warm, warm_bars, ma_cross_signal(warm_bars), config) + + t0 = time.perf_counter() + bars = store.bars(args.timeframe) + bars_ms = (time.perf_counter() - t0) * 1e3 + signal = ma_cross_signal(bars) + + timings = [] + for _ in range(3): + t0 = time.perf_counter() + result = tr.replay(store, bars, signal, config) + timings.append((time.perf_counter() - t0) * 1e3) + median_ms = statistics.median(timings) + + # The kernel only walks ticks while a position is open, so quote the covered + # fraction alongside the headline rate. + visited = int((result.arrays["exit_idx"] - result.arrays["entry_idx"]).sum()) + + print(f"ticks {len(store):>14,}") + print(f"{args.timeframe} bars{'':<10}{len(bars):>14,}") + print(f"trades {len(result):>14,}") + print(f"ticks in position{visited:>14,} ({visited / max(len(store), 1):.1%})") + print(f"generate {gen_ms:>14,.1f} ms") + print(f"derive bars {bars_ms:>14,.1f} ms") + print(f"replay (median) {median_ms:>14,.1f} ms of {[round(x, 1) for x in timings]}") + print(f"kernel rate {visited / median_ms / 1e3:>14,.1f} M ticks/s scanned") + print(f"end-to-end rate {len(store) / median_ms / 1e3:>14,.1f} M store ticks/s") + + +if __name__ == "__main__": + main() diff --git a/examples/breakout.py b/examples/breakout.py new file mode 100644 index 0000000..6663b51 --- /dev/null +++ b/examples/breakout.py @@ -0,0 +1,46 @@ +"""N-bar breakout on M5 bars, exiting on the opposite breakout. + +Run: python examples/breakout.py +""" +import numpy as np + +import tickreplay as tr + +LOOKBACK = 20 + + +def main(): + time_ns, bid, ask = tr.synth.generate_ticks(days=30, seed=11) + store = tr.TickStore(time_ns, bid, ask) + bars = store.bars("M5") + + # shift(1) keeps the current bar out of its own reference range. + prior_high = bars["high"].shift(1).rolling(LOOKBACK).max() + prior_low = bars["low"].shift(1).rolling(LOOKBACK).min() + state = np.where(bars["close"] > prior_high, 1, + np.where(bars["close"] < prior_low, -1, 0)).astype("int8") + + # Fire on the bar that breaks out, not on every bar that stays beyond the range. + signal = np.where(state != np.r_[np.int8(0), state[:-1]], state, 0).astype("int8") + + config = tr.Config(sl=4.0, tp=8.0, max_hold="8h", exit_on_opposite=True, cost=0.05) + result = tr.replay(store, bars, signal, config) + + print(f"ticks {len(store):>12,}") + print(f"M5 bars {len(bars):>12,}") + print(f"signals {int((signal != 0).sum()):>12,} ({LOOKBACK}-bar breakout)") + print("config sl=4.0 tp=8.0 max_hold=8h exit_on_opposite=True cost=0.05") + print() + for key, value in result.summary().items(): + print(f"{key:<14}{value:>12.4f}" if isinstance(value, float) + else f"{key:<14}{value:>12,}") + + print() + print("exit reasons") + counts = result.trades["exit_reason"].value_counts() + for reason, count in counts.items(): + print(f" {reason:<10}{count:>6,}{count / counts.sum():>8.1%}") + + +if __name__ == "__main__": + main() diff --git a/examples/ma_cross.py b/examples/ma_cross.py new file mode 100644 index 0000000..bf33f41 --- /dev/null +++ b/examples/ma_cross.py @@ -0,0 +1,41 @@ +"""Moving-average cross on M5 bars, replayed tick by tick. + +Run: python examples/ma_cross.py +""" +import numpy as np + +import tickreplay as tr + + +def main(): + time_ns, bid, ask = tr.synth.generate_ticks(days=30, seed=7) + store = tr.TickStore(time_ns, bid, ask) + bars = store.bars("M5") + + fast = bars["close"].rolling(10).mean() + slow = bars["close"].rolling(50).mean() + + # Regime is +1 while fast is above slow. The signal fires only on the bar where + # the regime flips, so one cross is one entry instead of one entry per bar. + regime = np.sign(fast - slow).fillna(0.0).to_numpy() + previous = np.r_[0.0, regime[:-1]] + signal = np.where(regime != previous, regime, 0).astype("int8") + + # signal[b] is built from closes up to and including bar b, so it is knowable + # only once bar b has closed. No extra shift is needed: the engine enters at the + # first tick after the signal bar closes. + config = tr.Config(sl=3.0, tp=6.0, max_hold="4h", cost=0.05) + result = tr.replay(store, bars, signal, config) + + print(f"ticks {len(store):>12,}") + print(f"M5 bars {len(bars):>12,}") + print(f"signals {int((signal != 0).sum()):>12,}") + print("config sl=3.0 tp=6.0 max_hold=4h cost=0.05 (price units)") + print() + for key, value in result.summary().items(): + print(f"{key:<14}{value:>12.4f}" if isinstance(value, float) + else f"{key:<14}{value:>12,}") + + +if __name__ == "__main__": + main() diff --git a/examples/sweep_and_validate.py b/examples/sweep_and_validate.py new file mode 100644 index 0000000..4c8b868 --- /dev/null +++ b/examples/sweep_and_validate.py @@ -0,0 +1,59 @@ +"""Sweep exit rules, then test the winner honestly. + +Picking the best cell of a grid and reporting its p-value as-is is the standard way +to manufacture an edge. This example sweeps, then thins overlapping trades before +testing, and reads the result against a selection-adjusted threshold. + +Run: python examples/sweep_and_validate.py +""" +import numpy as np + +import tickreplay as tr + +HOUR_NS = 3_600 * 10**9 + + +def main(): + time_ns, bid, ask = tr.synth.generate_ticks(days=60, seed=3) + store = tr.TickStore(time_ns, bid, ask) + bars = store.bars("M5") + + fast = bars["close"].rolling(10).mean() + slow = bars["close"].rolling(50).mean() + regime = np.sign(fast - slow).fillna(0.0).to_numpy() + signal = np.where(regime != np.r_[0.0, regime[:-1]], regime, 0).astype("int8") + + base = {"max_hold": "4h", "cost": 0.05} + grid = [{"sl": sl, "tp": tp} for sl in (2.0, 3.0, 4.0) for tp in (4.0, 6.0, 8.0)] + table = tr.sweep_exits(store, bars, signal, grid, base=base) + table = table.sort_values("total_net", ascending=False).reset_index(drop=True) + + print(f"{len(grid)} exit configs over {len(store):,} ticks\n") + print(table[["sl", "tp", "n_trades", "total_net", "hit_rate", + "max_drawdown"]].to_string(index=False, float_format="%.3f")) + + best = table.iloc[0] + result = tr.replay(store, bars, signal, + tr.Config(sl=best["sl"], tp=best["tp"], **base)) + pnl = result.arrays["pnl_net"] + + # Trades whose holding windows overlap are correlated; a t-stat over all of them + # is inflated. Test a non-overlapping subset spaced by the max hold time. + keep = tr.research.stats.thin_by_gap(result.arrays["entry_time_ns"], 4 * HOUR_NS) + t_stat, p_value = tr.research.stats.one_sided_positive(pnl[keep]) + threshold = 0.05 / len(grid) + + print(f"\nbest by total_net: sl={best['sl']} tp={best['tp']}") + print(f" trades {len(pnl):>8,} -> {int(keep.sum()):,} after thinning") + print(f" one-sided t {t_stat:>8.3f}") + print(f" p (mean > 0) {p_value:>8.4f}") + print(f" threshold (0.05/{len(grid)}) {threshold:>8.4f}") + print(f"\n verdict: {'NOT significant' if p_value > threshold else 'passes'}" + " once selection over the grid is accounted for.") + print(" Expected: this is a synthetic random walk with costs, so there is no edge" + "\n to find. A pass here would mean the procedure is broken, not that the" + "\n strategy works.") + + +if __name__ == "__main__": + main() diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..39ad4cf --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,38 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "tickreplay" +version = "0.1.0" +description = "Tick-accurate backtesting for MT5 data: memory-mapped tickstores, a JIT replay kernel, and research tools that guard against overfitting" +readme = "README.md" +license = { file = "LICENSE" } +authors = [{ name = "danielmjrr" }] +requires-python = ">=3.11" +dependencies = [ + "numpy>=1.26", + "pandas>=2.2", + "numba>=0.59", + "pyarrow>=15.0", + "scipy>=1.11", +] +classifiers = [ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", + "Intended Audience :: Financial and Insurance Industry", + "Topic :: Office/Business :: Financial :: Investment", +] + +[project.urls] +Homepage = "https://github.com/danielmjrr/tickreplay" + +[project.optional-dependencies] +dev = ["pytest>=8.0"] + +[tool.hatch.build.targets.wheel] +packages = ["src/tickreplay"] + +[tool.pytest.ini_options] +testpaths = ["tests"] diff --git a/src/tickreplay/__init__.py b/src/tickreplay/__init__.py new file mode 100644 index 0000000..ee6663e --- /dev/null +++ b/src/tickreplay/__init__.py @@ -0,0 +1,27 @@ +"""tickreplay: tick-accurate backtesting for MT5 data. + +Quickstart:: + + import numpy as np + import tickreplay as tr + + store = tr.TickStore.open("data/my_symbol") # built once from an MT5 export + bars = store.bars("M5") + + fast = bars.close.rolling(10).mean() + slow = bars.close.rolling(50).mean() + signal = np.sign(fast - slow).fillna(0).astype("int8") + signal[signal == np.roll(signal, 1)] = 0 # act on crosses only + + result = tr.replay(store, bars, signal, + tr.Config(sl=5.0, tp=10.0, max_hold="4h", cost=0.09)) + print(result.summary()) +""" +from . import ingest, research, synth +from .engine import Config, Result, replay +from .store import TIMEFRAMES, TickStore, derive_bars +from .sweep import run_sweep, sweep_exits + +__version__ = "0.1.0" +__all__ = ["Config", "Result", "TickStore", "TIMEFRAMES", "derive_bars", "ingest", + "replay", "research", "run_sweep", "sweep_exits", "synth"] diff --git a/src/tickreplay/engine.py b/src/tickreplay/engine.py new file mode 100644 index 0000000..ee87ebe --- /dev/null +++ b/src/tickreplay/engine.py @@ -0,0 +1,181 @@ +"""Single-position tick replay. + +Semantics, chosen to match how a live EA on a netting account behaves: + +- A nonzero signal on bar ``b`` enters at the first tick after the bar closes + (``tick_last + 1``) — buys at ask, sells at bid. No same-bar fills, no look-ahead. +- One position at a time. Signals arriving while a position is open are skipped, so + results measure the *tradable* expectancy of the strategy, not an overlapped ideal. +- Exit checks per tick, in priority order: stop loss, take profit, max hold time, + opposite signal. Anything still open at the last tick closes there (``eod``). +- Positions are marked to the liquidation side: longs to bid, shorts to ask. Stops + and targets are distances from the entry price in price units and may vary per bar + (e.g. ATR-scaled). +- A signal on the exit tick itself is ignored; the next entry needs a later tick. +""" +from dataclasses import dataclass, field + +import numpy as np +import pandas as pd +from numba import njit + +EXIT_REASONS = {1: "sl", 2: "tp", 3: "time", 4: "opposite", 5: "eod"} + + +@dataclass +class Config: + """Exit rules and costs for one replay. + + ``sl``/``tp``: distance from entry in price units — a scalar for all bars or an + array per bar; 0 disables. ``max_hold``: pandas offset string ("4h") or seconds; + 0 disables. ``cost``: round-trip cost added on top of the spread, in price units + (spread itself is already paid via bid/ask fills). + """ + sl: float | np.ndarray = 0.0 + tp: float | np.ndarray = 0.0 + max_hold: str | float = 0 + exit_on_opposite: bool = False + allow_buy: bool = True + allow_sell: bool = True + cost: float = 0.0 + + def max_hold_ns(self): + if isinstance(self.max_hold, str): + return pd.Timedelta(self.max_hold).value + return int(self.max_hold) * 10**9 + int(self.max_hold % 1 * 1e9) + + +@dataclass +class Result: + """Trade ledger from one replay. ``arrays`` holds the raw numpy columns.""" + arrays: dict = field(repr=False) + config: Config = field(repr=False) + + def __len__(self): + return len(self.arrays["dir"]) + + @property + def trades(self): + a = self.arrays + df = pd.DataFrame({k: v for k, v in a.items() if k not in + ("entry_time_ns", "exit_time_ns", "exit_reason")}) + df["entry_time"] = pd.to_datetime(a["entry_time_ns"], unit="ns") + df["exit_time"] = pd.to_datetime(a["exit_time_ns"], unit="ns") + df["exit_reason"] = pd.Series(a["exit_reason"]).map(EXIT_REASONS) + return df + + def summary(self): + pnl = self.arrays["pnl_net"] + if len(pnl) == 0: + return {"n_trades": 0, "total_net": 0.0, "mean_net": 0.0, "hit_rate": 0.0, + "max_drawdown": 0.0} + cum = np.cumsum(pnl) + dd = np.maximum.accumulate(cum) - cum + return { + "n_trades": len(pnl), + "total_net": float(cum[-1]), + "mean_net": float(pnl.mean()), + "hit_rate": float((pnl > 0).mean()), + "max_drawdown": float(dd.max()), + } + + +@njit(cache=True, nogil=True) +def _replay(time_ns, bid, ask, bar_tick_last, bar_signal, sl_dist, tp_dist, + allow_buy, allow_sell, max_hold_ns, exit_on_opposite, + o_dir, o_entry_idx, o_exit_idx, o_entry_price, o_exit_price, + o_pnl_gross, o_mfe, o_mae, o_reason): + n_ticks = time_ns.shape[0] + n_bars = bar_signal.shape[0] + t = 0 + min_trigger = 0 # post-exit: exit tick + 1 (signals at the exit tick are ignored) + for b in range(n_bars): + s = bar_signal[b] + if s == 0 or (s == 1 and not allow_buy) or (s == -1 and not allow_sell): + continue + e = bar_tick_last[b] + 1 + if e < min_trigger: + continue + if e >= n_ticks - 1: # no tick after entry to evaluate -> no trade + continue + entry_price = ask[e] if s == 1 else bid[e] + deadline = time_ns[e] + max_hold_ns + sl = sl_dist[b] + tp = tp_dist[b] + opp_trigger = n_ticks # sentinel: never reached + if exit_on_opposite: + for ob in range(b + 1, n_bars): + if bar_signal[ob] == -s: + opp_trigger = bar_tick_last[ob] + 1 + break + mfe = 0.0 + mae = 0.0 + exit_idx = n_ticks - 1 + reason = 5 # eod + for i in range(e + 1, n_ticks): + pnl_raw = bid[i] - entry_price if s == 1 else entry_price - ask[i] + if pnl_raw > mfe: + mfe = pnl_raw + if -pnl_raw > mae: + mae = -pnl_raw + if sl > 0 and pnl_raw <= -sl: + exit_idx = i; reason = 1; break + if tp > 0 and pnl_raw >= tp: + exit_idx = i; reason = 2; break + if max_hold_ns > 0 and time_ns[i] >= deadline: + exit_idx = i; reason = 3; break + if i == opp_trigger: + exit_idx = i; reason = 4; break + exit_price = bid[exit_idx] if s == 1 else ask[exit_idx] + o_dir[t] = s + o_entry_idx[t] = e + o_exit_idx[t] = exit_idx + o_entry_price[t] = entry_price + o_exit_price[t] = exit_price + o_pnl_gross[t] = exit_price - entry_price if s == 1 else entry_price - exit_price + o_mfe[t] = mfe + o_mae[t] = mae + o_reason[t] = reason + t += 1 + min_trigger = exit_idx + 1 + return t + + +def replay(store, bars, signal, config=None): + """Replay ``signal`` (+1 buy / -1 sell / 0 flat, one value per bar) through the + tick stream. Returns a :class:`Result`.""" + config = config or Config() + signal = np.ascontiguousarray(signal, dtype=np.int8) + if len(signal) != len(bars): + raise ValueError(f"signal has {len(signal)} values for {len(bars)} bars") + tick_last = np.ascontiguousarray(bars["tick_last"].to_numpy(), np.int64) + if len(tick_last) > 1 and (np.diff(tick_last) <= 0).any(): + raise ValueError("bars must have strictly increasing tick_last") + n_bars = len(bars) + sl = np.ascontiguousarray(np.broadcast_to(np.nan_to_num(config.sl), n_bars), + dtype=np.float64) + tp = np.ascontiguousarray(np.broadcast_to(np.nan_to_num(config.tp), n_bars), + dtype=np.float64) + cap = int((signal != 0).sum()) + 1 + o = { + "dir": np.zeros(cap, np.int8), + "entry_idx": np.zeros(cap, np.int64), "exit_idx": np.zeros(cap, np.int64), + "entry_price": np.zeros(cap, np.float64), "exit_price": np.zeros(cap, np.float64), + "pnl_gross": np.zeros(cap, np.float64), + "mfe": np.zeros(cap, np.float64), "mae": np.zeros(cap, np.float64), + "exit_reason": np.zeros(cap, np.int8), + } + time_ns = np.ascontiguousarray(store.time_ns, np.int64) + n = _replay(time_ns, + np.ascontiguousarray(store.bid, np.float64), + np.ascontiguousarray(store.ask, np.float64), + tick_last, + signal, sl, tp, config.allow_buy, config.allow_sell, + np.int64(config.max_hold_ns()), config.exit_on_opposite, + o["dir"], o["entry_idx"], o["exit_idx"], o["entry_price"], + o["exit_price"], o["pnl_gross"], o["mfe"], o["mae"], o["exit_reason"]) + arrays = {k: v[:n].copy() for k, v in o.items()} + arrays["entry_time_ns"] = time_ns[arrays["entry_idx"]] + arrays["exit_time_ns"] = time_ns[arrays["exit_idx"]] + arrays["pnl_net"] = arrays["pnl_gross"] - config.cost + return Result(arrays, config) diff --git a/src/tickreplay/ingest.py b/src/tickreplay/ingest.py new file mode 100644 index 0000000..dd1cb68 --- /dev/null +++ b/src/tickreplay/ingest.py @@ -0,0 +1,123 @@ +"""MT5 export ingestion: tick CSV -> monthly parquet, plus reconstruction checks. + +MT5 tick exports are tab-separated: