Pratik Bhadane
71b6343e92
Refresh the benchmark and performance surface across the repo. This updates the benchmark wrappers and helper scripts, regenerates the checked-in benchmark and perf-contract artifacts, and folds in the related roadmap, compatibility, and example notebook changes that belong with this performance-focused pass. Harden the Python CI and local pre-push flow so the same checks pass reliably in both places. The workflow and pre-push script now use module-safe uv typecheck invocations, the Python test environment installs the optional MCP dependency needed by the MCP server tests, and one-off root benchmark outputs are ignored to keep the repo clean. Align local tooling with the current project configuration by updating the Ruff pre-commit hook, tightening the API typing and MCP server helpers, and refreshing the lockfile to pick up the audited PyJWT fix while preserving the rest of the staged source changes.
482 lines
15 KiB
Python
482 lines
15 KiB
Python
"""
|
|
ferro_ta vs TA-Lib speed comparison.
|
|
|
|
Measures throughput (M bars/s) for both libraries on the same synthetic data
|
|
and parameters. The output is intentionally evidence-heavy:
|
|
|
|
- median timings
|
|
- per-run timing samples
|
|
- variability stats
|
|
- Python-tracked peak allocation snapshots
|
|
- machine, runtime, and build metadata
|
|
|
|
This is meant to support a narrow claim: ferro-ta is often faster on selected
|
|
indicators, not universally faster.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import math
|
|
import sys
|
|
import time
|
|
import tracemalloc
|
|
from typing import Any
|
|
|
|
import numpy as np
|
|
|
|
try:
|
|
import talib # noqa: F401
|
|
|
|
TALIB_AVAILABLE = True
|
|
except ImportError:
|
|
TALIB_AVAILABLE = False
|
|
talib = None # type: ignore[assignment]
|
|
|
|
import ferro_ta
|
|
|
|
try:
|
|
from benchmarks.metadata import benchmark_metadata, package_versions
|
|
except ModuleNotFoundError: # pragma: no cover - script execution fallback
|
|
from metadata import benchmark_metadata, package_versions
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Configuration
|
|
# ---------------------------------------------------------------------------
|
|
|
|
N_WARMUP = 1
|
|
N_RUNS = 7
|
|
DEFAULT_SIZES = [10_000, 100_000, 1_000_000]
|
|
TIE_EPSILON = 0.05
|
|
|
|
_rng = np.random.default_rng(42)
|
|
|
|
|
|
def _median(values: list[float]) -> float:
|
|
ordered = sorted(values)
|
|
mid = len(ordered) // 2
|
|
if len(ordered) % 2:
|
|
return ordered[mid]
|
|
return (ordered[mid - 1] + ordered[mid]) / 2.0
|
|
|
|
|
|
def _summary_stats(samples_ms: list[float]) -> dict[str, float]:
|
|
if not samples_ms:
|
|
return {
|
|
"median_ms": 0.0,
|
|
"mean_ms": 0.0,
|
|
"min_ms": 0.0,
|
|
"max_ms": 0.0,
|
|
"stddev_ms": 0.0,
|
|
"cv_pct": 0.0,
|
|
}
|
|
|
|
mean_ms = sum(samples_ms) / len(samples_ms)
|
|
variance = (
|
|
sum((sample - mean_ms) ** 2 for sample in samples_ms) / (len(samples_ms) - 1)
|
|
if len(samples_ms) > 1
|
|
else 0.0
|
|
)
|
|
stddev_ms = math.sqrt(variance)
|
|
cv_pct = (stddev_ms / mean_ms * 100.0) if mean_ms else 0.0
|
|
return {
|
|
"median_ms": round(_median(samples_ms), 4),
|
|
"mean_ms": round(mean_ms, 4),
|
|
"min_ms": round(min(samples_ms), 4),
|
|
"max_ms": round(max(samples_ms), 4),
|
|
"stddev_ms": round(stddev_ms, 4),
|
|
"cv_pct": round(cv_pct, 3),
|
|
}
|
|
|
|
|
|
def _outcome(speedup: float) -> str:
|
|
if speedup > 1.0 + TIE_EPSILON:
|
|
return "ferro_ta_win"
|
|
if speedup < 1.0 - TIE_EPSILON:
|
|
return "talib_win"
|
|
return "tie"
|
|
|
|
|
|
def _summary_for_size(results: list[dict[str, Any]], size: int) -> dict[str, Any]:
|
|
rows = [row for row in results if row.get("size") == size and "speedup" in row]
|
|
if not rows:
|
|
return {"size": size, "rows": 0}
|
|
|
|
speedups = [float(row["speedup"]) for row in rows]
|
|
wins = sum(1 for row in rows if row.get("outcome") == "ferro_ta_win")
|
|
ties = sum(1 for row in rows if row.get("outcome") == "tie")
|
|
losses = sum(1 for row in rows if row.get("outcome") == "talib_win")
|
|
return {
|
|
"size": size,
|
|
"rows": len(rows),
|
|
"wins": wins,
|
|
"ties": ties,
|
|
"losses": losses,
|
|
"win_rate": round(wins / len(rows), 4),
|
|
"non_loss_rate": round((wins + ties) / len(rows), 4),
|
|
"median_speedup": round(_median(speedups), 4),
|
|
"min_speedup": round(min(speedups), 4),
|
|
"max_speedup": round(max(speedups), 4),
|
|
"talib_wins_or_ties": [
|
|
row["indicator"]
|
|
for row in rows
|
|
if row.get("outcome") in {"talib_win", "tie"}
|
|
],
|
|
}
|
|
|
|
|
|
def _synthetic_ohlcv(
|
|
n: int,
|
|
) -> tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray]:
|
|
# Generate OHLCV so that ta crate DataItem constraints hold: low >= 0,
|
|
# volume >= 0, and low <= open, close <= high, high >= open.
|
|
close = 100.0 + np.cumsum(_rng.standard_normal(n) * 0.5)
|
|
open_ = close + _rng.standard_normal(n) * 0.2
|
|
high = np.maximum(open_, close) + np.abs(_rng.standard_normal(n) * 0.3)
|
|
low = np.minimum(open_, close) - np.abs(_rng.standard_normal(n) * 0.3)
|
|
high = np.maximum(high, low)
|
|
low = np.maximum(low, 0.0)
|
|
high = np.maximum(high, low)
|
|
open_ = np.clip(open_, low, high)
|
|
close = np.clip(close, low, high)
|
|
volume = np.abs(_rng.standard_normal(n) * 1_000_000) + 500_000
|
|
return open_, high, low, close, volume
|
|
|
|
|
|
def _timed_runs_ms(fn, *args, **kwargs) -> list[float]:
|
|
for _ in range(N_WARMUP):
|
|
fn(*args, **kwargs)
|
|
|
|
samples_ms: list[float] = []
|
|
for _ in range(N_RUNS):
|
|
t0 = time.perf_counter()
|
|
fn(*args, **kwargs)
|
|
samples_ms.append((time.perf_counter() - t0) * 1000.0)
|
|
return samples_ms
|
|
|
|
|
|
def _python_peak_bytes(fn, *args, **kwargs) -> int | None:
|
|
try:
|
|
tracemalloc.start()
|
|
tracemalloc.reset_peak()
|
|
fn(*args, **kwargs)
|
|
_, peak = tracemalloc.get_traced_memory()
|
|
return int(peak)
|
|
except Exception:
|
|
return None
|
|
finally:
|
|
tracemalloc.stop()
|
|
|
|
|
|
def _throughput_m_bars_s(size: int, median_ms: float) -> float:
|
|
if median_ms <= 0:
|
|
return 0.0
|
|
return (size / 1e6) / (median_ms / 1000.0)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Benchmarked callables
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _run_ft_sma(o, h, l, c, v, n):
|
|
return ferro_ta.SMA(c[:n], timeperiod=14)
|
|
|
|
|
|
def _run_ta_sma(o, h, l, c, v, n):
|
|
return talib.SMA(c[:n], timeperiod=14)
|
|
|
|
|
|
def _run_ft_ema(o, h, l, c, v, n):
|
|
return ferro_ta.EMA(c[:n], timeperiod=14)
|
|
|
|
|
|
def _run_ta_ema(o, h, l, c, v, n):
|
|
return talib.EMA(c[:n], timeperiod=14)
|
|
|
|
|
|
def _run_ft_rsi(o, h, l, c, v, n):
|
|
return ferro_ta.RSI(c[:n], timeperiod=14)
|
|
|
|
|
|
def _run_ta_rsi(o, h, l, c, v, n):
|
|
return talib.RSI(c[:n], timeperiod=14)
|
|
|
|
|
|
def _run_ft_bbands(o, h, l, c, v, n):
|
|
return ferro_ta.BBANDS(c[:n], timeperiod=20, nbdevup=2.0, nbdevdn=2.0)
|
|
|
|
|
|
def _run_ta_bbands(o, h, l, c, v, n):
|
|
return talib.BBANDS(c[:n], timeperiod=20, nbdevup=2.0, nbdevdn=2.0)
|
|
|
|
|
|
def _run_ft_macd(o, h, l, c, v, n):
|
|
return ferro_ta.MACD(c[:n], fastperiod=12, slowperiod=26, signalperiod=9)
|
|
|
|
|
|
def _run_ta_macd(o, h, l, c, v, n):
|
|
return talib.MACD(c[:n], fastperiod=12, slowperiod=26, signalperiod=9)
|
|
|
|
|
|
def _run_ft_atr(o, h, l, c, v, n):
|
|
return ferro_ta.ATR(h[:n], l[:n], c[:n], timeperiod=14)
|
|
|
|
|
|
def _run_ta_atr(o, h, l, c, v, n):
|
|
return talib.ATR(h[:n], l[:n], c[:n], timeperiod=14)
|
|
|
|
|
|
def _run_ft_stoch(o, h, l, c, v, n):
|
|
return ferro_ta.STOCH(h[:n], l[:n], c[:n])
|
|
|
|
|
|
def _run_ta_stoch(o, h, l, c, v, n):
|
|
return talib.STOCH(h[:n], l[:n], c[:n])
|
|
|
|
|
|
def _run_ft_adx(o, h, l, c, v, n):
|
|
return ferro_ta.ADX(h[:n], l[:n], c[:n], timeperiod=14)
|
|
|
|
|
|
def _run_ta_adx(o, h, l, c, v, n):
|
|
return talib.ADX(h[:n], l[:n], c[:n], timeperiod=14)
|
|
|
|
|
|
def _run_ft_cci(o, h, l, c, v, n):
|
|
return ferro_ta.CCI(h[:n], l[:n], c[:n], timeperiod=14)
|
|
|
|
|
|
def _run_ta_cci(o, h, l, c, v, n):
|
|
return talib.CCI(h[:n], l[:n], c[:n], timeperiod=14)
|
|
|
|
|
|
def _run_ft_obv(o, h, l, c, v, n):
|
|
return ferro_ta.OBV(c[:n], v[:n])
|
|
|
|
|
|
def _run_ta_obv(o, h, l, c, v, n):
|
|
return talib.OBV(c[:n], v[:n])
|
|
|
|
|
|
def _run_ft_mfi(o, h, l, c, v, n):
|
|
return ferro_ta.MFI(h[:n], l[:n], c[:n], v[:n], timeperiod=14)
|
|
|
|
|
|
def _run_ta_mfi(o, h, l, c, v, n):
|
|
return talib.MFI(h[:n], l[:n], c[:n], v[:n], timeperiod=14)
|
|
|
|
|
|
def _run_ft_wma(o, h, l, c, v, n):
|
|
return ferro_ta.WMA(c[:n], timeperiod=14)
|
|
|
|
|
|
def _run_ta_wma(o, h, l, c, v, n):
|
|
return talib.WMA(c[:n], timeperiod=14)
|
|
|
|
|
|
COMPARISON_CASES = [
|
|
("SMA", _run_ft_sma, _run_ta_sma),
|
|
("EMA", _run_ft_ema, _run_ta_ema),
|
|
("RSI", _run_ft_rsi, _run_ta_rsi),
|
|
("BBANDS", _run_ft_bbands, _run_ta_bbands),
|
|
("MACD", _run_ft_macd, _run_ta_macd),
|
|
("ATR", _run_ft_atr, _run_ta_atr),
|
|
("STOCH", _run_ft_stoch, _run_ta_stoch),
|
|
("ADX", _run_ft_adx, _run_ta_adx),
|
|
("CCI", _run_ft_cci, _run_ta_cci),
|
|
("OBV", _run_ft_obv, _run_ta_obv),
|
|
("MFI", _run_ft_mfi, _run_ta_mfi),
|
|
("WMA", _run_ft_wma, _run_ta_wma),
|
|
]
|
|
|
|
SKIP_1M_FOR = {"STOCH", "ADX"}
|
|
|
|
|
|
def run_comparison(sizes: list[int], json_path: str | None) -> list[dict[str, Any]]:
|
|
max_size = max(sizes)
|
|
open_, high, low, close, volume = _synthetic_ohlcv(max_size)
|
|
results: list[dict[str, Any]] = []
|
|
|
|
col_label = 10
|
|
col_size = 10
|
|
col_ft_ms = 12
|
|
col_ta_ms = 12
|
|
col_speedup = 10
|
|
col_ft_m = 12
|
|
col_ta_m = 12
|
|
|
|
if not TALIB_AVAILABLE:
|
|
print("Note: ta-lib not installed. Reporting ferro_ta timings only.")
|
|
print(
|
|
"Install with: pip install ta-lib (or conda install ta-lib) for comparison.\n"
|
|
)
|
|
|
|
print(
|
|
f"\nferro_ta vs TA-Lib — median of {N_RUNS} measured runs after {N_WARMUP} warmup"
|
|
)
|
|
print(f"Sizes: {sizes}")
|
|
print()
|
|
|
|
header = (
|
|
f"{'Indicator':<{col_label}} {'Size':<{col_size}} "
|
|
f"{'ferro_ta(ms)':<{col_ft_ms}} {'TA-Lib(ms)':<{col_ta_ms}} "
|
|
f"{'Speedup':<{col_speedup}} {'ferro_ta(M/s)':<{col_ft_m}} {'TA-Lib(M/s)':<{col_ta_m}}"
|
|
)
|
|
print(header)
|
|
print("-" * len(header))
|
|
|
|
for name, ft_run, ta_run in COMPARISON_CASES:
|
|
for size in sizes:
|
|
if size == 1_000_000 and name in SKIP_1M_FOR:
|
|
continue
|
|
|
|
ft_samples_ms = _timed_runs_ms(
|
|
ft_run, open_, high, low, close, volume, size
|
|
)
|
|
ft_stats = _summary_stats(ft_samples_ms)
|
|
ft_median_ms = float(ft_stats["median_ms"])
|
|
ft_m_bars_s = _throughput_m_bars_s(size, ft_median_ms)
|
|
ft_peak_bytes = _python_peak_bytes(
|
|
ft_run, open_, high, low, close, volume, size
|
|
)
|
|
|
|
row: dict[str, Any] = {
|
|
"indicator": name,
|
|
"size": size,
|
|
"input_layout": {
|
|
"dtype": "float64",
|
|
"contiguous": True,
|
|
},
|
|
"ferro_ta_ms": round(ft_median_ms, 4),
|
|
"ferro_ta_m_bars_s": round(ft_m_bars_s, 2),
|
|
"ferro_ta_runs_ms": [round(sample, 4) for sample in ft_samples_ms],
|
|
"ferro_ta_stats": ft_stats,
|
|
"python_peak_allocation_bytes": {
|
|
"ferro_ta": ft_peak_bytes,
|
|
},
|
|
}
|
|
|
|
if TALIB_AVAILABLE:
|
|
ta_samples_ms = _timed_runs_ms(
|
|
ta_run, open_, high, low, close, volume, size
|
|
)
|
|
ta_stats = _summary_stats(ta_samples_ms)
|
|
ta_median_ms = float(ta_stats["median_ms"])
|
|
ta_m_bars_s = _throughput_m_bars_s(size, ta_median_ms)
|
|
speedup = (
|
|
ta_median_ms / ft_median_ms if ft_median_ms > 0 else float("inf")
|
|
)
|
|
outcome = _outcome(speedup)
|
|
ta_peak_bytes = _python_peak_bytes(
|
|
ta_run, open_, high, low, close, volume, size
|
|
)
|
|
|
|
print(
|
|
f"{name:<{col_label}} {size:<{col_size}} "
|
|
f"{ft_median_ms:<{col_ft_ms}.3f} {ta_median_ms:<{col_ta_ms}.3f} "
|
|
f"{speedup:<{col_speedup}.2f}x {ft_m_bars_s:<{col_ft_m}.1f} {ta_m_bars_s:<{col_ta_m}.1f}"
|
|
)
|
|
|
|
row.update(
|
|
{
|
|
"talib_ms": round(ta_median_ms, 4),
|
|
"talib_m_bars_s": round(ta_m_bars_s, 2),
|
|
"talib_runs_ms": [round(sample, 4) for sample in ta_samples_ms],
|
|
"talib_stats": ta_stats,
|
|
"speedup": round(speedup, 4),
|
|
"outcome": outcome,
|
|
}
|
|
)
|
|
row["python_peak_allocation_bytes"]["talib"] = ta_peak_bytes
|
|
else:
|
|
print(
|
|
f"{name:<{col_label}} {size:<{col_size}} "
|
|
f"{ft_median_ms:<{col_ft_ms}.3f} {'N/A':<{col_ta_ms}} "
|
|
f"{'N/A':<{col_speedup}} {ft_m_bars_s:<{col_ft_m}.1f} {'N/A':<{col_ta_m}}"
|
|
)
|
|
|
|
results.append(row)
|
|
|
|
print()
|
|
if TALIB_AVAILABLE and results:
|
|
wins = sum(1 for row in results if row.get("outcome") == "ferro_ta_win")
|
|
total = len([row for row in results if "speedup" in row])
|
|
print(f"Summary: ferro_ta ahead outside the tie band on {wins}/{total} rows.")
|
|
print()
|
|
|
|
if json_path:
|
|
metadata = benchmark_metadata(
|
|
"benchmark_vs_talib",
|
|
extra={
|
|
"dataset": {
|
|
"generator": "synthetic_ohlcv",
|
|
"sizes": sizes,
|
|
"dtype": "float64",
|
|
"array_layout": "C-contiguous",
|
|
"seed": 42,
|
|
},
|
|
"methodology": {
|
|
"warmup_runs": N_WARMUP,
|
|
"measured_runs": N_RUNS,
|
|
"reported_metric": "median_ms",
|
|
"speedup_definition": "talib_median_ms / ferro_ta_median_ms",
|
|
"tie_band": f"{1.0 - TIE_EPSILON:.2f} to {1.0 + TIE_EPSILON:.2f}",
|
|
"input_layout_notes": (
|
|
"Benchmarks use contiguous float64 arrays. If your workload "
|
|
"passes non-contiguous arrays or other dtypes, benchmark that "
|
|
"separately because wrapper overhead can dominate."
|
|
),
|
|
"allocation_notes": (
|
|
"python_peak_allocation_bytes is a tracemalloc snapshot of "
|
|
"Python-tracked allocations only; it is not a full native RSS "
|
|
"or allocator profile."
|
|
),
|
|
},
|
|
"packages": package_versions("numpy", "ferro-ta", "TA-Lib"),
|
|
},
|
|
)
|
|
out = {
|
|
"schema_version": 2,
|
|
"command": " ".join(["python", *sys.argv]),
|
|
"n_warmup": N_WARMUP,
|
|
"n_runs": N_RUNS,
|
|
"sizes": sizes,
|
|
"talib_available": TALIB_AVAILABLE,
|
|
"runtime": metadata["runtime"],
|
|
"git": metadata["git"],
|
|
"metadata": metadata,
|
|
"summary": {
|
|
"total_rows": len(results),
|
|
"by_size": [_summary_for_size(results, size) for size in sizes],
|
|
},
|
|
"results": results,
|
|
}
|
|
if not TALIB_AVAILABLE:
|
|
out["note"] = "ferro_ta only; ta-lib not installed"
|
|
with open(json_path, "w", encoding="utf-8") as handle:
|
|
json.dump(out, handle, indent=2)
|
|
print(f"Results written to {json_path}")
|
|
|
|
return results
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description="ferro_ta vs TA-Lib speed comparison")
|
|
parser.add_argument("--json", default=None, help="Write results to JSON file")
|
|
parser.add_argument(
|
|
"--sizes",
|
|
type=int,
|
|
nargs="+",
|
|
default=DEFAULT_SIZES,
|
|
help="Bar counts to benchmark (default: 10000 100000 1000000)",
|
|
)
|
|
args = parser.parse_args()
|
|
run_comparison(args.sizes, args.json)
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|