chore: release v1.0.3
This commit is contained in:
+51
-6
@@ -1,14 +1,21 @@
|
||||
# ferro-ta Benchmark Suite
|
||||
|
||||
> **62 indicators × 6 libraries** — accuracy and speed verified on **100,000 bars** (LARGE dataset).
|
||||
> Reproducible speed and accuracy comparisons across 62 indicators and the
|
||||
> libraries available in your environment.
|
||||
|
||||
## Overview
|
||||
|
||||
The benchmark suite compares **ferro-ta** against five popular Python technical-analysis libraries on a common dataset and shared wrappers so timings are directly comparable.
|
||||
The benchmark suite compares **ferro-ta** against other Python
|
||||
technical-analysis libraries on a common dataset and shared wrappers so the
|
||||
results are easier to reproduce and critique.
|
||||
|
||||
It is not designed to prove that ferro-ta wins everywhere. It is designed to
|
||||
show where ferro-ta is faster, where it only ties, and where another library
|
||||
still wins.
|
||||
|
||||
| Library | Notes |
|
||||
|-----------|-------|
|
||||
| **TA-Lib** | C extension; gold standard for accuracy and speed |
|
||||
| **TA-Lib** | C extension; widely used comparison baseline |
|
||||
| **pandas-ta** | Pure Python; broad indicator set |
|
||||
| **ta** | Simple API; some indicators use O(n²) loops and are very slow |
|
||||
| **Tulipy** | C extension; truncated output (no leading NaN padding) |
|
||||
@@ -37,9 +44,23 @@ from benchmarks.data_generator import SMALL, MEDIUM, LARGE
|
||||
|
||||
- **Harness:** [pytest-benchmark](https://pytest-benchmark.readthedocs.io/) with `benchmark.pedantic(..., iterations=5, rounds=20, warmup_rounds=2)`.
|
||||
- **Reported metric:** **Median time per call** in **microseconds (µs)** — lower is better.
|
||||
- **Machine info:** Stored in `benchmarks/results.json` (`machine_info`, `commit_info`) for reproducibility.
|
||||
- **TA-Lib head-to-head JSON:** `benchmarks/bench_vs_talib.py` records per-run samples, variance stats, machine/runtime/build metadata, and Python-tracked peak allocation snapshots.
|
||||
- **Machine info:** Stored in the generated JSON artifacts for reproducibility.
|
||||
- **Libraries:** Only libraries present in the environment are benchmarked; missing ones are skipped.
|
||||
|
||||
## Current checked-in TA-Lib artifact
|
||||
|
||||
The checked-in `benchmarks/artifacts/latest/benchmark_vs_talib.json` artifact
|
||||
uses contiguous `float64` arrays at 10k and 100k bars on an Apple M3 Max,
|
||||
CPython 3.13.5, and Rust 1.91.1 with the default release profile
|
||||
(`lto = true`, `codegen-units = 1`).
|
||||
|
||||
- ferro-ta is ahead outside the tie band on 6 of 12 rows at 10k bars and 6 of 12 rows at 100k bars.
|
||||
- TA-Lib still wins in the current artifact on `STOCH` and `ADX`, and remains close on `EMA`, `RSI`, `ATR`, and `OBV` depending on size.
|
||||
- The public claim should therefore be read as "often faster on selected indicators," not "faster everywhere."
|
||||
- When publishing performance statements, point readers to the raw JSON artifact, not just the summary table.
|
||||
- The artifact now includes per-run samples, variance stats, and Python-tracked allocation snapshots for each compared indicator.
|
||||
|
||||
## Reproducible Perf Artifacts
|
||||
|
||||
Use the perf-contract runner when you want a compact set of machine-readable
|
||||
@@ -140,7 +161,7 @@ The speed table includes **all 62 indicators**. **Number** = median µs; **N/A**
|
||||
**Takeaways:**
|
||||
|
||||
- **`ta`** is 20–350× slower on ATR, CCI, ADX, MFI (O(n²) Python loops).
|
||||
- **ferro-ta** is typically 2–4× faster than **pandas-ta** across indicators.
|
||||
- **ferro-ta** is often materially faster than **pandas-ta** on the checked-in 100k-bar table.
|
||||
- **TA-Lib** and **Tulipy** (C extensions) are strong; ferro-ta is competitive and avoids native dependencies.
|
||||
|
||||
---
|
||||
@@ -160,9 +181,14 @@ uv run pytest benchmarks/test_speed.py --benchmark-only -k "test_large_dataset"
|
||||
# Regenerate the Speed Comparison markdown table from results.json
|
||||
uv run python benchmarks/benchmark_table.py
|
||||
|
||||
# TA-Lib head-to-head with machine-readable summary + git/runtime metadata
|
||||
# TA-Lib head-to-head with machine/runtime/build metadata, per-run samples,
|
||||
# variance stats, and Python-tracked allocation snapshots
|
||||
uv run python benchmarks/bench_vs_talib.py --sizes 10000 100000 --json benchmark_vs_talib.json
|
||||
|
||||
# Selected derivatives analytics comparison (BSM price, IV, Greeks, Black-76)
|
||||
# against built-in analytical references plus optional installed libraries
|
||||
uv run python benchmarks/bench_derivatives_compare.py --sizes 1000 10000 --json benchmark_derivatives_compare.json
|
||||
|
||||
# Optional regression check used in CI
|
||||
uv run python benchmarks/check_vs_talib_regression.py --input benchmark_vs_talib.json
|
||||
|
||||
@@ -184,6 +210,25 @@ uv run python benchmarks/run_perf_contract.py --output-dir benchmarks/artifacts/
|
||||
|
||||
Without `uv`: use `pytest` and `python` from the same environment where `ferro_ta` and optional libs (e.g. `talib`, `pandas_ta`, `ta`, `tulipy`, `finta`) are installed.
|
||||
|
||||
### Derivatives analytics
|
||||
|
||||
`benchmarks/bench_derivatives_compare.py` focuses on selected options-analytics
|
||||
paths rather than the full surface area:
|
||||
|
||||
- `BSM` call pricing
|
||||
- call implied-volatility recovery
|
||||
- call Greeks
|
||||
- `Black-76` call pricing
|
||||
|
||||
The script always includes two analytical baselines:
|
||||
|
||||
- `reference_numpy` — pure NumPy formulas with vectorized IV bisection
|
||||
- `reference_python_loop` — scalar `math`-based reference for sanity checking
|
||||
|
||||
If `py_vollib` is installed, it is added automatically as an extra baseline.
|
||||
The output JSON includes runtime/build metadata, per-run timing samples,
|
||||
variance stats, and Python-tracked peak allocation snapshots.
|
||||
|
||||
### WASM
|
||||
|
||||
From the `wasm/` directory:
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
+210
-112
@@ -1,37 +1,34 @@
|
||||
"""
|
||||
ferro_ta vs TA-Lib speed comparison.
|
||||
|
||||
Measures throughput (M bars/s) for both libraries on the same data and parameters,
|
||||
and reports speedup (talib_time / ferro_ta_time; > 1 means ferro_ta is faster).
|
||||
Measures throughput (M bars/s) for both libraries on the same synthetic data
|
||||
and parameters. The output is intentionally evidence-heavy:
|
||||
|
||||
Requirements:
|
||||
pip install ta-lib # or conda install ta-lib
|
||||
- median timings
|
||||
- per-run timing samples
|
||||
- variability stats
|
||||
- Python-tracked peak allocation snapshots
|
||||
- machine, runtime, and build metadata
|
||||
|
||||
Run:
|
||||
python benchmarks/bench_vs_talib.py
|
||||
python benchmarks/bench_vs_talib.py --json results.json
|
||||
python benchmarks/bench_vs_talib.py --sizes 10000 100000 # default: 10k, 100k, 1M
|
||||
|
||||
If ta-lib is not installed, the script still runs and reports ferro_ta timings only (no speedup).
|
||||
Methodology: same synthetic data, same parameters, median of 7 runs after warmup.
|
||||
Environment: document Python version and OS when publishing results.
|
||||
This is meant to support a narrow claim: ferro-ta is often faster on selected
|
||||
indicators, not universally faster.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from datetime import datetime, timezone
|
||||
import json
|
||||
import platform
|
||||
import subprocess
|
||||
import math
|
||||
import sys
|
||||
import time
|
||||
import tracemalloc
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
|
||||
try:
|
||||
import talib # noqa: F401
|
||||
|
||||
TALIB_AVAILABLE = True
|
||||
except ImportError:
|
||||
TALIB_AVAILABLE = False
|
||||
@@ -39,6 +36,11 @@ except ImportError:
|
||||
|
||||
import ferro_ta
|
||||
|
||||
try:
|
||||
from benchmarks.metadata import benchmark_metadata, package_versions
|
||||
except ModuleNotFoundError: # pragma: no cover - script execution fallback
|
||||
from metadata import benchmark_metadata, package_versions
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Configuration
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -46,100 +48,138 @@ import ferro_ta
|
||||
N_WARMUP = 1
|
||||
N_RUNS = 7
|
||||
DEFAULT_SIZES = [10_000, 100_000, 1_000_000]
|
||||
TIE_EPSILON = 0.05
|
||||
|
||||
_rng = np.random.default_rng(42)
|
||||
|
||||
|
||||
def _git_info() -> dict[str, Any]:
|
||||
"""Best-effort git metadata for benchmark reproducibility."""
|
||||
try:
|
||||
commit = subprocess.check_output(
|
||||
["git", "rev-parse", "HEAD"], text=True, stderr=subprocess.DEVNULL
|
||||
).strip()
|
||||
except Exception:
|
||||
commit = None
|
||||
|
||||
try:
|
||||
dirty = bool(
|
||||
subprocess.check_output(
|
||||
["git", "status", "--porcelain"],
|
||||
text=True,
|
||||
stderr=subprocess.DEVNULL,
|
||||
).strip()
|
||||
)
|
||||
except Exception:
|
||||
dirty = None
|
||||
|
||||
return {"commit": commit, "dirty": dirty}
|
||||
def _median(values: list[float]) -> float:
|
||||
ordered = sorted(values)
|
||||
mid = len(ordered) // 2
|
||||
if len(ordered) % 2:
|
||||
return ordered[mid]
|
||||
return (ordered[mid - 1] + ordered[mid]) / 2.0
|
||||
|
||||
|
||||
def _runtime_info() -> dict[str, Any]:
|
||||
def _summary_stats(samples_ms: list[float]) -> dict[str, float]:
|
||||
if not samples_ms:
|
||||
return {
|
||||
"median_ms": 0.0,
|
||||
"mean_ms": 0.0,
|
||||
"min_ms": 0.0,
|
||||
"max_ms": 0.0,
|
||||
"stddev_ms": 0.0,
|
||||
"cv_pct": 0.0,
|
||||
}
|
||||
|
||||
mean_ms = sum(samples_ms) / len(samples_ms)
|
||||
variance = (
|
||||
sum((sample - mean_ms) ** 2 for sample in samples_ms) / (len(samples_ms) - 1)
|
||||
if len(samples_ms) > 1
|
||||
else 0.0
|
||||
)
|
||||
stddev_ms = math.sqrt(variance)
|
||||
cv_pct = (stddev_ms / mean_ms * 100.0) if mean_ms else 0.0
|
||||
return {
|
||||
"generated_at_utc": datetime.now(timezone.utc).isoformat(),
|
||||
"python_version": sys.version.split()[0],
|
||||
"platform": platform.platform(),
|
||||
"machine": platform.machine(),
|
||||
"median_ms": round(_median(samples_ms), 4),
|
||||
"mean_ms": round(mean_ms, 4),
|
||||
"min_ms": round(min(samples_ms), 4),
|
||||
"max_ms": round(max(samples_ms), 4),
|
||||
"stddev_ms": round(stddev_ms, 4),
|
||||
"cv_pct": round(cv_pct, 3),
|
||||
}
|
||||
|
||||
|
||||
def _outcome(speedup: float) -> str:
|
||||
if speedup > 1.0 + TIE_EPSILON:
|
||||
return "ferro_ta_win"
|
||||
if speedup < 1.0 - TIE_EPSILON:
|
||||
return "talib_win"
|
||||
return "tie"
|
||||
|
||||
|
||||
def _summary_for_size(results: list[dict[str, Any]], size: int) -> dict[str, Any]:
|
||||
rows = [r for r in results if r.get("size") == size and "speedup" in r]
|
||||
rows = [row for row in results if row.get("size") == size and "speedup" in row]
|
||||
if not rows:
|
||||
return {"size": size, "rows": 0}
|
||||
|
||||
speedups = [float(r["speedup"]) for r in rows]
|
||||
wins = sum(1 for s in speedups if s > 1.0)
|
||||
speedups_sorted = sorted(speedups)
|
||||
mid = len(speedups_sorted) // 2
|
||||
if len(speedups_sorted) % 2:
|
||||
median = speedups_sorted[mid]
|
||||
else:
|
||||
median = (speedups_sorted[mid - 1] + speedups_sorted[mid]) / 2.0
|
||||
|
||||
speedups = [float(row["speedup"]) for row in rows]
|
||||
wins = sum(1 for row in rows if row.get("outcome") == "ferro_ta_win")
|
||||
ties = sum(1 for row in rows if row.get("outcome") == "tie")
|
||||
losses = sum(1 for row in rows if row.get("outcome") == "talib_win")
|
||||
return {
|
||||
"size": size,
|
||||
"rows": len(rows),
|
||||
"wins": wins,
|
||||
"win_rate": wins / len(rows),
|
||||
"median_speedup": round(median, 4),
|
||||
"ties": ties,
|
||||
"losses": losses,
|
||||
"win_rate": round(wins / len(rows), 4),
|
||||
"non_loss_rate": round((wins + ties) / len(rows), 4),
|
||||
"median_speedup": round(_median(speedups), 4),
|
||||
"min_speedup": round(min(speedups), 4),
|
||||
"max_speedup": round(max(speedups), 4),
|
||||
"talib_wins_or_ties": [
|
||||
row["indicator"]
|
||||
for row in rows
|
||||
if row.get("outcome") in {"talib_win", "tie"}
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def _synthetic_ohlcv(n: int) -> tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray]:
|
||||
# Generate OHLCV so that ta crate DataItem constraints hold: low >= 0, volume >= 0,
|
||||
# and low <= open, close <= high, high >= open (see ta DataItemBuilder::build).
|
||||
def _synthetic_ohlcv(
|
||||
n: int,
|
||||
) -> tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray]:
|
||||
# Generate OHLCV so that ta crate DataItem constraints hold: low >= 0,
|
||||
# volume >= 0, and low <= open, close <= high, high >= open.
|
||||
close = 100.0 + np.cumsum(_rng.standard_normal(n) * 0.5)
|
||||
open_ = close + _rng.standard_normal(n) * 0.2
|
||||
high = np.maximum(open_, close) + np.abs(_rng.standard_normal(n) * 0.3)
|
||||
low = np.minimum(open_, close) - np.abs(_rng.standard_normal(n) * 0.3)
|
||||
# Enforce high >= low and low >= 0 (ta requires non-negative prices)
|
||||
high = np.maximum(high, low)
|
||||
low = np.maximum(low, 0.0)
|
||||
high = np.maximum(high, low) # again after clamping low
|
||||
high = np.maximum(high, low)
|
||||
open_ = np.clip(open_, low, high)
|
||||
close = np.clip(close, low, high)
|
||||
volume = np.abs(_rng.standard_normal(n) * 1_000_000) + 500_000
|
||||
return open_, high, low, close, volume
|
||||
|
||||
|
||||
def _median_time_ms(fn, *args, **kwargs) -> float:
|
||||
def _timed_runs_ms(fn, *args, **kwargs) -> list[float]:
|
||||
for _ in range(N_WARMUP):
|
||||
fn(*args, **kwargs)
|
||||
times = []
|
||||
|
||||
samples_ms: list[float] = []
|
||||
for _ in range(N_RUNS):
|
||||
t0 = time.perf_counter()
|
||||
fn(*args, **kwargs)
|
||||
times.append((time.perf_counter() - t0) * 1000)
|
||||
times.sort()
|
||||
return times[len(times) // 2]
|
||||
samples_ms.append((time.perf_counter() - t0) * 1000.0)
|
||||
return samples_ms
|
||||
|
||||
|
||||
def _python_peak_bytes(fn, *args, **kwargs) -> int | None:
|
||||
try:
|
||||
tracemalloc.start()
|
||||
tracemalloc.reset_peak()
|
||||
fn(*args, **kwargs)
|
||||
_, peak = tracemalloc.get_traced_memory()
|
||||
return int(peak)
|
||||
except Exception:
|
||||
return None
|
||||
finally:
|
||||
tracemalloc.stop()
|
||||
|
||||
|
||||
def _throughput_m_bars_s(size: int, median_ms: float) -> float:
|
||||
if median_ms <= 0:
|
||||
return 0.0
|
||||
return (size / 1e6) / (median_ms / 1000.0)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Benchmarked callables
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
# Each entry: (label, ferro_ta_callable, talib_callable, needs_ohlcv)
|
||||
# ferro_ta_callable / talib_callable receive (open_, high, low, close, volume) and size;
|
||||
# they return (args, ft_kwargs, ta_kwargs) or we use a simpler convention:
|
||||
# we pass (o, h, l, c, v) and size; each runner knows how to slice and call.
|
||||
def _run_ft_sma(o, h, l, c, v, n):
|
||||
return ferro_ta.SMA(c[:n], timeperiod=14)
|
||||
|
||||
@@ -236,7 +276,6 @@ def _run_ta_wma(o, h, l, c, v, n):
|
||||
return talib.WMA(c[:n], timeperiod=14)
|
||||
|
||||
|
||||
# List of (indicator_name, ft_runner, ta_runner); skip 1M for very slow indicators if needed
|
||||
COMPARISON_CASES = [
|
||||
("SMA", _run_ft_sma, _run_ta_sma),
|
||||
("EMA", _run_ft_ema, _run_ta_ema),
|
||||
@@ -252,14 +291,14 @@ COMPARISON_CASES = [
|
||||
("WMA", _run_ft_wma, _run_ta_wma),
|
||||
]
|
||||
|
||||
# For STOCH/ADX and other heavier indicators, optionally skip 1M to keep runtime reasonable
|
||||
SKIP_1M_FOR = {"STOCH", "ADX"}
|
||||
|
||||
|
||||
def run_comparison(sizes: list[int], json_path: str | None) -> list[dict[str, Any]]:
|
||||
max_size = max(sizes)
|
||||
open_, high, low, close, volume = _synthetic_ohlcv(max_size)
|
||||
results = []
|
||||
results: list[dict[str, Any]] = []
|
||||
|
||||
col_label = 10
|
||||
col_size = 10
|
||||
col_ft_ms = 12
|
||||
@@ -269,12 +308,13 @@ def run_comparison(sizes: list[int], json_path: str | None) -> list[dict[str, An
|
||||
col_ta_m = 12
|
||||
|
||||
if not TALIB_AVAILABLE:
|
||||
print("Note: ta-lib not installed — reporting ferro_ta timings only (no speedup).")
|
||||
print("Note: ta-lib not installed. Reporting ferro_ta timings only.")
|
||||
print("Install with: pip install ta-lib (or conda install ta-lib) for comparison.\n")
|
||||
|
||||
print(f"\nferro_ta vs TA-Lib — median of {N_RUNS} runs (after {N_WARMUP} warmup)")
|
||||
print(f"\nferro_ta vs TA-Lib — median of {N_RUNS} measured runs after {N_WARMUP} warmup")
|
||||
print(f"Sizes: {sizes}")
|
||||
print()
|
||||
|
||||
header = (
|
||||
f"{'Indicator':<{col_label}} {'Size':<{col_size}} "
|
||||
f"{'ferro_ta(ms)':<{col_ft_ms}} {'TA-Lib(ms)':<{col_ta_ms}} "
|
||||
@@ -287,82 +327,140 @@ def run_comparison(sizes: list[int], json_path: str | None) -> list[dict[str, An
|
||||
for size in sizes:
|
||||
if size == 1_000_000 and name in SKIP_1M_FOR:
|
||||
continue
|
||||
ms_ft = _median_time_ms(ft_run, open_, high, low, close, volume, size)
|
||||
|
||||
ft_samples_ms = _timed_runs_ms(ft_run, open_, high, low, close, volume, size)
|
||||
ft_stats = _summary_stats(ft_samples_ms)
|
||||
ft_median_ms = float(ft_stats["median_ms"])
|
||||
ft_m_bars_s = _throughput_m_bars_s(size, ft_median_ms)
|
||||
ft_peak_bytes = _python_peak_bytes(ft_run, open_, high, low, close, volume, size)
|
||||
|
||||
row: dict[str, Any] = {
|
||||
"indicator": name,
|
||||
"size": size,
|
||||
"input_layout": {
|
||||
"dtype": "float64",
|
||||
"contiguous": True,
|
||||
},
|
||||
"ferro_ta_ms": round(ft_median_ms, 4),
|
||||
"ferro_ta_m_bars_s": round(ft_m_bars_s, 2),
|
||||
"ferro_ta_runs_ms": [round(sample, 4) for sample in ft_samples_ms],
|
||||
"ferro_ta_stats": ft_stats,
|
||||
"python_peak_allocation_bytes": {
|
||||
"ferro_ta": ft_peak_bytes,
|
||||
},
|
||||
}
|
||||
|
||||
if TALIB_AVAILABLE:
|
||||
ms_ta = _median_time_ms(ta_run, open_, high, low, close, volume, size)
|
||||
speedup = ms_ta / ms_ft if ms_ft > 0 else float("inf")
|
||||
m_bars_ft = (size / 1e6) / (ms_ft / 1000) if ms_ft > 0 else 0
|
||||
m_bars_ta = (size / 1e6) / (ms_ta / 1000) if ms_ta > 0 else 0
|
||||
ta_samples_ms = _timed_runs_ms(ta_run, open_, high, low, close, volume, size)
|
||||
ta_stats = _summary_stats(ta_samples_ms)
|
||||
ta_median_ms = float(ta_stats["median_ms"])
|
||||
ta_m_bars_s = _throughput_m_bars_s(size, ta_median_ms)
|
||||
speedup = ta_median_ms / ft_median_ms if ft_median_ms > 0 else float("inf")
|
||||
outcome = _outcome(speedup)
|
||||
ta_peak_bytes = _python_peak_bytes(
|
||||
ta_run, open_, high, low, close, volume, size
|
||||
)
|
||||
|
||||
print(
|
||||
f"{name:<{col_label}} {size:<{col_size}} "
|
||||
f"{ms_ft:<{col_ft_ms}.3f} {ms_ta:<{col_ta_ms}.3f} "
|
||||
f"{speedup:<{col_speedup}.2f}x {m_bars_ft:<{col_ft_m}.1f} {m_bars_ta:<{col_ta_m}.1f}"
|
||||
f"{ft_median_ms:<{col_ft_ms}.3f} {ta_median_ms:<{col_ta_ms}.3f} "
|
||||
f"{speedup:<{col_speedup}.2f}x {ft_m_bars_s:<{col_ft_m}.1f} {ta_m_bars_s:<{col_ta_m}.1f}"
|
||||
)
|
||||
row = {
|
||||
"indicator": name,
|
||||
"size": size,
|
||||
"ferro_ta_ms": round(ms_ft, 4),
|
||||
"talib_ms": round(ms_ta, 4),
|
||||
"speedup": round(speedup, 4),
|
||||
"ferro_ta_m_bars_s": round(m_bars_ft, 2),
|
||||
"talib_m_bars_s": round(m_bars_ta, 2),
|
||||
}
|
||||
|
||||
row.update(
|
||||
{
|
||||
"talib_ms": round(ta_median_ms, 4),
|
||||
"talib_m_bars_s": round(ta_m_bars_s, 2),
|
||||
"talib_runs_ms": [round(sample, 4) for sample in ta_samples_ms],
|
||||
"talib_stats": ta_stats,
|
||||
"speedup": round(speedup, 4),
|
||||
"outcome": outcome,
|
||||
}
|
||||
)
|
||||
row["python_peak_allocation_bytes"]["talib"] = ta_peak_bytes
|
||||
else:
|
||||
m_bars_ft = (size / 1e6) / (ms_ft / 1000) if ms_ft > 0 else 0
|
||||
print(
|
||||
f"{name:<{col_label}} {size:<{col_size}} "
|
||||
f"{ms_ft:<{col_ft_ms}.3f} {'N/A':<{col_ta_ms}} "
|
||||
f"{'N/A':<{col_speedup}} {m_bars_ft:<{col_ft_m}.1f} {'N/A':<{col_ta_m}}"
|
||||
f"{ft_median_ms:<{col_ft_ms}.3f} {'N/A':<{col_ta_ms}} "
|
||||
f"{'N/A':<{col_speedup}} {ft_m_bars_s:<{col_ft_m}.1f} {'N/A':<{col_ta_m}}"
|
||||
)
|
||||
row = {
|
||||
"indicator": name,
|
||||
"size": size,
|
||||
"ferro_ta_ms": round(ms_ft, 4),
|
||||
"ferro_ta_m_bars_s": round(m_bars_ft, 2),
|
||||
}
|
||||
|
||||
results.append(row)
|
||||
|
||||
print()
|
||||
if TALIB_AVAILABLE and results:
|
||||
wins = sum(1 for r in results if r.get("speedup", 0) > 1)
|
||||
total = len(results)
|
||||
print(f"Summary: ferro_ta faster on {wins}/{total} rows (speedup > 1).")
|
||||
wins = sum(1 for row in results if row.get("outcome") == "ferro_ta_win")
|
||||
total = len([row for row in results if "speedup" in row])
|
||||
print(f"Summary: ferro_ta ahead outside the tie band on {wins}/{total} rows.")
|
||||
print()
|
||||
|
||||
if json_path:
|
||||
metadata = benchmark_metadata(
|
||||
"benchmark_vs_talib",
|
||||
extra={
|
||||
"dataset": {
|
||||
"generator": "synthetic_ohlcv",
|
||||
"sizes": sizes,
|
||||
"dtype": "float64",
|
||||
"array_layout": "C-contiguous",
|
||||
"seed": 42,
|
||||
},
|
||||
"methodology": {
|
||||
"warmup_runs": N_WARMUP,
|
||||
"measured_runs": N_RUNS,
|
||||
"reported_metric": "median_ms",
|
||||
"speedup_definition": "talib_median_ms / ferro_ta_median_ms",
|
||||
"tie_band": f"{1.0 - TIE_EPSILON:.2f} to {1.0 + TIE_EPSILON:.2f}",
|
||||
"input_layout_notes": (
|
||||
"Benchmarks use contiguous float64 arrays. If your workload "
|
||||
"passes non-contiguous arrays or other dtypes, benchmark that "
|
||||
"separately because wrapper overhead can dominate."
|
||||
),
|
||||
"allocation_notes": (
|
||||
"python_peak_allocation_bytes is a tracemalloc snapshot of "
|
||||
"Python-tracked allocations only; it is not a full native RSS "
|
||||
"or allocator profile."
|
||||
),
|
||||
},
|
||||
"packages": package_versions("numpy", "ferro-ta", "TA-Lib"),
|
||||
},
|
||||
)
|
||||
out = {
|
||||
"schema_version": 1,
|
||||
"command": "python benchmarks/bench_vs_talib.py",
|
||||
"schema_version": 2,
|
||||
"command": " ".join(["python", *sys.argv]),
|
||||
"n_warmup": N_WARMUP,
|
||||
"n_runs": N_RUNS,
|
||||
"sizes": sizes,
|
||||
"talib_available": TALIB_AVAILABLE,
|
||||
"runtime": _runtime_info(),
|
||||
"git": _git_info(),
|
||||
"runtime": metadata["runtime"],
|
||||
"git": metadata["git"],
|
||||
"metadata": metadata,
|
||||
"summary": {
|
||||
"total_rows": len(results),
|
||||
"by_size": [_summary_for_size(results, s) for s in sizes],
|
||||
"by_size": [_summary_for_size(results, size) for size in sizes],
|
||||
},
|
||||
"results": results,
|
||||
}
|
||||
if not TALIB_AVAILABLE:
|
||||
out["note"] = "ferro_ta only — ta-lib not installed"
|
||||
with open(json_path, "w") as f:
|
||||
json.dump(out, f, indent=2)
|
||||
out["note"] = "ferro_ta only; ta-lib not installed"
|
||||
with open(json_path, "w", encoding="utf-8") as handle:
|
||||
json.dump(out, handle, indent=2)
|
||||
print(f"Results written to {json_path}")
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser(description="ferro_ta vs TA-Lib speed comparison")
|
||||
ap.add_argument("--json", default=None, help="Write results to JSON file")
|
||||
ap.add_argument(
|
||||
parser = argparse.ArgumentParser(description="ferro_ta vs TA-Lib speed comparison")
|
||||
parser.add_argument("--json", default=None, help="Write results to JSON file")
|
||||
parser.add_argument(
|
||||
"--sizes",
|
||||
type=int,
|
||||
nargs="+",
|
||||
default=DEFAULT_SIZES,
|
||||
help="Bar counts to benchmark (default: 10000 100000 1000000)",
|
||||
)
|
||||
args = ap.parse_args()
|
||||
args = parser.parse_args()
|
||||
run_comparison(args.sizes, args.json)
|
||||
return 0
|
||||
|
||||
|
||||
+135
-22
@@ -1,56 +1,167 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import os
|
||||
import platform
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
from importlib import metadata as importlib_metadata
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
def git_info() -> dict[str, Any]:
|
||||
"""Best-effort git metadata for reproducible benchmark artifacts."""
|
||||
try:
|
||||
import tomllib
|
||||
except ImportError: # pragma: no cover
|
||||
try:
|
||||
commit = subprocess.check_output(
|
||||
["git", "rev-parse", "HEAD"], text=True, stderr=subprocess.DEVNULL
|
||||
).strip()
|
||||
except Exception:
|
||||
commit = None
|
||||
import tomli as tomllib # type: ignore[no-redef]
|
||||
except ImportError: # pragma: no cover
|
||||
tomllib = None # type: ignore[assignment]
|
||||
|
||||
try:
|
||||
dirty = bool(
|
||||
subprocess.check_output(
|
||||
["git", "status", "--porcelain"],
|
||||
text=True,
|
||||
stderr=subprocess.DEVNULL,
|
||||
).strip()
|
||||
)
|
||||
except Exception:
|
||||
dirty = None
|
||||
|
||||
_ROOT = Path(__file__).resolve().parent.parent
|
||||
|
||||
|
||||
def _run_cmd(command: list[str]) -> str | None:
|
||||
try:
|
||||
branch = subprocess.check_output(
|
||||
["git", "rev-parse", "--abbrev-ref", "HEAD"],
|
||||
return subprocess.check_output(
|
||||
command,
|
||||
text=True,
|
||||
stderr=subprocess.DEVNULL,
|
||||
).strip()
|
||||
except Exception:
|
||||
branch = None
|
||||
return None
|
||||
|
||||
return {"commit": commit, "dirty": dirty, "branch": branch}
|
||||
|
||||
def _read_toml(path: Path) -> dict[str, Any] | None:
|
||||
if tomllib is None or not path.exists():
|
||||
return None
|
||||
try:
|
||||
with path.open("rb") as handle:
|
||||
return tomllib.load(handle)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _cpu_model() -> str | None:
|
||||
if sys.platform == "darwin":
|
||||
return (
|
||||
_run_cmd(["sysctl", "-n", "machdep.cpu.brand_string"])
|
||||
or _run_cmd(["sysctl", "-n", "hw.model"])
|
||||
or platform.processor()
|
||||
or None
|
||||
)
|
||||
if sys.platform.startswith("linux"):
|
||||
cpuinfo = Path("/proc/cpuinfo")
|
||||
if cpuinfo.exists():
|
||||
text = cpuinfo.read_text(encoding="utf-8", errors="ignore")
|
||||
for pattern in (r"model name\s+:\s+(.+)", r"Hardware\s+:\s+(.+)"):
|
||||
match = re.search(pattern, text)
|
||||
if match:
|
||||
return match.group(1).strip()
|
||||
return platform.processor() or None
|
||||
if sys.platform.startswith("win"):
|
||||
return os.environ.get("PROCESSOR_IDENTIFIER") or platform.processor() or None
|
||||
return platform.processor() or None
|
||||
|
||||
|
||||
def _total_memory_bytes() -> int | None:
|
||||
if sys.platform == "darwin":
|
||||
raw = _run_cmd(["sysctl", "-n", "hw.memsize"])
|
||||
return int(raw) if raw and raw.isdigit() else None
|
||||
|
||||
if sys.platform.startswith("linux"):
|
||||
meminfo = Path("/proc/meminfo")
|
||||
if meminfo.exists():
|
||||
text = meminfo.read_text(encoding="utf-8", errors="ignore")
|
||||
match = re.search(r"MemTotal:\s+(\d+)\s+kB", text)
|
||||
if match:
|
||||
return int(match.group(1)) * 1024
|
||||
return None
|
||||
|
||||
if sys.platform.startswith("win"): # pragma: no cover
|
||||
try:
|
||||
import ctypes
|
||||
|
||||
class MEMORYSTATUSEX(ctypes.Structure):
|
||||
_fields_ = [
|
||||
("dwLength", ctypes.c_ulong),
|
||||
("dwMemoryLoad", ctypes.c_ulong),
|
||||
("ullTotalPhys", ctypes.c_ulonglong),
|
||||
("ullAvailPhys", ctypes.c_ulonglong),
|
||||
("ullTotalPageFile", ctypes.c_ulonglong),
|
||||
("ullAvailPageFile", ctypes.c_ulonglong),
|
||||
("ullTotalVirtual", ctypes.c_ulonglong),
|
||||
("ullAvailVirtual", ctypes.c_ulonglong),
|
||||
("ullAvailExtendedVirtual", ctypes.c_ulonglong),
|
||||
]
|
||||
|
||||
status = MEMORYSTATUSEX()
|
||||
status.dwLength = ctypes.sizeof(MEMORYSTATUSEX)
|
||||
ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(status))
|
||||
return int(status.ullTotalPhys)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def _cargo_release_profile() -> dict[str, Any] | None:
|
||||
cargo_toml = _read_toml(_ROOT / "Cargo.toml")
|
||||
if not cargo_toml:
|
||||
return None
|
||||
profile = cargo_toml.get("profile", {}).get("release")
|
||||
return profile if isinstance(profile, dict) else None
|
||||
|
||||
|
||||
def git_info() -> dict[str, Any]:
|
||||
"""Best-effort git metadata for reproducible benchmark artifacts."""
|
||||
return {
|
||||
"commit": _run_cmd(["git", "rev-parse", "HEAD"]),
|
||||
"dirty": bool(_run_cmd(["git", "status", "--porcelain"]) or ""),
|
||||
"branch": _run_cmd(["git", "rev-parse", "--abbrev-ref", "HEAD"]),
|
||||
}
|
||||
|
||||
|
||||
def runtime_info() -> dict[str, Any]:
|
||||
return {
|
||||
"generated_at_utc": datetime.now(timezone.utc).isoformat(),
|
||||
"python_version": sys.version.split()[0],
|
||||
"python_implementation": platform.python_implementation(),
|
||||
"python_executable": sys.executable,
|
||||
"platform": platform.platform(),
|
||||
"system": platform.system(),
|
||||
"release": platform.release(),
|
||||
"machine": platform.machine(),
|
||||
"processor": platform.processor() or None,
|
||||
"cpu_model": _cpu_model(),
|
||||
"cpu_count_logical": os.cpu_count(),
|
||||
"total_memory_bytes": _total_memory_bytes(),
|
||||
}
|
||||
|
||||
|
||||
def build_info() -> dict[str, Any]:
|
||||
return {
|
||||
"rustc": _run_cmd(["rustc", "-Vv"]),
|
||||
"cargo": _run_cmd(["cargo", "-VV"]) or _run_cmd(["cargo", "-V"]),
|
||||
"cargo_release_profile": _cargo_release_profile(),
|
||||
"rustflags": os.environ.get("RUSTFLAGS"),
|
||||
"cargo_build_rustflags": os.environ.get("CARGO_BUILD_RUSTFLAGS"),
|
||||
"maturin_flags": os.environ.get("MATURIN_EXTRA_ARGS"),
|
||||
}
|
||||
|
||||
|
||||
def package_versions(*names: str) -> dict[str, str | None]:
|
||||
versions: dict[str, str | None] = {}
|
||||
for name in names:
|
||||
try:
|
||||
versions[name] = importlib_metadata.version(name)
|
||||
except importlib_metadata.PackageNotFoundError:
|
||||
versions[name] = None
|
||||
return versions
|
||||
|
||||
|
||||
def file_info(path: str | Path) -> dict[str, Any]:
|
||||
file_path = Path(path)
|
||||
data = file_path.read_bytes()
|
||||
@@ -71,6 +182,8 @@ def benchmark_metadata(
|
||||
"suite": suite,
|
||||
"runtime": runtime_info(),
|
||||
"git": git_info(),
|
||||
"build": build_info(),
|
||||
"packages": package_versions("numpy", "ferro-ta"),
|
||||
}
|
||||
if fixtures:
|
||||
metadata["fixtures"] = [file_info(path) for path in fixtures]
|
||||
|
||||
Reference in New Issue
Block a user