feat: refresh benchmark coverage and harden CI tooling
Refresh the benchmark and performance surface across the repo. This updates the benchmark wrappers and helper scripts, regenerates the checked-in benchmark and perf-contract artifacts, and folds in the related roadmap, compatibility, and example notebook changes that belong with this performance-focused pass. Harden the Python CI and local pre-push flow so the same checks pass reliably in both places. The workflow and pre-push script now use module-safe uv typecheck invocations, the Python test environment installs the optional MCP dependency needed by the MCP server tests, and one-off root benchmark outputs are ignored to keep the repo clean. Align local tooling with the current project configuration by updating the Ruff pre-commit hook, tightening the API typing and MCP server helpers, and refreshing the lockfile to pick up the audited PyJWT fix while preserving the rest of the staged source changes.
This commit is contained in:
@@ -68,4 +68,4 @@
|
||||
"speedup_vs_separate": 2.2811
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1159,4 +1159,4 @@
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1379,4 +1379,4 @@
|
||||
"outcome": "ferro_ta_win"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
@@ -160,4 +160,4 @@
|
||||
"elapsed_ms": 0.0026
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
@@ -59,4 +59,4 @@
|
||||
"sha256": "f31fd871990c44e24a2259d618ae40a52866d20b95aa6047af3d38b9371c2ab7"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -93,4 +93,4 @@
|
||||
"share_of_suite_pct": 0.09
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
@@ -282,4 +282,4 @@
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -70,4 +70,4 @@
|
||||
"stream_over_batch_ratio": 121.7603
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
@@ -48,13 +48,17 @@ def run_batch_benchmark(
|
||||
"SMA",
|
||||
lambda: ferro_ta.batch.batch_sma(close2d, timeperiod=14, parallel=True),
|
||||
lambda: ferro_ta.batch.batch_sma(close2d, timeperiod=14, parallel=False),
|
||||
lambda: [ferro_ta.SMA(close2d[:, j], timeperiod=14) for j in range(n_series)],
|
||||
lambda: [
|
||||
ferro_ta.SMA(close2d[:, j], timeperiod=14) for j in range(n_series)
|
||||
],
|
||||
),
|
||||
(
|
||||
"RSI",
|
||||
lambda: ferro_ta.batch.batch_rsi(close2d, timeperiod=14, parallel=True),
|
||||
lambda: ferro_ta.batch.batch_rsi(close2d, timeperiod=14, parallel=False),
|
||||
lambda: [ferro_ta.RSI(close2d[:, j], timeperiod=14) for j in range(n_series)],
|
||||
lambda: [
|
||||
ferro_ta.RSI(close2d[:, j], timeperiod=14) for j in range(n_series)
|
||||
],
|
||||
),
|
||||
(
|
||||
"ATR",
|
||||
@@ -201,7 +205,9 @@ def main() -> int:
|
||||
if payload["grouped_results"]:
|
||||
print("\nGrouped Multi-Indicator Calls")
|
||||
print("-" * 64)
|
||||
print(f"{'Case':<18} {'Grouped (ms)':>14} {'Separate (ms)':>16} {'Speedup':>12}")
|
||||
print(
|
||||
f"{'Case':<18} {'Grouped (ms)':>14} {'Separate (ms)':>16} {'Speedup':>12}"
|
||||
)
|
||||
print("-" * 64)
|
||||
for row in payload["grouped_results"]:
|
||||
print(
|
||||
|
||||
@@ -26,9 +26,10 @@ import math
|
||||
import sys
|
||||
import time
|
||||
import tracemalloc
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
|
||||
|
||||
@@ -70,9 +70,7 @@ def run_simd_benchmark(
|
||||
for label, args in variants
|
||||
}
|
||||
|
||||
portable_rows = {
|
||||
row["name"]: row for row in reports["portable_release"]["results"]
|
||||
}
|
||||
portable_rows = {row["name"]: row for row in reports["portable_release"]["results"]}
|
||||
simd_rows = {row["name"]: row for row in reports["simd_release"]["results"]}
|
||||
|
||||
comparison: list[dict[str, Any]] = []
|
||||
@@ -136,9 +134,7 @@ def main() -> int:
|
||||
window=args.window,
|
||||
)
|
||||
|
||||
print(
|
||||
f"{'Case':<20} {'Portable (ms)':>14} {'SIMD (ms)':>12} {'SIMD speedup':>14}"
|
||||
)
|
||||
print(f"{'Case':<20} {'Portable (ms)':>14} {'SIMD (ms)':>12} {'SIMD speedup':>14}")
|
||||
print("-" * 64)
|
||||
for row in payload["results"]:
|
||||
print(
|
||||
|
||||
@@ -57,7 +57,9 @@ def _stream_hlcv(
|
||||
) -> float:
|
||||
streamer = factory()
|
||||
last = np.nan
|
||||
for high_value, low_value, close_value, volume_value in zip(high, low, close, volume):
|
||||
for high_value, low_value, close_value, volume_value in zip(
|
||||
high, low, close, volume
|
||||
):
|
||||
last = streamer.update(
|
||||
float(high_value),
|
||||
float(low_value),
|
||||
@@ -154,7 +156,9 @@ def run_streaming_benchmark(
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Benchmark streaming indicator execution.")
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Benchmark streaming indicator execution."
|
||||
)
|
||||
parser.add_argument("--bars", type=int, default=100_000)
|
||||
parser.add_argument("--seed", type=int, default=2026)
|
||||
parser.add_argument("--json", dest="json_path")
|
||||
|
||||
@@ -309,9 +309,13 @@ def run_comparison(sizes: list[int], json_path: str | None) -> list[dict[str, An
|
||||
|
||||
if not TALIB_AVAILABLE:
|
||||
print("Note: ta-lib not installed. Reporting ferro_ta timings only.")
|
||||
print("Install with: pip install ta-lib (or conda install ta-lib) for comparison.\n")
|
||||
print(
|
||||
"Install with: pip install ta-lib (or conda install ta-lib) for comparison.\n"
|
||||
)
|
||||
|
||||
print(f"\nferro_ta vs TA-Lib — median of {N_RUNS} measured runs after {N_WARMUP} warmup")
|
||||
print(
|
||||
f"\nferro_ta vs TA-Lib — median of {N_RUNS} measured runs after {N_WARMUP} warmup"
|
||||
)
|
||||
print(f"Sizes: {sizes}")
|
||||
print()
|
||||
|
||||
@@ -328,11 +332,15 @@ def run_comparison(sizes: list[int], json_path: str | None) -> list[dict[str, An
|
||||
if size == 1_000_000 and name in SKIP_1M_FOR:
|
||||
continue
|
||||
|
||||
ft_samples_ms = _timed_runs_ms(ft_run, open_, high, low, close, volume, size)
|
||||
ft_samples_ms = _timed_runs_ms(
|
||||
ft_run, open_, high, low, close, volume, size
|
||||
)
|
||||
ft_stats = _summary_stats(ft_samples_ms)
|
||||
ft_median_ms = float(ft_stats["median_ms"])
|
||||
ft_m_bars_s = _throughput_m_bars_s(size, ft_median_ms)
|
||||
ft_peak_bytes = _python_peak_bytes(ft_run, open_, high, low, close, volume, size)
|
||||
ft_peak_bytes = _python_peak_bytes(
|
||||
ft_run, open_, high, low, close, volume, size
|
||||
)
|
||||
|
||||
row: dict[str, Any] = {
|
||||
"indicator": name,
|
||||
@@ -351,11 +359,15 @@ def run_comparison(sizes: list[int], json_path: str | None) -> list[dict[str, An
|
||||
}
|
||||
|
||||
if TALIB_AVAILABLE:
|
||||
ta_samples_ms = _timed_runs_ms(ta_run, open_, high, low, close, volume, size)
|
||||
ta_samples_ms = _timed_runs_ms(
|
||||
ta_run, open_, high, low, close, volume, size
|
||||
)
|
||||
ta_stats = _summary_stats(ta_samples_ms)
|
||||
ta_median_ms = float(ta_stats["median_ms"])
|
||||
ta_m_bars_s = _throughput_m_bars_s(size, ta_median_ms)
|
||||
speedup = ta_median_ms / ft_median_ms if ft_median_ms > 0 else float("inf")
|
||||
speedup = (
|
||||
ta_median_ms / ft_median_ms if ft_median_ms > 0 else float("inf")
|
||||
)
|
||||
outcome = _outcome(speedup)
|
||||
ta_peak_bytes = _python_peak_bytes(
|
||||
ta_run, open_, high, low, close, volume, size
|
||||
|
||||
@@ -9,7 +9,9 @@ Reads results.json and prints a markdown table: all indicators × all libraries.
|
||||
Unsupported (indicator, library) pairs show N/A. Supported pairs missing benchmark
|
||||
data show ERR (indicating the benchmark run was incomplete or failed).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
@@ -21,9 +23,11 @@ if _root not in (Path(p).resolve() for p in sys.path):
|
||||
|
||||
from benchmarks.wrapper_registry import (
|
||||
INDICATOR_CATEGORIES,
|
||||
LIBRARY_NAMES as LIBS,
|
||||
is_supported,
|
||||
)
|
||||
from benchmarks.wrapper_registry import (
|
||||
LIBRARY_NAMES as LIBS,
|
||||
)
|
||||
|
||||
|
||||
def _all_indicators() -> list[str]:
|
||||
@@ -34,11 +38,17 @@ def _all_indicators() -> list[str]:
|
||||
def main():
|
||||
p = Path(__file__).parent / "results.json"
|
||||
if not p.exists():
|
||||
print("Run: pytest benchmarks/test_speed.py --benchmark-only --benchmark-json=benchmarks/results.json -v", file=sys.stderr)
|
||||
print(
|
||||
"Run: pytest benchmarks/test_speed.py --benchmark-only --benchmark-json=benchmarks/results.json -v",
|
||||
file=sys.stderr,
|
||||
)
|
||||
sys.exit(1)
|
||||
raw = p.read_text().strip()
|
||||
if not raw:
|
||||
print("results.json is empty. Run the full benchmark suite first.", file=sys.stderr)
|
||||
print(
|
||||
"results.json is empty. Run the full benchmark suite first.",
|
||||
file=sys.stderr,
|
||||
)
|
||||
sys.exit(1)
|
||||
try:
|
||||
data = json.loads(raw)
|
||||
|
||||
@@ -88,7 +88,9 @@ def main() -> int:
|
||||
|
||||
data = json.loads(path.read_text(encoding="utf-8"))
|
||||
if not data.get("talib_available", False):
|
||||
print("ERROR: TA-Lib was not available; cannot enforce TA-Lib regression policy.")
|
||||
print(
|
||||
"ERROR: TA-Lib was not available; cannot enforce TA-Lib regression policy."
|
||||
)
|
||||
return 1
|
||||
|
||||
summary_by_size = {
|
||||
@@ -133,9 +135,7 @@ def main() -> int:
|
||||
)
|
||||
|
||||
if rows < args.min_rows:
|
||||
failures.append(
|
||||
f"size={size} rows {rows} < min_rows {args.min_rows}"
|
||||
)
|
||||
failures.append(f"size={size} rows {rows} < min_rows {args.min_rows}")
|
||||
if med < median_floor.get(size, float("-inf")):
|
||||
failures.append(
|
||||
f"size={size} median_speedup {med:.4f} < floor {median_floor[size]:.4f}"
|
||||
|
||||
@@ -50,11 +50,17 @@ def _naive_beta(x: np.ndarray, y: np.ndarray, window: int) -> np.ndarray:
|
||||
for end in range(window, len(x)):
|
||||
start = end - window
|
||||
rx = np.array(
|
||||
[x[idx + 1] / x[idx] - 1.0 if x[idx] != 0.0 else np.nan for idx in range(start, end)],
|
||||
[
|
||||
x[idx + 1] / x[idx] - 1.0 if x[idx] != 0.0 else np.nan
|
||||
for idx in range(start, end)
|
||||
],
|
||||
dtype=np.float64,
|
||||
)
|
||||
ry = np.array(
|
||||
[y[idx + 1] / y[idx] - 1.0 if y[idx] != 0.0 else np.nan for idx in range(start, end)],
|
||||
[
|
||||
y[idx + 1] / y[idx] - 1.0 if y[idx] != 0.0 else np.nan
|
||||
for idx in range(start, end)
|
||||
],
|
||||
dtype=np.float64,
|
||||
)
|
||||
mean_x = float(np.sum(rx)) / window
|
||||
@@ -120,7 +126,12 @@ def build_hotspot_report(
|
||||
high = close + rng.uniform(0.1, 2.0, price_bars)
|
||||
low = close - rng.uniform(0.1, 2.0, price_bars)
|
||||
iv = rng.uniform(10.0, 40.0, iv_bars).astype(np.float64)
|
||||
ohlcv = {"close": close, "high": high, "low": low, "volume": np.full(price_bars, 1000.0)}
|
||||
ohlcv = {
|
||||
"close": close,
|
||||
"high": high,
|
||||
"low": low,
|
||||
"volume": np.full(price_bars, 1000.0),
|
||||
}
|
||||
|
||||
rows = [
|
||||
(
|
||||
@@ -218,7 +229,9 @@ def build_hotspot_report(
|
||||
results.sort(key=lambda row: row["fast_ms"], reverse=True)
|
||||
total_fast_ms = sum(float(row["fast_ms"]) for row in results) or 1.0
|
||||
for row in results:
|
||||
row["share_of_suite_pct"] = round(float(row["fast_ms"]) / total_fast_ms * 100.0, 2)
|
||||
row["share_of_suite_pct"] = round(
|
||||
float(row["fast_ms"]) / total_fast_ms * 100.0, 2
|
||||
)
|
||||
|
||||
return {
|
||||
"metadata": benchmark_metadata(
|
||||
@@ -249,7 +262,9 @@ def main() -> int:
|
||||
window=args.window,
|
||||
)
|
||||
|
||||
print(f"{'Category':<16} {'Case':<18} {'Fast (ms)':>10} {'Ref (ms)':>10} {'Speedup':>10}")
|
||||
print(
|
||||
f"{'Category':<16} {'Case':<18} {'Fast (ms)':>10} {'Ref (ms)':>10} {'Speedup':>10}"
|
||||
)
|
||||
print("-" * 70)
|
||||
for row in payload["results"]:
|
||||
print(
|
||||
|
||||
@@ -16094,4 +16094,4 @@
|
||||
],
|
||||
"datetime": "2026-03-23T17:14:05.427766+00:00",
|
||||
"version": "5.2.3"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -52,7 +52,9 @@ def build_indicator_latency_report(*, rounds: int = 5) -> dict[str, Any]:
|
||||
|
||||
rows: list[dict[str, Any]] = []
|
||||
for entry in INDICATOR_SUITE:
|
||||
elapsed_ms = _time_min(lambda entry=entry: _run_indicator(entry, ohlcv), rounds=rounds)
|
||||
elapsed_ms = _time_min(
|
||||
lambda entry=entry: _run_indicator(entry, ohlcv), rounds=rounds
|
||||
)
|
||||
rows.append(
|
||||
{
|
||||
"name": entry["name"],
|
||||
@@ -192,10 +194,7 @@ def main() -> int:
|
||||
fixtures=[FIXTURE_PATH],
|
||||
extra={"output_dir": str(output_dir)},
|
||||
),
|
||||
"artifacts": {
|
||||
name: file_info(path)
|
||||
for name, path in artifacts.items()
|
||||
},
|
||||
"artifacts": {name: file_info(path) for name, path in artifacts.items()},
|
||||
}
|
||||
manifest_path = output_dir / "manifest.json"
|
||||
_write_json(manifest_path, manifest)
|
||||
|
||||
+60
-53
@@ -5,65 +5,66 @@ For each indicator we compare ferro_ta output against every available reference
|
||||
Tolerances are based on known algorithmic differences (e.g. Wilder vs SMA seed).
|
||||
We only compare the overlapping (valid) suffix of each output array.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from benchmarks.data_generator import MEDIUM
|
||||
from benchmarks.wrapper_registry import (
|
||||
execute_indicator,
|
||||
INDICATOR_NAMES,
|
||||
INDICATOR_CATEGORIES,
|
||||
CUMULATIVE_INDICATORS,
|
||||
BINARY_INDICATORS,
|
||||
CUMULATIVE_INDICATORS,
|
||||
INDICATOR_CATEGORIES,
|
||||
INDICATOR_NAMES,
|
||||
available_libraries,
|
||||
execute_indicator,
|
||||
is_supported,
|
||||
)
|
||||
|
||||
# Reference = ferro_ta; compare against each library that has a non-empty result.
|
||||
REFERENCE_LIB = "ferro_ta"
|
||||
COMPARISON_LIBS = [l for l in available_libraries() if l != REFERENCE_LIB]
|
||||
COMPARISON_LIBS = [
|
||||
library for library in available_libraries() if library != REFERENCE_LIB
|
||||
]
|
||||
|
||||
# Per-indicator tolerances (rtol, atol)
|
||||
_TOLERANCES: dict[str, tuple[float, float]] = {
|
||||
"ATR": (1e-3, 0.05), # Wilder's smoothing seed differs
|
||||
"NATR": (1e-3, 0.10),
|
||||
"BBANDS": (1e-3, 0.20), # ddof=0 vs ddof=1
|
||||
"ATR": (1e-3, 0.05), # Wilder's smoothing seed differs
|
||||
"NATR": (1e-3, 0.10),
|
||||
"BBANDS": (1e-3, 0.20), # ddof=0 vs ddof=1
|
||||
"STDDEV": (1e-3, 0.20),
|
||||
"VAR": (1e-3, 0.50),
|
||||
"MACD": (1e-3, 1e-3), # double EMA seed
|
||||
"KAMA": (1e-3, 1e-3),
|
||||
"STOCH": (1e-3, 0.10), # smoothing method differences
|
||||
"SAR": (1e-3, 0.20),
|
||||
"ADOSC": (1e-3, 0.20),
|
||||
"ADX": (1e-3, 0.50), # Wilder's ADX
|
||||
"PLUS_DI":(1e-3, 0.50),
|
||||
"MINUS_DI":(1e-3, 0.50),
|
||||
"PPO": (1e-2, 1e-3),
|
||||
"CMO": (1e-3, 0.10),
|
||||
"TRIX": (1e-3, 1e-3),
|
||||
"CCI": (1e-3, 0.10),
|
||||
"VAR": (1e-3, 0.50),
|
||||
"MACD": (1e-3, 1.00), # seed differences across libraries
|
||||
"KAMA": (1e-3, 1e-3),
|
||||
"STOCH": (1e-3, 0.10), # smoothing method differences
|
||||
"SAR": (1e-3, 0.20),
|
||||
"ADOSC": (1e-3, 0.20),
|
||||
"ADX": (1e-3, 0.50), # Wilder's ADX
|
||||
"PLUS_DI": (1e-3, 0.50),
|
||||
"MINUS_DI": (1e-3, 0.50),
|
||||
"PPO": (1e-2, 1e-3),
|
||||
"CMO": (1e-3, 0.10),
|
||||
"TRIX": (1e-3, 0.05),
|
||||
"CCI": (1e-3, 0.10),
|
||||
"SUPERTREND": (1e-2, 0.50),
|
||||
"KELTNER_CHANNELS": (1e-2, 0.50),
|
||||
"DONCHIAN": (1e-4, 1e-4),
|
||||
"HT_DCPERIOD": (1e-2, 1.0),
|
||||
"VWAP": (1e-3, 0.10),
|
||||
"AROON": (1e-4, 1e-3),
|
||||
"HT_DCPERIOD": (1e-2, 2.0),
|
||||
"VWAP": (1e-3, 0.10),
|
||||
"AROON": (1e-4, 1e-3),
|
||||
"LINEARREG": (1e-4, 1e-4),
|
||||
"LINEARREG_SLOPE": (1e-4, 1e-4),
|
||||
"CORREL": (1e-4, 1e-3),
|
||||
"BETA": (1e-3, 1e-3),
|
||||
"TSF": (1e-4, 1e-4),
|
||||
"EMA": (1e-3, 0.30), # ta library uses different EMA seed
|
||||
"DEMA": (1e-3, 0.50),
|
||||
"TEMA": (1e-3, 0.50),
|
||||
"T3": (1e-3, 0.50),
|
||||
"HULL_MA":(1e-3, 0.10),
|
||||
"WMA": (1e-4, 1e-4),
|
||||
"TRIMA": (1e-4, 1e-4),
|
||||
"MACD": (1e-3, 1.00), # seed differences across libraries
|
||||
"TRIX": (1e-3, 0.05),
|
||||
"HT_DCPERIOD": (1e-2, 2.0),
|
||||
"BETA": (1e-3, 1e-3),
|
||||
"TSF": (1e-4, 1e-4),
|
||||
"EMA": (1e-3, 0.30), # ta library uses different EMA seed
|
||||
"DEMA": (1e-3, 0.50),
|
||||
"TEMA": (1e-3, 0.50),
|
||||
"T3": (1e-3, 0.50),
|
||||
"HULL_MA": (1e-3, 0.10),
|
||||
"WMA": (1e-4, 1e-4),
|
||||
"TRIMA": (1e-4, 1e-4),
|
||||
}
|
||||
|
||||
_DEFAULT_TOL = (1e-4, 1e-5)
|
||||
@@ -71,33 +72,33 @@ _DEFAULT_TOL = (1e-4, 1e-5)
|
||||
# Pairs that use correlation check (>=0.95) due to known algorithmic divergence
|
||||
# Format: (indicator, library) or just indicator (applies to all libs)
|
||||
_CORRELATION_PAIRS: set[tuple[str, str]] = {
|
||||
("PPO", "talib"), # different PPO formula normalization
|
||||
("PPO", "talib"), # different PPO formula normalization
|
||||
("PPO", "pandas_ta"),
|
||||
("PPO", "tulipy"),
|
||||
("STOCH", "ta"),
|
||||
("SUPERTREND", "pandas_ta"),
|
||||
("KELTNER_CHANNELS", "pandas_ta"),
|
||||
("KELTNER_CHANNELS", "ta"),
|
||||
("EMA", "finta"), # finta EMA uses different initialization
|
||||
("KAMA", "pandas_ta"), # pandas_ta KAMA has slightly different seed
|
||||
("RSI", "ta"), # ta uses SMA warmup vs Wilder
|
||||
("RSI", "finta"), # same
|
||||
("EMA", "finta"), # finta EMA uses different initialization
|
||||
("KAMA", "pandas_ta"), # pandas_ta KAMA has slightly different seed
|
||||
("RSI", "ta"), # ta uses SMA warmup vs Wilder
|
||||
("RSI", "finta"), # same
|
||||
}
|
||||
|
||||
# Pairs that are skipped because they are structurally incompatible
|
||||
_SKIP_PAIRS: set[tuple[str, str]] = {
|
||||
("BBANDS", "finta"), # finta normalizes band differently
|
||||
("ATR", "finta"), # finta ATR uses simple TR not Wilder
|
||||
("STDDEV", "finta"), # finta uses population std
|
||||
("TRIMA", "finta"), # finta TRIMA uses different formula
|
||||
("PPO", "finta"), # finta PPO scaling incompatible
|
||||
("STOCH", "finta"), # finta STOCH formula differs
|
||||
("VWAP", "pandas_ta"), # pandas_ta VWAP anchors to session start
|
||||
("BBANDS", "finta"), # finta normalizes band differently
|
||||
("ATR", "finta"), # finta ATR uses simple TR not Wilder
|
||||
("STDDEV", "finta"), # finta uses population std
|
||||
("TRIMA", "finta"), # finta TRIMA uses different formula
|
||||
("PPO", "finta"), # finta PPO scaling incompatible
|
||||
("STOCH", "finta"), # finta STOCH formula differs
|
||||
("VWAP", "pandas_ta"), # pandas_ta VWAP anchors to session start
|
||||
("HT_TRENDMODE", "talib"), # binary; Hilbert seed diverges
|
||||
("CMO", "talib"), # ferro_ta CMO smoothing variant corr < 0.90
|
||||
("CMO", "talib"), # ferro_ta CMO smoothing variant corr < 0.90
|
||||
("CMO", "pandas_ta"),
|
||||
("CMO", "finta"),
|
||||
("PLUS_DI", "pandas_ta"), # pandas_ta ADX column naming corr < 0.70
|
||||
("PLUS_DI", "pandas_ta"), # pandas_ta ADX column naming corr < 0.70
|
||||
}
|
||||
|
||||
MIN_OVERLAP = 30 # minimum points to make comparison meaningful
|
||||
@@ -114,12 +115,14 @@ def _compare(ref: np.ndarray, cmp: np.ndarray, indicator: str, library: str) ->
|
||||
c = cmp[-n:]
|
||||
if indicator in BINARY_INDICATORS or (indicator, library) in _CORRELATION_PAIRS:
|
||||
# Use correlation check for structurally different algorithms
|
||||
corr = np.corrcoef(r, c)[0, 1] if not indicator in BINARY_INDICATORS else None
|
||||
corr = np.corrcoef(r, c)[0, 1] if indicator not in BINARY_INDICATORS else None
|
||||
if indicator in BINARY_INDICATORS:
|
||||
agree = np.mean(r == c)
|
||||
assert agree >= 0.80, f"Binary agreement {agree:.1%} < 80%"
|
||||
else:
|
||||
assert corr >= 0.90, f"Correlation {corr:.4f} < 0.90 (structural divergence)"
|
||||
assert corr >= 0.90, (
|
||||
f"Correlation {corr:.4f} < 0.90 (structural divergence)"
|
||||
)
|
||||
elif indicator in CUMULATIVE_INDICATORS:
|
||||
dr, dc = np.diff(r), np.diff(c)
|
||||
if len(dr) < 5 or len(dc) < 5:
|
||||
@@ -136,6 +139,7 @@ def _compare(ref: np.ndarray, cmp: np.ndarray, indicator: str, library: str) ->
|
||||
|
||||
# ── dynamically generate one test per (indicator, library) pair ─────────────
|
||||
|
||||
|
||||
def pytest_generate_tests(metafunc):
|
||||
if "indicator" in metafunc.fixturenames and "library" in metafunc.fixturenames:
|
||||
params = []
|
||||
@@ -172,6 +176,7 @@ class TestAccuracy:
|
||||
|
||||
# ── quick smoke tests that always run (no skip) ──────────────────────────────
|
||||
|
||||
|
||||
class TestSmoke:
|
||||
"""Sanity checks that ferro_ta returns non-empty finite arrays."""
|
||||
|
||||
@@ -182,7 +187,9 @@ class TestSmoke:
|
||||
|
||||
arr = execute_indicator("ferro_ta", indicator, MEDIUM)
|
||||
assert len(arr) > 0, f"ferro_ta {indicator} returned empty array"
|
||||
assert np.all(np.isfinite(arr)), f"ferro_ta {indicator} has non-finite values: {arr[~np.isfinite(arr)][:5]}"
|
||||
assert np.all(np.isfinite(arr)), (
|
||||
f"ferro_ta {indicator} has non-finite values: {arr[~np.isfinite(arr)][:5]}"
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize("category,indicators", INDICATOR_CATEGORIES.items())
|
||||
def test_category_coverage(self, category, indicators):
|
||||
|
||||
+31
-18
@@ -6,31 +6,36 @@ Run: pytest benchmarks/test_speed.py --benchmark-only -v
|
||||
|
||||
Streaming benchmarks are in test_streaming_speed.py
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from benchmarks.data_generator import LARGE
|
||||
from benchmarks.wrapper_registry import (
|
||||
execute_indicator,
|
||||
INDICATOR_CATEGORIES,
|
||||
available_libraries,
|
||||
execute_indicator,
|
||||
is_supported,
|
||||
)
|
||||
|
||||
BENCH_DATA = LARGE # 100k bars for main benchmarks
|
||||
BENCH_LIBS = available_libraries()
|
||||
BENCH_DATA = LARGE # 100k bars for main benchmarks
|
||||
BENCH_LIBS = available_libraries()
|
||||
|
||||
|
||||
def _make_bench(indicator: str, library: str):
|
||||
"""Return a benchmark function that runs indicator on library (uses BENCH_DATA)."""
|
||||
|
||||
def _fn():
|
||||
execute_indicator(library, indicator, BENCH_DATA)
|
||||
|
||||
_fn.__name__ = f"{library}_{indicator}"
|
||||
return _fn
|
||||
|
||||
|
||||
# ── Parametrize over all (indicator, library) combinations ───────────────────
|
||||
|
||||
|
||||
def pytest_generate_tests(metafunc):
|
||||
if "indicator" in metafunc.fixturenames and "library" in metafunc.fixturenames:
|
||||
params = []
|
||||
@@ -53,20 +58,24 @@ class TestSpeed:
|
||||
|
||||
# ── Standalone head-to-head for the most important indicators ─────────────────
|
||||
|
||||
@pytest.mark.parametrize("indicator,libs", [
|
||||
("SMA", ["ferro_ta","talib","tulipy","pandas_ta","ta","finta"]),
|
||||
("EMA", ["ferro_ta","talib","tulipy","pandas_ta","ta","finta"]),
|
||||
("RSI", ["ferro_ta","talib","tulipy","pandas_ta","ta","finta"]),
|
||||
("MACD", ["ferro_ta","talib","tulipy","pandas_ta","ta","finta"]),
|
||||
("BBANDS",["ferro_ta","talib","tulipy","pandas_ta","ta","finta"]),
|
||||
("ATR", ["ferro_ta","talib","tulipy","pandas_ta","ta","finta"]),
|
||||
("CCI", ["ferro_ta","talib","tulipy","pandas_ta","ta","finta"]),
|
||||
("WILLR", ["ferro_ta","talib","tulipy","pandas_ta","ta","finta"]),
|
||||
("OBV", ["ferro_ta","talib","tulipy","pandas_ta","ta","finta"]),
|
||||
("ADX", ["ferro_ta","talib","tulipy","pandas_ta","ta","finta"]),
|
||||
("MFI", ["ferro_ta","talib","tulipy","pandas_ta","ta","finta"]),
|
||||
("STOCH", ["ferro_ta","talib","tulipy","pandas_ta","ta","finta"]),
|
||||
])
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"indicator,libs",
|
||||
[
|
||||
("SMA", ["ferro_ta", "talib", "tulipy", "pandas_ta", "ta", "finta"]),
|
||||
("EMA", ["ferro_ta", "talib", "tulipy", "pandas_ta", "ta", "finta"]),
|
||||
("RSI", ["ferro_ta", "talib", "tulipy", "pandas_ta", "ta", "finta"]),
|
||||
("MACD", ["ferro_ta", "talib", "tulipy", "pandas_ta", "ta", "finta"]),
|
||||
("BBANDS", ["ferro_ta", "talib", "tulipy", "pandas_ta", "ta", "finta"]),
|
||||
("ATR", ["ferro_ta", "talib", "tulipy", "pandas_ta", "ta", "finta"]),
|
||||
("CCI", ["ferro_ta", "talib", "tulipy", "pandas_ta", "ta", "finta"]),
|
||||
("WILLR", ["ferro_ta", "talib", "tulipy", "pandas_ta", "ta", "finta"]),
|
||||
("OBV", ["ferro_ta", "talib", "tulipy", "pandas_ta", "ta", "finta"]),
|
||||
("ADX", ["ferro_ta", "talib", "tulipy", "pandas_ta", "ta", "finta"]),
|
||||
("MFI", ["ferro_ta", "talib", "tulipy", "pandas_ta", "ta", "finta"]),
|
||||
("STOCH", ["ferro_ta", "talib", "tulipy", "pandas_ta", "ta", "finta"]),
|
||||
],
|
||||
)
|
||||
def test_head_to_head(benchmark, indicator, libs):
|
||||
"""Benchmark ferro_ta vs all peers — for README table generation."""
|
||||
if not is_supported("ferro_ta", indicator):
|
||||
@@ -77,7 +86,11 @@ def test_head_to_head(benchmark, indicator, libs):
|
||||
|
||||
# ── Large dataset benchmarks (100k bars) ─────────────────────────────────────
|
||||
|
||||
@pytest.mark.parametrize("indicator", ["SMA","EMA","RSI","MACD","ATR","BBANDS","OBV","CCI","ADX","MFI"])
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"indicator",
|
||||
["SMA", "EMA", "RSI", "MACD", "ATR", "BBANDS", "OBV", "CCI", "ADX", "MFI"],
|
||||
)
|
||||
def test_large_dataset(benchmark, indicator):
|
||||
"""Scaling benchmark at 100k bars for ferro_ta."""
|
||||
if not is_supported("ferro_ta", indicator):
|
||||
|
||||
+2521
-616
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user