feat: refresh benchmark coverage and harden CI tooling
Refresh the benchmark and performance surface across the repo. This updates the benchmark wrappers and helper scripts, regenerates the checked-in benchmark and perf-contract artifacts, and folds in the related roadmap, compatibility, and example notebook changes that belong with this performance-focused pass. Harden the Python CI and local pre-push flow so the same checks pass reliably in both places. The workflow and pre-push script now use module-safe uv typecheck invocations, the Python test environment installs the optional MCP dependency needed by the MCP server tests, and one-off root benchmark outputs are ignored to keep the repo clean. Align local tooling with the current project configuration by updating the Ruff pre-commit hook, tightening the API typing and MCP server helpers, and refreshing the lockfile to pick up the audited PyJWT fix while preserving the rest of the staged source changes.
This commit is contained in:
Pratik Bhadane
+60
-53
@@ -5,65 +5,66 @@ For each indicator we compare ferro_ta output against every available reference
|
||||
Tolerances are based on known algorithmic differences (e.g. Wilder vs SMA seed).
|
||||
We only compare the overlapping (valid) suffix of each output array.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from benchmarks.data_generator import MEDIUM
|
||||
from benchmarks.wrapper_registry import (
|
||||
execute_indicator,
|
||||
INDICATOR_NAMES,
|
||||
INDICATOR_CATEGORIES,
|
||||
CUMULATIVE_INDICATORS,
|
||||
BINARY_INDICATORS,
|
||||
CUMULATIVE_INDICATORS,
|
||||
INDICATOR_CATEGORIES,
|
||||
INDICATOR_NAMES,
|
||||
available_libraries,
|
||||
execute_indicator,
|
||||
is_supported,
|
||||
)
|
||||
|
||||
# Reference = ferro_ta; compare against each library that has a non-empty result.
|
||||
REFERENCE_LIB = "ferro_ta"
|
||||
COMPARISON_LIBS = [l for l in available_libraries() if l != REFERENCE_LIB]
|
||||
COMPARISON_LIBS = [
|
||||
library for library in available_libraries() if library != REFERENCE_LIB
|
||||
]
|
||||
|
||||
# Per-indicator tolerances (rtol, atol)
|
||||
_TOLERANCES: dict[str, tuple[float, float]] = {
|
||||
"ATR": (1e-3, 0.05), # Wilder's smoothing seed differs
|
||||
"NATR": (1e-3, 0.10),
|
||||
"BBANDS": (1e-3, 0.20), # ddof=0 vs ddof=1
|
||||
"ATR": (1e-3, 0.05), # Wilder's smoothing seed differs
|
||||
"NATR": (1e-3, 0.10),
|
||||
"BBANDS": (1e-3, 0.20), # ddof=0 vs ddof=1
|
||||
"STDDEV": (1e-3, 0.20),
|
||||
"VAR": (1e-3, 0.50),
|
||||
"MACD": (1e-3, 1e-3), # double EMA seed
|
||||
"KAMA": (1e-3, 1e-3),
|
||||
"STOCH": (1e-3, 0.10), # smoothing method differences
|
||||
"SAR": (1e-3, 0.20),
|
||||
"ADOSC": (1e-3, 0.20),
|
||||
"ADX": (1e-3, 0.50), # Wilder's ADX
|
||||
"PLUS_DI":(1e-3, 0.50),
|
||||
"MINUS_DI":(1e-3, 0.50),
|
||||
"PPO": (1e-2, 1e-3),
|
||||
"CMO": (1e-3, 0.10),
|
||||
"TRIX": (1e-3, 1e-3),
|
||||
"CCI": (1e-3, 0.10),
|
||||
"VAR": (1e-3, 0.50),
|
||||
"MACD": (1e-3, 1.00), # seed differences across libraries
|
||||
"KAMA": (1e-3, 1e-3),
|
||||
"STOCH": (1e-3, 0.10), # smoothing method differences
|
||||
"SAR": (1e-3, 0.20),
|
||||
"ADOSC": (1e-3, 0.20),
|
||||
"ADX": (1e-3, 0.50), # Wilder's ADX
|
||||
"PLUS_DI": (1e-3, 0.50),
|
||||
"MINUS_DI": (1e-3, 0.50),
|
||||
"PPO": (1e-2, 1e-3),
|
||||
"CMO": (1e-3, 0.10),
|
||||
"TRIX": (1e-3, 0.05),
|
||||
"CCI": (1e-3, 0.10),
|
||||
"SUPERTREND": (1e-2, 0.50),
|
||||
"KELTNER_CHANNELS": (1e-2, 0.50),
|
||||
"DONCHIAN": (1e-4, 1e-4),
|
||||
"HT_DCPERIOD": (1e-2, 1.0),
|
||||
"VWAP": (1e-3, 0.10),
|
||||
"AROON": (1e-4, 1e-3),
|
||||
"HT_DCPERIOD": (1e-2, 2.0),
|
||||
"VWAP": (1e-3, 0.10),
|
||||
"AROON": (1e-4, 1e-3),
|
||||
"LINEARREG": (1e-4, 1e-4),
|
||||
"LINEARREG_SLOPE": (1e-4, 1e-4),
|
||||
"CORREL": (1e-4, 1e-3),
|
||||
"BETA": (1e-3, 1e-3),
|
||||
"TSF": (1e-4, 1e-4),
|
||||
"EMA": (1e-3, 0.30), # ta library uses different EMA seed
|
||||
"DEMA": (1e-3, 0.50),
|
||||
"TEMA": (1e-3, 0.50),
|
||||
"T3": (1e-3, 0.50),
|
||||
"HULL_MA":(1e-3, 0.10),
|
||||
"WMA": (1e-4, 1e-4),
|
||||
"TRIMA": (1e-4, 1e-4),
|
||||
"MACD": (1e-3, 1.00), # seed differences across libraries
|
||||
"TRIX": (1e-3, 0.05),
|
||||
"HT_DCPERIOD": (1e-2, 2.0),
|
||||
"BETA": (1e-3, 1e-3),
|
||||
"TSF": (1e-4, 1e-4),
|
||||
"EMA": (1e-3, 0.30), # ta library uses different EMA seed
|
||||
"DEMA": (1e-3, 0.50),
|
||||
"TEMA": (1e-3, 0.50),
|
||||
"T3": (1e-3, 0.50),
|
||||
"HULL_MA": (1e-3, 0.10),
|
||||
"WMA": (1e-4, 1e-4),
|
||||
"TRIMA": (1e-4, 1e-4),
|
||||
}
|
||||
|
||||
_DEFAULT_TOL = (1e-4, 1e-5)
|
||||
@@ -71,33 +72,33 @@ _DEFAULT_TOL = (1e-4, 1e-5)
|
||||
# Pairs that use correlation check (>=0.95) due to known algorithmic divergence
|
||||
# Format: (indicator, library) or just indicator (applies to all libs)
|
||||
_CORRELATION_PAIRS: set[tuple[str, str]] = {
|
||||
("PPO", "talib"), # different PPO formula normalization
|
||||
("PPO", "talib"), # different PPO formula normalization
|
||||
("PPO", "pandas_ta"),
|
||||
("PPO", "tulipy"),
|
||||
("STOCH", "ta"),
|
||||
("SUPERTREND", "pandas_ta"),
|
||||
("KELTNER_CHANNELS", "pandas_ta"),
|
||||
("KELTNER_CHANNELS", "ta"),
|
||||
("EMA", "finta"), # finta EMA uses different initialization
|
||||
("KAMA", "pandas_ta"), # pandas_ta KAMA has slightly different seed
|
||||
("RSI", "ta"), # ta uses SMA warmup vs Wilder
|
||||
("RSI", "finta"), # same
|
||||
("EMA", "finta"), # finta EMA uses different initialization
|
||||
("KAMA", "pandas_ta"), # pandas_ta KAMA has slightly different seed
|
||||
("RSI", "ta"), # ta uses SMA warmup vs Wilder
|
||||
("RSI", "finta"), # same
|
||||
}
|
||||
|
||||
# Pairs that are skipped because they are structurally incompatible
|
||||
_SKIP_PAIRS: set[tuple[str, str]] = {
|
||||
("BBANDS", "finta"), # finta normalizes band differently
|
||||
("ATR", "finta"), # finta ATR uses simple TR not Wilder
|
||||
("STDDEV", "finta"), # finta uses population std
|
||||
("TRIMA", "finta"), # finta TRIMA uses different formula
|
||||
("PPO", "finta"), # finta PPO scaling incompatible
|
||||
("STOCH", "finta"), # finta STOCH formula differs
|
||||
("VWAP", "pandas_ta"), # pandas_ta VWAP anchors to session start
|
||||
("BBANDS", "finta"), # finta normalizes band differently
|
||||
("ATR", "finta"), # finta ATR uses simple TR not Wilder
|
||||
("STDDEV", "finta"), # finta uses population std
|
||||
("TRIMA", "finta"), # finta TRIMA uses different formula
|
||||
("PPO", "finta"), # finta PPO scaling incompatible
|
||||
("STOCH", "finta"), # finta STOCH formula differs
|
||||
("VWAP", "pandas_ta"), # pandas_ta VWAP anchors to session start
|
||||
("HT_TRENDMODE", "talib"), # binary; Hilbert seed diverges
|
||||
("CMO", "talib"), # ferro_ta CMO smoothing variant corr < 0.90
|
||||
("CMO", "talib"), # ferro_ta CMO smoothing variant corr < 0.90
|
||||
("CMO", "pandas_ta"),
|
||||
("CMO", "finta"),
|
||||
("PLUS_DI", "pandas_ta"), # pandas_ta ADX column naming corr < 0.70
|
||||
("PLUS_DI", "pandas_ta"), # pandas_ta ADX column naming corr < 0.70
|
||||
}
|
||||
|
||||
MIN_OVERLAP = 30 # minimum points to make comparison meaningful
|
||||
@@ -114,12 +115,14 @@ def _compare(ref: np.ndarray, cmp: np.ndarray, indicator: str, library: str) ->
|
||||
c = cmp[-n:]
|
||||
if indicator in BINARY_INDICATORS or (indicator, library) in _CORRELATION_PAIRS:
|
||||
# Use correlation check for structurally different algorithms
|
||||
corr = np.corrcoef(r, c)[0, 1] if not indicator in BINARY_INDICATORS else None
|
||||
corr = np.corrcoef(r, c)[0, 1] if indicator not in BINARY_INDICATORS else None
|
||||
if indicator in BINARY_INDICATORS:
|
||||
agree = np.mean(r == c)
|
||||
assert agree >= 0.80, f"Binary agreement {agree:.1%} < 80%"
|
||||
else:
|
||||
assert corr >= 0.90, f"Correlation {corr:.4f} < 0.90 (structural divergence)"
|
||||
assert corr >= 0.90, (
|
||||
f"Correlation {corr:.4f} < 0.90 (structural divergence)"
|
||||
)
|
||||
elif indicator in CUMULATIVE_INDICATORS:
|
||||
dr, dc = np.diff(r), np.diff(c)
|
||||
if len(dr) < 5 or len(dc) < 5:
|
||||
@@ -136,6 +139,7 @@ def _compare(ref: np.ndarray, cmp: np.ndarray, indicator: str, library: str) ->
|
||||
|
||||
# ── dynamically generate one test per (indicator, library) pair ─────────────
|
||||
|
||||
|
||||
def pytest_generate_tests(metafunc):
|
||||
if "indicator" in metafunc.fixturenames and "library" in metafunc.fixturenames:
|
||||
params = []
|
||||
@@ -172,6 +176,7 @@ class TestAccuracy:
|
||||
|
||||
# ── quick smoke tests that always run (no skip) ──────────────────────────────
|
||||
|
||||
|
||||
class TestSmoke:
|
||||
"""Sanity checks that ferro_ta returns non-empty finite arrays."""
|
||||
|
||||
@@ -182,7 +187,9 @@ class TestSmoke:
|
||||
|
||||
arr = execute_indicator("ferro_ta", indicator, MEDIUM)
|
||||
assert len(arr) > 0, f"ferro_ta {indicator} returned empty array"
|
||||
assert np.all(np.isfinite(arr)), f"ferro_ta {indicator} has non-finite values: {arr[~np.isfinite(arr)][:5]}"
|
||||
assert np.all(np.isfinite(arr)), (
|
||||
f"ferro_ta {indicator} has non-finite values: {arr[~np.isfinite(arr)][:5]}"
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize("category,indicators", INDICATOR_CATEGORIES.items())
|
||||
def test_category_coverage(self, category, indicators):
|
||||
|
||||
Reference in New Issue
Block a user