feat: broaden CPU and platform coverage across PyPI, nodes, and crate… (#26)
* feat: broaden CPU and platform coverage across PyPI, nodes, and crates.io Replace static SIMD with runtime CPU-feature dispatch and expand the release wheel matrix so one set of artifacts runs on any target CPU and platform without illegal-instruction crashes. Rust core: - Add multiversion runtime dispatch (crates/ferro_ta_core/src/simd.rs); drop compile-time `wide`. `simd` feature is now default-on and forwarded through the pyo3 crate, and stays compatible with #![forbid(unsafe_code)]. Packaging: - abi3-py310: one cp310-abi3 wheel per platform (covers CPython 3.10+). - CI matrix adds Linux aarch64 + musllinux (x86_64/aarch64) and Windows arm64. Node/Docker + docs: - api/Dockerfile: document baseline+dispatch (no target-cpu pin) and add a fail-fast import check; aarch64 containers now install cleanly. - Rewrite docs/guides/simd.md; fix stale `wide` mention in ADR 0003. - Add ADR 0006 (CPU coverage strategy). Also bundles in-flight release prep already staged in the tree (DTW exception types, SBOM/provenance security, supporting docs). * fix(ci): clear cargo-deny and pip-audit failures; apply dependency bumps cargo-deny (advisories): - Ignore pyo3 RUSTSEC-2026-0176 / RUSTSEC-2026-0177 in deny.toml with a documented rationale: ferro-ta uses neither affected code path (PyList/PyTuple nth iterators; PyCFunction::new_closure). Upstream fix needs pyo3 >=0.29 (large API migration), tracked as a follow-up. pip-audit: - Bump dev lockfile idna 3.18, pytest 9.1.1, urllib3 2.7.0 to clear PYSEC-2026-215, CVE-2025-71176, PYSEC-2026-141/142. Dependency bumps (supersede open dependabot PRs; they auto-close on merge): - cargo: log 0.4.32, serde_json 1.0.150, rayon 1.12.0 - api/requirements.txt: uvicorn>=0.49.0, pydantic>=2.13.4, ferro-ta>=1.1.4 - CI actions: deploy-pages v5, upload-pages-artifact v5, action-gh-release v3 The open `wide` 1.5.0 bump (PR #24) is obsolete — the crate is removed in this branch. * chore: address CodeRabbit review; remove docs/adr section CodeRabbit findings: - CI sbom job: add `attestations: write` so attest-build-provenance can run (it had only contents:write + id-token:write). - simd.rs: vectorize `wma_seed` with lane-local accumulators — it was scalar behind the multiversion wrapper, adding dispatch overhead for no SIMD gain. - CHANGELOG: consolidate the duplicate `### Changed` heading. - python/ferro_ta/__init__.py: also re-export the `FerroTaError` alias. - docs/guides/dtw.md: soften "byte-for-byte" parity to within-tolerance. Remove docs/adr/ at maintainer request and clean up the ADR links in the SIMD and DTW guides. The ADR files remain in commit 9506a30 if ever needed.
This commit is contained in:
Pratik Bhadane
@@ -16,7 +16,7 @@ name = "ferro_ta_core"
|
||||
crate-type = ["lib"]
|
||||
|
||||
[dependencies]
|
||||
wide = { version = "1.1.1", optional = true }
|
||||
multiversion = { version = "0.8", optional = true }
|
||||
serde = { version = "1.0", features = ["derive"], optional = true }
|
||||
serde_json = { version = "1.0", optional = true }
|
||||
|
||||
@@ -28,6 +28,12 @@ name = "indicators"
|
||||
harness = false
|
||||
|
||||
[features]
|
||||
wide = ["dep:wide"]
|
||||
simd = ["wide"]
|
||||
# Runtime CPU-feature dispatch (multiversion). Default ON so `cargo add
|
||||
# ferro_ta_core` and the published wheels get SIMD-accelerated reductions
|
||||
# that adapt to the running CPU (baseline .. AVX-512 / NEON) WITHOUT pinning
|
||||
# a target-cpu — one binary runs on any CPU of the target arch, with no
|
||||
# illegal-instruction crashes on older chips. Disable with
|
||||
# `--no-default-features` for a pure-scalar build.
|
||||
default = ["simd"]
|
||||
simd = ["dep:multiversion"]
|
||||
serde = ["dep:serde", "dep:serde_json"]
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
#![forbid(unsafe_code)]
|
||||
|
||||
/*!
|
||||
ferro_ta_core — Pure Rust indicator library.
|
||||
|
||||
@@ -49,6 +51,8 @@ pub mod price_transform;
|
||||
pub mod regime;
|
||||
pub mod resampling;
|
||||
pub mod signals;
|
||||
/// Runtime-dispatched SIMD reduction primitives (internal).
|
||||
pub(crate) mod simd;
|
||||
pub mod statistic;
|
||||
pub mod streaming;
|
||||
pub mod volatility;
|
||||
|
||||
@@ -38,27 +38,10 @@ pub fn sma_into(src: &[f64], timeperiod: usize, dest: &mut [f64], dest_offset: u
|
||||
return;
|
||||
}
|
||||
|
||||
#[cfg(feature = "simd")]
|
||||
let window_sum_init = {
|
||||
use wide::f64x4;
|
||||
let p_data = &src[..timeperiod];
|
||||
let mut sum = f64x4::splat(0.0);
|
||||
let mut chunks = p_data.chunks_exact(4);
|
||||
for chunk in &mut chunks {
|
||||
sum += f64x4::new([chunk[0], chunk[1], chunk[2], chunk[3]]);
|
||||
}
|
||||
let arr = sum.to_array();
|
||||
let mut total = arr[0] + arr[1] + arr[2] + arr[3];
|
||||
for &v in chunks.remainder() {
|
||||
total += v;
|
||||
}
|
||||
total
|
||||
};
|
||||
|
||||
#[cfg(not(feature = "simd"))]
|
||||
let window_sum_init: f64 = src[..timeperiod].iter().sum();
|
||||
|
||||
let mut window_sum = window_sum_init;
|
||||
// Seed the rolling window with a runtime-dispatched reduction. The O(n)
|
||||
// streaming recurrence below is inherently sequential, so SIMD only ever
|
||||
// applies to this initial window sum.
|
||||
let mut window_sum = crate::simd::sum(&src[..timeperiod]);
|
||||
let tp_f64 = timeperiod as f64;
|
||||
dest[dest_offset + timeperiod - 1] = window_sum / tp_f64;
|
||||
|
||||
@@ -124,46 +107,9 @@ pub fn wma(close: &[f64], timeperiod: usize) -> Vec<f64> {
|
||||
let denom: f64 = (timeperiod * (timeperiod + 1) / 2) as f64;
|
||||
let p = timeperiod as f64;
|
||||
|
||||
// Seed: compute T and S for the first window.
|
||||
#[cfg(feature = "simd")]
|
||||
let (mut t, mut s) = {
|
||||
use wide::f64x4;
|
||||
let p_data = &close[..timeperiod];
|
||||
let mut t_simd = f64x4::splat(0.0);
|
||||
let mut s_simd = f64x4::splat(0.0);
|
||||
let mut chunks = p_data.chunks_exact(4);
|
||||
let mut idx = 1.0;
|
||||
let step = f64x4::new([0.0, 1.0, 2.0, 3.0]);
|
||||
|
||||
for chunk in &mut chunks {
|
||||
let vals = f64x4::new([chunk[0], chunk[1], chunk[2], chunk[3]]);
|
||||
let mults = f64x4::splat(idx) + step;
|
||||
t_simd += vals * mults;
|
||||
s_simd += vals;
|
||||
idx += 4.0;
|
||||
}
|
||||
let t_arr = t_simd.to_array();
|
||||
let s_arr = s_simd.to_array();
|
||||
let mut t = t_arr[0] + t_arr[1] + t_arr[2] + t_arr[3];
|
||||
let mut s = s_arr[0] + s_arr[1] + s_arr[2] + s_arr[3];
|
||||
for &v in chunks.remainder() {
|
||||
t += v * idx;
|
||||
s += v;
|
||||
idx += 1.0;
|
||||
}
|
||||
(t, s)
|
||||
};
|
||||
|
||||
#[cfg(not(feature = "simd"))]
|
||||
let (mut t, mut s) = {
|
||||
let t_val: f64 = close[..timeperiod]
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(k, &v)| v * (k + 1) as f64)
|
||||
.sum();
|
||||
let s_val: f64 = close[..timeperiod].iter().sum();
|
||||
(t_val, s_val)
|
||||
};
|
||||
// Seed: compute T and S for the first window via a runtime-dispatched
|
||||
// reduction (the streaming recurrence below is sequential).
|
||||
let (mut t, mut s) = crate::simd::wma_seed(&close[..timeperiod]);
|
||||
|
||||
result[timeperiod - 1] = t / denom;
|
||||
|
||||
|
||||
@@ -0,0 +1,161 @@
|
||||
//! Runtime-dispatched SIMD primitives.
|
||||
//!
|
||||
//! Each public reduction here is compiled into several CPU-feature-specific
|
||||
//! variants (baseline, SSE, AVX2/FMA, AVX-512 on x86_64; NEON on aarch64; …)
|
||||
//! by [`multiversion`]. The fastest variant the *current* CPU supports is
|
||||
//! chosen at runtime via CPUID. This gives one binary that:
|
||||
//!
|
||||
//! * runs on **any** CPU of the target architecture — no illegal-instruction
|
||||
//! (SIGILL) crashes on pre-AVX2 chips, unlike a static `-C target-cpu=…`;
|
||||
//! * still uses wide vector units where the hardware has them.
|
||||
//!
|
||||
//! The hot loops accumulate into **independent lanes** before a final
|
||||
//! horizontal combine. That is what lets the optimizer auto-vectorize them:
|
||||
//! a plain sequential `iter().sum()` is a dependency chain LLVM may not
|
||||
//! reorder (doing so would change floating-point rounding). As a consequence
|
||||
//! these results differ from a strict left-to-right sum by a few ULPs — well
|
||||
//! inside every indicator's documented tolerance.
|
||||
|
||||
/// Number of independent accumulator lanes. Eight `f64` lanes cover the
|
||||
/// widest target we dispatch to (AVX-512 = 8×f64); narrower targets (AVX2,
|
||||
/// NEON) simply use a subset.
|
||||
#[cfg(feature = "simd")]
|
||||
const LANES: usize = 8;
|
||||
|
||||
/// Sum of a slice of `f64`, runtime-dispatched.
|
||||
#[cfg(feature = "simd")]
|
||||
#[multiversion::multiversion(targets = "simd")]
|
||||
pub(crate) fn sum(data: &[f64]) -> f64 {
|
||||
let mut acc = [0.0f64; LANES];
|
||||
let mut chunks = data.chunks_exact(LANES);
|
||||
for chunk in &mut chunks {
|
||||
for (a, &v) in acc.iter_mut().zip(chunk) {
|
||||
*a += v;
|
||||
}
|
||||
}
|
||||
let remainder: f64 = chunks.remainder().iter().sum();
|
||||
remainder + acc.iter().sum::<f64>()
|
||||
}
|
||||
|
||||
/// Pure-scalar fallback when the `simd` feature is disabled.
|
||||
#[cfg(not(feature = "simd"))]
|
||||
pub(crate) fn sum(data: &[f64]) -> f64 {
|
||||
data.iter().sum()
|
||||
}
|
||||
|
||||
/// Weighted-moving-average seed for the first window.
|
||||
///
|
||||
/// Returns `(t, s)` where `t = Σ data[k] * (k + 1)` (1-based linear weights)
|
||||
/// and `s = Σ data[k]`. Used to seed the O(n) WMA recurrence.
|
||||
#[cfg(feature = "simd")]
|
||||
#[multiversion::multiversion(targets = "simd")]
|
||||
pub(crate) fn wma_seed(data: &[f64]) -> (f64, f64) {
|
||||
// Lane-local accumulation (same idea as `sum`) so each CPU-feature clone
|
||||
// can vectorize: `t` weights each value by its 1-based global index.
|
||||
let mut t_acc = [0.0f64; LANES];
|
||||
let mut s_acc = [0.0f64; LANES];
|
||||
let mut chunks = data.chunks_exact(LANES);
|
||||
let mut base = 0.0f64; // global index of this chunk's first element
|
||||
for chunk in &mut chunks {
|
||||
for (lane, ((t, s), &v)) in t_acc
|
||||
.iter_mut()
|
||||
.zip(s_acc.iter_mut())
|
||||
.zip(chunk)
|
||||
.enumerate()
|
||||
{
|
||||
*t += v * (base + lane as f64 + 1.0);
|
||||
*s += v;
|
||||
}
|
||||
base += LANES as f64;
|
||||
}
|
||||
let mut t = 0.0;
|
||||
let mut s = 0.0;
|
||||
for (i, &v) in chunks.remainder().iter().enumerate() {
|
||||
t += v * (base + i as f64 + 1.0);
|
||||
s += v;
|
||||
}
|
||||
(t + t_acc.iter().sum::<f64>(), s + s_acc.iter().sum::<f64>())
|
||||
}
|
||||
|
||||
/// Pure-scalar fallback when the `simd` feature is disabled.
|
||||
#[cfg(not(feature = "simd"))]
|
||||
pub(crate) fn wma_seed(data: &[f64]) -> (f64, f64) {
|
||||
let mut t = 0.0;
|
||||
let mut s = 0.0;
|
||||
for (k, &v) in data.iter().enumerate() {
|
||||
t += v * (k + 1) as f64;
|
||||
s += v;
|
||||
}
|
||||
(t, s)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// Strict sequential reference — the ground truth we compare against.
|
||||
fn naive_sum(data: &[f64]) -> f64 {
|
||||
data.iter().sum()
|
||||
}
|
||||
|
||||
fn naive_wma_seed(data: &[f64]) -> (f64, f64) {
|
||||
let t = data
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(k, &v)| v * (k + 1) as f64)
|
||||
.sum();
|
||||
let s = data.iter().sum();
|
||||
(t, s)
|
||||
}
|
||||
|
||||
/// Deterministic test vectors spanning the lane boundaries: empty, a
|
||||
/// partial chunk (< LANES), an exact multiple, and an exact-multiple +
|
||||
/// remainder. This exercises every branch of the chunked reduction.
|
||||
fn cases() -> Vec<Vec<f64>> {
|
||||
let big: Vec<f64> = (0..1000).map(|i| (i as f64) * 0.5 - 123.0).collect();
|
||||
vec![
|
||||
vec![],
|
||||
vec![42.0],
|
||||
vec![1.0, 2.0, 3.0], // < LANES
|
||||
(1..=8).map(|i| i as f64).collect(), // exactly LANES
|
||||
(1..=17).map(|i| i as f64).collect(), // LANES*2 + 1
|
||||
big,
|
||||
]
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sum_matches_sequential_within_tolerance() {
|
||||
for data in cases() {
|
||||
let got = sum(&data);
|
||||
let want = naive_sum(&data);
|
||||
assert!(
|
||||
(got - want).abs() <= 1e-9 * want.abs().max(1.0),
|
||||
"sum mismatch: got {got}, want {want}, len {}",
|
||||
data.len()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn wma_seed_matches_sequential_within_tolerance() {
|
||||
for data in cases() {
|
||||
let (t, s) = wma_seed(&data);
|
||||
let (wt, ws) = naive_wma_seed(&data);
|
||||
assert!(
|
||||
(t - wt).abs() <= 1e-9 * wt.abs().max(1.0),
|
||||
"wma t mismatch: got {t}, want {wt}, len {}",
|
||||
data.len()
|
||||
);
|
||||
assert!(
|
||||
(s - ws).abs() <= 1e-9 * ws.abs().max(1.0),
|
||||
"wma s mismatch: got {s}, want {ws}, len {}",
|
||||
data.len()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sum_empty_is_zero() {
|
||||
assert_eq!(sum(&[]), 0.0);
|
||||
}
|
||||
}
|
||||
@@ -449,6 +449,36 @@ mod tests {
|
||||
assert!((d1 - d2).abs() < 1e-12);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dtw_nan_in_input_propagates() {
|
||||
// NaN in either input must propagate to the distance (IEEE 754 semantics).
|
||||
let a = vec![1.0, 2.0, f64::NAN, 4.0];
|
||||
let b = vec![1.0, 2.0, 3.0, 4.0];
|
||||
assert!(dtw_distance(&a, &b, None).is_nan());
|
||||
assert!(dtw_distance(&b, &a, None).is_nan());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dtw_is_symmetric() {
|
||||
let a = vec![1.0, 4.0, 2.0, 8.0, 3.0, 6.0, 5.0];
|
||||
let b = vec![2.0, 3.0, 7.0, 4.0, 5.0, 1.0, 9.0];
|
||||
let d_ab = dtw_distance(&a, &b, None);
|
||||
let d_ba = dtw_distance(&b, &a, None);
|
||||
assert!((d_ab - d_ba).abs() < 1e-12);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dtw_path_length_bounded() {
|
||||
// A valid warp path has length between max(n, m) and n + m - 1.
|
||||
let a: Vec<f64> = (0..7).map(|x| x as f64).collect();
|
||||
let b: Vec<f64> = (0..10).map(|x| (x as f64).sin()).collect();
|
||||
let (_, path) = dtw_path(&a, &b, None);
|
||||
let n = a.len();
|
||||
let m = b.len();
|
||||
assert!(path.len() >= n.max(m));
|
||||
assert!(path.len() <= n + m - 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dtw_window_constrained_ge_unconstrained() {
|
||||
// window convention matches dtaidistance: Some(w) means |i-j| < w.
|
||||
|
||||
Reference in New Issue
Block a user