From 288b1546b236fd91b7a5e0cce2b80a2680bdb25f Mon Sep 17 00:00:00 2001 From: Pratik Bhadane Date: Mon, 29 Jun 2026 18:21:22 +0530 Subject: [PATCH] =?UTF-8?q?feat:=20broaden=20CPU=20and=20platform=20covera?= =?UTF-8?q?ge=20across=20PyPI,=20nodes,=20and=20crate=E2=80=A6=20(#26)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: broaden CPU and platform coverage across PyPI, nodes, and crates.io Replace static SIMD with runtime CPU-feature dispatch and expand the release wheel matrix so one set of artifacts runs on any target CPU and platform without illegal-instruction crashes. Rust core: - Add multiversion runtime dispatch (crates/ferro_ta_core/src/simd.rs); drop compile-time `wide`. `simd` feature is now default-on and forwarded through the pyo3 crate, and stays compatible with #![forbid(unsafe_code)]. Packaging: - abi3-py310: one cp310-abi3 wheel per platform (covers CPython 3.10+). - CI matrix adds Linux aarch64 + musllinux (x86_64/aarch64) and Windows arm64. Node/Docker + docs: - api/Dockerfile: document baseline+dispatch (no target-cpu pin) and add a fail-fast import check; aarch64 containers now install cleanly. - Rewrite docs/guides/simd.md; fix stale `wide` mention in ADR 0003. - Add ADR 0006 (CPU coverage strategy). Also bundles in-flight release prep already staged in the tree (DTW exception types, SBOM/provenance security, supporting docs). * fix(ci): clear cargo-deny and pip-audit failures; apply dependency bumps cargo-deny (advisories): - Ignore pyo3 RUSTSEC-2026-0176 / RUSTSEC-2026-0177 in deny.toml with a documented rationale: ferro-ta uses neither affected code path (PyList/PyTuple nth iterators; PyCFunction::new_closure). Upstream fix needs pyo3 >=0.29 (large API migration), tracked as a follow-up. pip-audit: - Bump dev lockfile idna 3.18, pytest 9.1.1, urllib3 2.7.0 to clear PYSEC-2026-215, CVE-2025-71176, PYSEC-2026-141/142. Dependency bumps (supersede open dependabot PRs; they auto-close on merge): - cargo: log 0.4.32, serde_json 1.0.150, rayon 1.12.0 - api/requirements.txt: uvicorn>=0.49.0, pydantic>=2.13.4, ferro-ta>=1.1.4 - CI actions: deploy-pages v5, upload-pages-artifact v5, action-gh-release v3 The open `wide` 1.5.0 bump (PR #24) is obsolete — the crate is removed in this branch. * chore: address CodeRabbit review; remove docs/adr section CodeRabbit findings: - CI sbom job: add `attestations: write` so attest-build-provenance can run (it had only contents:write + id-token:write). - simd.rs: vectorize `wma_seed` with lane-local accumulators — it was scalar behind the multiversion wrapper, adding dispatch overhead for no SIMD gain. - CHANGELOG: consolidate the duplicate `### Changed` heading. - python/ferro_ta/__init__.py: also re-export the `FerroTaError` alias. - docs/guides/dtw.md: soften "byte-for-byte" parity to within-tolerance. Remove docs/adr/ at maintainer request and clean up the ADR links in the SIMD and DTW guides. The ADR files remain in commit 9506a30 if ever needed. --- .github/dependabot.yml | 11 ++ .github/workflows/CI.yml | 152 +++++++++++++++++------- .github/workflows/ci-docs.yml | 2 +- .github/workflows/nightly-bench.yml | 100 ++++++++++++++++ .github/workflows/release.yml | 2 +- CHANGELOG.md | 68 +++++++++++ Cargo.lock | 67 ++++++----- Cargo.toml | 14 ++- api/Dockerfile | 29 +++-- api/requirements.txt | 6 +- benchmarks/bench_simd.py | 5 +- crates/ferro_ta_core/Cargo.toml | 12 +- crates/ferro_ta_core/src/lib.rs | 4 + crates/ferro_ta_core/src/overlap.rs | 68 ++--------- crates/ferro_ta_core/src/simd.rs | 161 ++++++++++++++++++++++++++ crates/ferro_ta_core/src/statistic.rs | 30 +++++ deny.toml | 14 ++- docs/guides/dtw.md | 80 +++++++++++++ docs/guides/simd.md | 92 +++++++++++++++ pyproject.toml | 2 +- python/ferro_ta/__init__.py | 6 + python/ferro_ta/core/exceptions.py | 65 ++++++++++- uv.lock | 18 +-- 23 files changed, 839 insertions(+), 169 deletions(-) create mode 100644 .github/workflows/nightly-bench.yml create mode 100644 crates/ferro_ta_core/src/simd.rs create mode 100644 docs/guides/dtw.md create mode 100644 docs/guides/simd.md diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 3117a57..363716c 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -34,3 +34,14 @@ updates: labels: - "dependencies" - "github-actions" + + # WASM JavaScript package + - package-ecosystem: "npm" + directory: "/wasm" + schedule: + interval: "weekly" + day: "monday" + open-pull-requests-limit: 5 + labels: + - "dependencies" + - "javascript" diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index d4788e5..f8b132f 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -87,7 +87,7 @@ jobs: steps: - name: Deploy to GitHub Pages id: deployment - uses: actions/deploy-pages@v4 + uses: actions/deploy-pages@v5 # ------------------------------------------------------------------------- # CI gate — all required jobs must pass before this job succeeds. @@ -136,88 +136,123 @@ jobs: # Keep release jobs here because trusted-publisher configuration points to # CI.yml specifically. # ------------------------------------------------------------------------- + # One abi3 wheel per (platform, arch). abi3-py310 means a single wheel + # covers CPython 3.10+ on each target, so there is no python-version axis. + # extension-module + abi3 link no libpython, which is what lets the linux + # jobs cross-compile aarch64/musl from an x86_64 runner via maturin-action. + build-wheels-linux: - name: Build wheels (linux / py${{ matrix.python-version }}) + name: Build wheels (linux-gnu / ${{ matrix.target }}) runs-on: ubuntu-latest if: (github.event_name == 'release' && github.event.action == 'published') || (github.event_name == 'workflow_dispatch' && inputs.release) strategy: fail-fast: false matrix: - python-version: ["3.10", "3.11", "3.12", "3.13"] + target: [x86_64, aarch64] steps: - uses: actions/checkout@v6 - - name: Build wheel + - name: Build abi3 manylinux wheel uses: PyO3/maturin-action@v1 with: command: build - args: --release --out dist --compatibility pypi -i python${{ matrix.python-version }} + target: ${{ matrix.target }} + args: --release --out dist manylinux: "2_17" - name: Upload wheels as artifact uses: actions/upload-artifact@v7 with: - name: wheels-linux-py${{ matrix.python-version }} + name: wheels-linux-gnu-${{ matrix.target }} + path: dist/*.whl + + build-wheels-musllinux: + name: Build wheels (linux-musl / ${{ matrix.target }}) + runs-on: ubuntu-latest + if: (github.event_name == 'release' && github.event.action == 'published') || (github.event_name == 'workflow_dispatch' && inputs.release) + strategy: + fail-fast: false + matrix: + target: [x86_64, aarch64] + + steps: + - uses: actions/checkout@v6 + + - name: Build abi3 musllinux wheel + uses: PyO3/maturin-action@v1 + with: + command: build + target: ${{ matrix.target }} + args: --release --out dist + manylinux: musllinux_1_2 + + - name: Upload wheels as artifact + uses: actions/upload-artifact@v7 + with: + name: wheels-linux-musl-${{ matrix.target }} path: dist/*.whl build-wheels-macos: - name: Build wheels (macos / py${{ matrix.python-version }}) + name: Build wheels (macos / universal2) runs-on: macos-latest if: (github.event_name == 'release' && github.event.action == 'published') || (github.event_name == 'workflow_dispatch' && inputs.release) - strategy: - fail-fast: false - matrix: - python-version: ["3.10", "3.11", "3.12", "3.13"] - steps: - uses: actions/checkout@v6 - - name: Set up Python ${{ matrix.python-version }} + - name: Set up Python uses: actions/setup-python@v6 with: - python-version: ${{ matrix.python-version }} + python-version: "3.10" - - name: Build universal2 wheel + - name: Build universal2 abi3 wheel uses: PyO3/maturin-action@v1 with: command: build - args: --release --out dist --compatibility pypi -i python target: universal2-apple-darwin + args: --release --out dist - name: Upload wheels as artifact uses: actions/upload-artifact@v7 with: - name: wheels-macos-py${{ matrix.python-version }} + name: wheels-macos-universal2 path: dist/*.whl build-wheels-windows: - name: Build wheels (windows / py${{ matrix.python-version }}) - runs-on: windows-latest + name: Build wheels (windows / ${{ matrix.platform.arch }}) + runs-on: ${{ matrix.platform.runner }} if: (github.event_name == 'release' && github.event.action == 'published') || (github.event_name == 'workflow_dispatch' && inputs.release) strategy: fail-fast: false matrix: - python-version: ["3.10", "3.11", "3.12", "3.13"] + platform: + - runner: windows-latest + arch: x64 + target: x64 + - runner: windows-11-arm + arch: arm64 + target: aarch64 steps: - uses: actions/checkout@v6 - - name: Set up Python ${{ matrix.python-version }} + - name: Set up Python uses: actions/setup-python@v6 with: - python-version: ${{ matrix.python-version }} + python-version: "3.11" + architecture: ${{ matrix.platform.arch }} - - name: Build wheel + - name: Build abi3 wheel uses: PyO3/maturin-action@v1 with: command: build - args: --release --out dist --compatibility pypi -i python + target: ${{ matrix.platform.target }} + args: --release --out dist - name: Upload wheels as artifact uses: actions/upload-artifact@v7 with: - name: wheels-windows-py${{ matrix.python-version }} + name: wheels-windows-${{ matrix.platform.arch }} path: dist/*.whl build-sdist: @@ -247,6 +282,7 @@ jobs: runs-on: ubuntu-latest needs: - build-wheels-linux + - build-wheels-musllinux - build-wheels-macos - build-wheels-windows - build-sdist @@ -256,6 +292,8 @@ jobs: url: https://pypi.org/p/ferro-ta permissions: id-token: write + attestations: write + contents: read steps: - name: Download all wheels uses: actions/download-artifact@v8 @@ -270,6 +308,11 @@ jobs: name: sdist path: dist + - name: Generate SLSA build provenance attestations + uses: actions/attest-build-provenance@v2 + with: + subject-path: "dist/*" + - name: Verify distribution coverage run: | python3 - <<'PY' @@ -283,19 +326,15 @@ jobs: for name in files: print(f" - {name}") + # One abi3 wheel (cp310-abi3) per platform/arch — covers CPython 3.10+. expected = [ - "ferro_ta-*-cp310-cp310-manylinux*_x86_64.whl", - "ferro_ta-*-cp311-cp311-manylinux*_x86_64.whl", - "ferro_ta-*-cp312-cp312-manylinux*_x86_64.whl", - "ferro_ta-*-cp313-cp313-manylinux*_x86_64.whl", - "ferro_ta-*-cp310-cp310-win_amd64.whl", - "ferro_ta-*-cp311-cp311-win_amd64.whl", - "ferro_ta-*-cp312-cp312-win_amd64.whl", - "ferro_ta-*-cp313-cp313-win_amd64.whl", - "ferro_ta-*-cp310-cp310-macosx*_universal2.whl", - "ferro_ta-*-cp311-cp311-macosx*_universal2.whl", - "ferro_ta-*-cp312-cp312-macosx*_universal2.whl", - "ferro_ta-*-cp313-cp313-macosx*_universal2.whl", + "ferro_ta-*-cp310-abi3-manylinux*_x86_64.whl", + "ferro_ta-*-cp310-abi3-manylinux*_aarch64.whl", + "ferro_ta-*-cp310-abi3-musllinux*_x86_64.whl", + "ferro_ta-*-cp310-abi3-musllinux*_aarch64.whl", + "ferro_ta-*-cp310-abi3-macosx*_universal2.whl", + "ferro_ta-*-cp310-abi3-win_amd64.whl", + "ferro_ta-*-cp310-abi3-win_arm64.whl", "ferro_ta-*.tar.gz", ] @@ -346,6 +385,7 @@ jobs: permissions: contents: write id-token: write + attestations: write steps: - uses: actions/checkout@v6 @@ -380,12 +420,40 @@ jobs: - name: Generate Rust SBOM (CycloneDX) run: cargo cyclonedx --format json --override-filename ferro-ta-rust-sbom.cdx - - name: Upload Python SBOM to release - uses: softprops/action-gh-release@v2 + - name: Validate SBOMs (schema check) + run: | + python3 -c "import json, sys; json.load(open('ferro-ta-python-sbom.spdx.json')); print('Python SBOM valid JSON')" + python3 -c "import json, sys; data=json.load(open('ferro-ta-rust-sbom.cdx.json')); assert data.get('bomFormat')=='CycloneDX', 'not CycloneDX'; print('Rust SBOM valid CycloneDX')" + + - name: Install cosign + uses: sigstore/cosign-installer@v3 + + - name: Sign SBOMs with cosign (keyless) + env: + COSIGN_EXPERIMENTAL: "1" + run: | + cosign sign-blob --yes ferro-ta-python-sbom.spdx.json --output-signature ferro-ta-python-sbom.spdx.json.sig --output-certificate ferro-ta-python-sbom.spdx.json.pem + cosign sign-blob --yes ferro-ta-rust-sbom.cdx.json --output-signature ferro-ta-rust-sbom.cdx.json.sig --output-certificate ferro-ta-rust-sbom.cdx.json.pem + + - name: Attest SBOM provenance + uses: actions/attest-build-provenance@v2 with: - files: ferro-ta-python-sbom.spdx.json + subject-path: | + ferro-ta-python-sbom.spdx.json + ferro-ta-rust-sbom.cdx.json + + - name: Upload Python SBOM to release + uses: softprops/action-gh-release@v3 + with: + files: | + ferro-ta-python-sbom.spdx.json + ferro-ta-python-sbom.spdx.json.sig + ferro-ta-python-sbom.spdx.json.pem - name: Upload Rust SBOM to release - uses: softprops/action-gh-release@v2 + uses: softprops/action-gh-release@v3 with: - files: ferro-ta-rust-sbom.cdx.json + files: | + ferro-ta-rust-sbom.cdx.json + ferro-ta-rust-sbom.cdx.json.sig + ferro-ta-rust-sbom.cdx.json.pem diff --git a/.github/workflows/ci-docs.yml b/.github/workflows/ci-docs.yml index f662568..b85a386 100644 --- a/.github/workflows/ci-docs.yml +++ b/.github/workflows/ci-docs.yml @@ -44,6 +44,6 @@ jobs: - name: Upload GitHub Pages artifact if: ${{ inputs.upload-pages-artifact }} - uses: actions/upload-pages-artifact@v4 + uses: actions/upload-pages-artifact@v5 with: path: docs/_build/ diff --git a/.github/workflows/nightly-bench.yml b/.github/workflows/nightly-bench.yml new file mode 100644 index 0000000..b18d0d1 --- /dev/null +++ b/.github/workflows/nightly-bench.yml @@ -0,0 +1,100 @@ +name: Nightly benchmarks + +on: + schedule: + # 03:15 UTC every day — off peak, avoids colliding with the regular CI surge. + - cron: "15 3 * * *" + workflow_dispatch: + +permissions: + contents: read + issues: write + +jobs: + bench: + name: Run perf contract and regression checks + runs-on: ubuntu-latest + timeout-minutes: 60 + env: + RUSTFLAGS: "" # override target-cpu=native from .cargo/config.toml + steps: + - uses: actions/checkout@v6 + + - uses: dtolnay/rust-toolchain@v1 + with: + toolchain: stable + + - uses: Swatinem/rust-cache@v2 + + - uses: actions/setup-python@v6 + with: + python-version: "3.12" + + - name: Install uv + run: pip install uv + + - name: Build ferro-ta wheel + run: | + uv sync --extra dev + uv run maturin build --release --out dist + uv run pip install --force-reinstall dist/*.whl + + - name: Run vs-TA-Lib benchmark + run: uv run python benchmarks/bench_vs_talib.py --output benchmarks/artifacts/nightly_vs_talib.json + + - name: Run SIMD benchmark matrix + run: uv run python benchmarks/bench_simd.py --output benchmarks/artifacts/nightly_simd.json + + - name: Check hotspot regression + id: hotspot + continue-on-error: true + run: uv run python benchmarks/check_hotspot_regression.py --tolerance 0.05 + + - name: Check vs-TA-Lib regression + id: vs_talib + continue-on-error: true + run: uv run python benchmarks/check_vs_talib_regression.py --tolerance 0.05 + + - name: Run criterion bench (build only) + run: cargo bench -p ferro_ta_core --no-run + + - name: Upload nightly artifacts + uses: actions/upload-artifact@v7 + if: always() + with: + name: nightly-bench-${{ github.run_id }} + path: benchmarks/artifacts/ + retention-days: 30 + + - name: Open issue on regression + if: steps.hotspot.outcome == 'failure' || steps.vs_talib.outcome == 'failure' + uses: actions/github-script@v8 + env: + HOTSPOT_OUTCOME: ${{ steps.hotspot.outcome }} + VS_TALIB_OUTCOME: ${{ steps.vs_talib.outcome }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + with: + script: | + const hotspot = process.env.HOTSPOT_OUTCOME; + const vsTalib = process.env.VS_TALIB_OUTCOME; + const runUrl = process.env.RUN_URL; + const failed = []; + if (hotspot === 'failure') failed.push('hotspot'); + if (vsTalib === 'failure') failed.push('vs-TA-Lib'); + const title = `Nightly benchmark regression: ${failed.join(' + ')}`; + const body = [ + `The nightly benchmark workflow reported a >5% regression in: **${failed.join(', ')}**.`, + '', + `Run: ${runUrl}`, + '', + 'Artifacts with the raw JSON are attached to the run above.', + 'If this is a known regression, update the baseline JSON and close.', + 'If it is unexpected, investigate the last commit on `main` before the nightly ran.', + ].join('\n'); + await github.rest.issues.create({ + owner: context.repo.owner, + repo: context.repo.repo, + title, + body, + labels: ['performance', 'regression', 'nightly'], + }); diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5e9026b..d4ffd5a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -52,7 +52,7 @@ jobs: PY - name: Create GitHub Release - uses: softprops/action-gh-release@v2 + uses: softprops/action-gh-release@v3 with: tag_name: ${{ github.ref_name }} name: v${{ steps.version.outputs.version }} diff --git a/CHANGELOG.md b/CHANGELOG.md index 2b4c1ba..4b7e215 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,74 @@ and the project uses [Semantic Versioning](https://semver.org/). ## [Unreleased] +### Added + +- **Runtime CPU-feature dispatch** for SIMD hot paths via the `multiversion` + crate (`ferro_ta_core/src/simd.rs`). One binary now selects baseline / + AVX2-FMA / AVX-512 / NEON kernels at load time via CPUID instead of being + pinned at build time, so it runs on any CPU of the target architecture with + no illegal-instruction crashes on older chips. The `simd` feature is on by + default; `--no-default-features` gives a pure-scalar build. +- **Broader wheel coverage**: release builds now also produce Linux `aarch64` + (manylinux) and `musllinux` (x86_64 + aarch64) wheels and a Windows `arm64` + wheel, alongside the existing Linux x86_64, macOS universal2, and Windows + x64 wheels. +- **abi3 wheels** (`cp310-abi3`): a single stable-ABI wheel per platform now + covers CPython 3.10+ (including future 3.14+), replacing the per-version + wheel matrix. +- **Dynamic Time Warping** (`DTW`, `DTW_DISTANCE`, `BATCH_DTW`): Euclidean-cost + DTW with optional Sakoe-Chiba band (`window=` parameter). `DTW()` returns + `(distance, path)` with the optimal warping path as an `(N, 2)` index array; + `DTW_DISTANCE()` is the faster distance-only variant; `BATCH_DTW()` computes + distances from each row of a 2-D matrix to a reference series in parallel + via rayon. Distance convention matches `dtaidistance.dtw.distance()`. +- **Indicator-specific exception types** (`FerroTaError`, `InvalidPeriodError`, + `InsufficientDataError`, `LengthMismatchError`, `NumericConvergenceError`, + `InvalidInputError`): finer-grained errors for catching specific failure + modes. All subclass `ValueError` so existing `except ValueError` code keeps + working (backward compatible). + +### Changed + +- **SIMD is now enabled by default and runtime-dispatched.** Replaced the + compile-time `wide` crate (which was never actually enabled in published + wheels) with `multiversion`. The `wide` Cargo feature is removed; use the + default `simd` feature instead. +- **Dependency bumps.** Rust: `log` 0.4.32, `serde_json` 1.0.150, + `rayon` 1.12.0. API service (`api/requirements.txt`): `uvicorn>=0.49.0`, + `pydantic>=2.13.4`, `ferro-ta>=1.1.4`. CI actions: `actions/deploy-pages` v5, + `actions/upload-pages-artifact` v5, `softprops/action-gh-release` v3. +- Python coverage threshold raised from 65% to 80% and enforced in CI. + +### Fixed + +- **aarch64 Linux containers can now install ferro-ta.** Previously no Linux + `aarch64` wheel was published, so arm64 images (e.g. AWS Graviton) fell back + to an sdist build that failed without a Rust toolchain. A manylinux/musllinux + aarch64 wheel is now published. +- Published wheels now actually ship SIMD-accelerated kernels; the prior build + enabled no SIMD feature at all. + +### Security + +- **SLSA build provenance** attestations are now generated for every PyPI + wheel and sdist via `actions/attest-build-provenance`. Verify with + `gh attestation verify `. +- **Sigstore keyless signatures** are now published alongside both + CycloneDX/SPDX SBOMs on every GitHub Release (`.sig` + `.pem` files). +- `ferro_ta_core` crate now declares `#![forbid(unsafe_code)]` to prevent + regression — the pure-logic layer has no unsafe code and never will. +- Dependabot now covers the WASM npm package in addition to pip, cargo, + and GitHub Actions. +- **pip-audit fixes**: bumped dev lockfile deps `idna` 3.18, `pytest` 9.1.1, + and `urllib3` 2.7.0 to clear PYSEC-2026-215, CVE-2025-71176, and + PYSEC-2026-141/142. +- **pyo3 advisories triaged**: RUSTSEC-2026-0176 and RUSTSEC-2026-0177 are + ignored in `deny.toml` with rationale — ferro-ta uses neither affected code + path (PyList/PyTuple `nth` iterators; `PyCFunction::new_closure`). The + upstream fix requires pyo3 >=0.29 (a large API migration), tracked for a + follow-up. + ## [1.1.3] — 2026-04-02 ### Added diff --git a/Cargo.lock b/Cargo.lock index 1dd0dfd..279f3c9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -53,12 +53,6 @@ version = "3.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" -[[package]] -name = "bytemuck" -version = "1.25.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8efb64bd706a16a1bdde310ae86b351e4d21550d98d056f22f8a7f7a2183fec" - [[package]] name = "cast" version = "0.3.0" @@ -225,9 +219,9 @@ name = "ferro_ta_core" version = "1.1.4" dependencies = [ "criterion", + "multiversion", "serde", "serde_json", - "wide", ] [[package]] @@ -295,9 +289,9 @@ checksum = "48f5d2a454e16a5ea0f4ced81bd44e4cfc7bd3a507b61887c99fd3538b28e4af" [[package]] name = "log" -version = "0.4.29" +version = "0.4.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" +checksum = "953f07c43838f8e6f9758cab68bf5bed85465e7587ebe0b823f1bcd81978ad3a" [[package]] name = "matrixmultiply" @@ -324,6 +318,28 @@ dependencies = [ "autocfg", ] +[[package]] +name = "multiversion" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7edb7f0ff51249dfda9ab96b5823695e15a052dc15074c9dbf3d118afaf2c201" +dependencies = [ + "multiversion-macros", + "target-features", +] + +[[package]] +name = "multiversion-macros" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b093064383341eb3271f42e381cb8f10a01459478446953953c75d24bd339fc0" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "target-features", +] + [[package]] name = "ndarray" version = "0.16.1" @@ -546,9 +562,9 @@ checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3" [[package]] name = "rayon" -version = "1.11.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "368f01d005bf8fd9b1206fb6fa653e6c4a81ceb1466406b81792d87c5677a58f" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" dependencies = [ "either", "rayon-core", @@ -605,15 +621,6 @@ version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" -[[package]] -name = "safe_arch" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f7caad094bd561859bcd467734a720c3c1f5d1f338995351fefe2190c45efed" -dependencies = [ - "bytemuck", -] - [[package]] name = "same-file" version = "1.0.6" @@ -655,9 +662,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.149" +version = "1.0.150" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" dependencies = [ "itoa", "memchr", @@ -689,6 +696,12 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "609409d472a0a7d8d4dd9e19891bbdef546b9dce670c3057d0e02192dc541226" +[[package]] +name = "target-features" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1bbb9f3c5c463a01705937a24fdabc5047929ac764b2d5b9cf681c1f5041ed5" + [[package]] name = "target-lexicon" version = "0.13.5" @@ -782,16 +795,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "wide" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "198f6abc41fab83526d10880fa5c17e2b4ee44e763949b4bb34e2fd1e8ca48e4" -dependencies = [ - "bytemuck", - "safe_arch", -] - [[package]] name = "winapi" version = "0.3.9" diff --git a/Cargo.toml b/Cargo.toml index f0c2b45..fe3ef75 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,7 +22,9 @@ name = "ferro_ta" crate-type = ["cdylib"] [dependencies] -pyo3 = { version = "0.25", features = ["extension-module"] } +# abi3-py310: build a single stable-ABI wheel that runs on CPython 3.10+ +# (including future 3.14+), instead of one wheel per minor version. +pyo3 = { version = "0.25", features = ["extension-module", "abi3-py310"] } ta = "0.5.0" numpy = "0.25" # Must be < 0.17 while numpy 0.25 is used (numpy's IntoPyArray is for its own ndarray only). @@ -30,7 +32,11 @@ ndarray = "0.16" rayon = "1.10" log = "0.4" pyo3-log = "0.12" -ferro_ta_core = { path = "crates/ferro_ta_core", version = "1.1.4", features = ["serde"] } +# default-features = false so the `simd` toggle is forwarded explicitly via +# this crate's own `simd` feature (below). Without this, core's default `simd` +# would always be on and `--no-default-features` could never produce a true +# pure-scalar build (used by the SIMD benchmark baseline). +ferro_ta_core = { path = "crates/ferro_ta_core", version = "1.1.4", default-features = false, features = ["serde"] } [dev-dependencies] criterion = { version = "0.8", features = ["html_reports"] } @@ -40,5 +46,7 @@ lto = true codegen-units = 1 [features] -default = [] +# SIMD runtime dispatch ON by default → published wheels ship the adaptive +# fast path with no extra flags. `--no-default-features` yields pure scalar. +default = ["simd"] simd = ["ferro_ta_core/simd"] diff --git a/api/Dockerfile b/api/Dockerfile index 490576b..5443bb9 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -8,21 +8,36 @@ # # Environment variables (override at runtime): # MAX_SERIES_LENGTH=100000 # maximum data-point count per request +# +# CPU portability +# --------------- +# This image installs the PRE-BUILT ferro-ta wheel from PyPI — we do NOT +# recompile from sdist with `RUSTFLAGS=-C target-cpu=...`. The wheel is built +# at the manylinux baseline (x86-64-v1) and selects AVX2/AVX-512/NEON kernels +# at RUNTIME via CPU dispatch. One image therefore runs on any node — old or +# new CPU, x86_64 or arm64 — with no illegal-instruction (SIGILL) crashes. +# Pinning a target-cpu would be faster on a uniform fleet but would crash on +# any older/heterogeneous node, which is the opposite of broad coverage. +# +# Build this image for whichever arch your nodes use: +# docker build --platform linux/amd64 -t ferro-ta-api . +# docker build --platform linux/arm64 -t ferro-ta-api . # Graviton/Ampere +# Both resolve a matching manylinux wheel — no Rust toolchain needed here. FROM python:3.11-slim WORKDIR /app -# Install system dependencies required to build ferro_ta (Rust is pre-compiled -# into the wheel, so only pip + wheel tooling is needed at runtime). -RUN apt-get update && apt-get install -y --no-install-recommends \ - gcc \ - && rm -rf /var/lib/apt/lists/* - -# Copy and install dependencies first (cache layer) +# Copy and install dependencies first (cache layer). No compiler is needed: +# ferro-ta, numpy, and pydantic-core all ship prebuilt wheels for linux +# x86_64 and aarch64. COPY requirements.txt ./ RUN pip install --no-cache-dir -r requirements.txt +# Fail the build immediately if the wheel did not resolve for this arch +# (e.g. an exotic platform that fell back to an sdist build without Rust). +RUN python -c "import ferro_ta, numpy as np; ferro_ta.SMA(np.arange(10.0), 3); print('ferro_ta', ferro_ta.__version__, 'import OK')" + # Copy API source COPY main.py ./ diff --git a/api/requirements.txt b/api/requirements.txt index 345001f..bb102cd 100644 --- a/api/requirements.txt +++ b/api/requirements.txt @@ -1,6 +1,6 @@ # Runtime dependencies for ferro-ta API -ferro_ta>=1.0.0 +ferro_ta>=1.1.4 fastapi>=0.110.0 -uvicorn[standard]>=0.27.0 -pydantic>=2.0.0 +uvicorn[standard]>=0.49.0 +pydantic>=2.13.4 numpy>=1.20 diff --git a/benchmarks/bench_simd.py b/benchmarks/bench_simd.py index 13340d5..d2e4896 100644 --- a/benchmarks/bench_simd.py +++ b/benchmarks/bench_simd.py @@ -55,8 +55,11 @@ def run_simd_benchmark( iv_bars: int = 50_000, window: int = 252, ) -> dict[str, Any]: + # `simd` is a default feature, so a pure-scalar baseline must explicitly + # opt out via --no-default-features; otherwise both builds would be + # identical and every reported speedup would collapse to 1.0. variants = [ - ("portable_release", []), + ("portable_release", ["--no-default-features"]), ("simd_release", ["--features", "simd"]), ] reports = { diff --git a/crates/ferro_ta_core/Cargo.toml b/crates/ferro_ta_core/Cargo.toml index afe4370..ca8d5bd 100644 --- a/crates/ferro_ta_core/Cargo.toml +++ b/crates/ferro_ta_core/Cargo.toml @@ -16,7 +16,7 @@ name = "ferro_ta_core" crate-type = ["lib"] [dependencies] -wide = { version = "1.1.1", optional = true } +multiversion = { version = "0.8", optional = true } serde = { version = "1.0", features = ["derive"], optional = true } serde_json = { version = "1.0", optional = true } @@ -28,6 +28,12 @@ name = "indicators" harness = false [features] -wide = ["dep:wide"] -simd = ["wide"] +# Runtime CPU-feature dispatch (multiversion). Default ON so `cargo add +# ferro_ta_core` and the published wheels get SIMD-accelerated reductions +# that adapt to the running CPU (baseline .. AVX-512 / NEON) WITHOUT pinning +# a target-cpu — one binary runs on any CPU of the target arch, with no +# illegal-instruction crashes on older chips. Disable with +# `--no-default-features` for a pure-scalar build. +default = ["simd"] +simd = ["dep:multiversion"] serde = ["dep:serde", "dep:serde_json"] diff --git a/crates/ferro_ta_core/src/lib.rs b/crates/ferro_ta_core/src/lib.rs index b5ddd96..2de6ea3 100644 --- a/crates/ferro_ta_core/src/lib.rs +++ b/crates/ferro_ta_core/src/lib.rs @@ -1,3 +1,5 @@ +#![forbid(unsafe_code)] + /*! ferro_ta_core — Pure Rust indicator library. @@ -49,6 +51,8 @@ pub mod price_transform; pub mod regime; pub mod resampling; pub mod signals; +/// Runtime-dispatched SIMD reduction primitives (internal). +pub(crate) mod simd; pub mod statistic; pub mod streaming; pub mod volatility; diff --git a/crates/ferro_ta_core/src/overlap.rs b/crates/ferro_ta_core/src/overlap.rs index 9404b0a..1c203de 100644 --- a/crates/ferro_ta_core/src/overlap.rs +++ b/crates/ferro_ta_core/src/overlap.rs @@ -38,27 +38,10 @@ pub fn sma_into(src: &[f64], timeperiod: usize, dest: &mut [f64], dest_offset: u return; } - #[cfg(feature = "simd")] - let window_sum_init = { - use wide::f64x4; - let p_data = &src[..timeperiod]; - let mut sum = f64x4::splat(0.0); - let mut chunks = p_data.chunks_exact(4); - for chunk in &mut chunks { - sum += f64x4::new([chunk[0], chunk[1], chunk[2], chunk[3]]); - } - let arr = sum.to_array(); - let mut total = arr[0] + arr[1] + arr[2] + arr[3]; - for &v in chunks.remainder() { - total += v; - } - total - }; - - #[cfg(not(feature = "simd"))] - let window_sum_init: f64 = src[..timeperiod].iter().sum(); - - let mut window_sum = window_sum_init; + // Seed the rolling window with a runtime-dispatched reduction. The O(n) + // streaming recurrence below is inherently sequential, so SIMD only ever + // applies to this initial window sum. + let mut window_sum = crate::simd::sum(&src[..timeperiod]); let tp_f64 = timeperiod as f64; dest[dest_offset + timeperiod - 1] = window_sum / tp_f64; @@ -124,46 +107,9 @@ pub fn wma(close: &[f64], timeperiod: usize) -> Vec { let denom: f64 = (timeperiod * (timeperiod + 1) / 2) as f64; let p = timeperiod as f64; - // Seed: compute T and S for the first window. - #[cfg(feature = "simd")] - let (mut t, mut s) = { - use wide::f64x4; - let p_data = &close[..timeperiod]; - let mut t_simd = f64x4::splat(0.0); - let mut s_simd = f64x4::splat(0.0); - let mut chunks = p_data.chunks_exact(4); - let mut idx = 1.0; - let step = f64x4::new([0.0, 1.0, 2.0, 3.0]); - - for chunk in &mut chunks { - let vals = f64x4::new([chunk[0], chunk[1], chunk[2], chunk[3]]); - let mults = f64x4::splat(idx) + step; - t_simd += vals * mults; - s_simd += vals; - idx += 4.0; - } - let t_arr = t_simd.to_array(); - let s_arr = s_simd.to_array(); - let mut t = t_arr[0] + t_arr[1] + t_arr[2] + t_arr[3]; - let mut s = s_arr[0] + s_arr[1] + s_arr[2] + s_arr[3]; - for &v in chunks.remainder() { - t += v * idx; - s += v; - idx += 1.0; - } - (t, s) - }; - - #[cfg(not(feature = "simd"))] - let (mut t, mut s) = { - let t_val: f64 = close[..timeperiod] - .iter() - .enumerate() - .map(|(k, &v)| v * (k + 1) as f64) - .sum(); - let s_val: f64 = close[..timeperiod].iter().sum(); - (t_val, s_val) - }; + // Seed: compute T and S for the first window via a runtime-dispatched + // reduction (the streaming recurrence below is sequential). + let (mut t, mut s) = crate::simd::wma_seed(&close[..timeperiod]); result[timeperiod - 1] = t / denom; diff --git a/crates/ferro_ta_core/src/simd.rs b/crates/ferro_ta_core/src/simd.rs new file mode 100644 index 0000000..91f78d2 --- /dev/null +++ b/crates/ferro_ta_core/src/simd.rs @@ -0,0 +1,161 @@ +//! Runtime-dispatched SIMD primitives. +//! +//! Each public reduction here is compiled into several CPU-feature-specific +//! variants (baseline, SSE, AVX2/FMA, AVX-512 on x86_64; NEON on aarch64; …) +//! by [`multiversion`]. The fastest variant the *current* CPU supports is +//! chosen at runtime via CPUID. This gives one binary that: +//! +//! * runs on **any** CPU of the target architecture — no illegal-instruction +//! (SIGILL) crashes on pre-AVX2 chips, unlike a static `-C target-cpu=…`; +//! * still uses wide vector units where the hardware has them. +//! +//! The hot loops accumulate into **independent lanes** before a final +//! horizontal combine. That is what lets the optimizer auto-vectorize them: +//! a plain sequential `iter().sum()` is a dependency chain LLVM may not +//! reorder (doing so would change floating-point rounding). As a consequence +//! these results differ from a strict left-to-right sum by a few ULPs — well +//! inside every indicator's documented tolerance. + +/// Number of independent accumulator lanes. Eight `f64` lanes cover the +/// widest target we dispatch to (AVX-512 = 8×f64); narrower targets (AVX2, +/// NEON) simply use a subset. +#[cfg(feature = "simd")] +const LANES: usize = 8; + +/// Sum of a slice of `f64`, runtime-dispatched. +#[cfg(feature = "simd")] +#[multiversion::multiversion(targets = "simd")] +pub(crate) fn sum(data: &[f64]) -> f64 { + let mut acc = [0.0f64; LANES]; + let mut chunks = data.chunks_exact(LANES); + for chunk in &mut chunks { + for (a, &v) in acc.iter_mut().zip(chunk) { + *a += v; + } + } + let remainder: f64 = chunks.remainder().iter().sum(); + remainder + acc.iter().sum::() +} + +/// Pure-scalar fallback when the `simd` feature is disabled. +#[cfg(not(feature = "simd"))] +pub(crate) fn sum(data: &[f64]) -> f64 { + data.iter().sum() +} + +/// Weighted-moving-average seed for the first window. +/// +/// Returns `(t, s)` where `t = Σ data[k] * (k + 1)` (1-based linear weights) +/// and `s = Σ data[k]`. Used to seed the O(n) WMA recurrence. +#[cfg(feature = "simd")] +#[multiversion::multiversion(targets = "simd")] +pub(crate) fn wma_seed(data: &[f64]) -> (f64, f64) { + // Lane-local accumulation (same idea as `sum`) so each CPU-feature clone + // can vectorize: `t` weights each value by its 1-based global index. + let mut t_acc = [0.0f64; LANES]; + let mut s_acc = [0.0f64; LANES]; + let mut chunks = data.chunks_exact(LANES); + let mut base = 0.0f64; // global index of this chunk's first element + for chunk in &mut chunks { + for (lane, ((t, s), &v)) in t_acc + .iter_mut() + .zip(s_acc.iter_mut()) + .zip(chunk) + .enumerate() + { + *t += v * (base + lane as f64 + 1.0); + *s += v; + } + base += LANES as f64; + } + let mut t = 0.0; + let mut s = 0.0; + for (i, &v) in chunks.remainder().iter().enumerate() { + t += v * (base + i as f64 + 1.0); + s += v; + } + (t + t_acc.iter().sum::(), s + s_acc.iter().sum::()) +} + +/// Pure-scalar fallback when the `simd` feature is disabled. +#[cfg(not(feature = "simd"))] +pub(crate) fn wma_seed(data: &[f64]) -> (f64, f64) { + let mut t = 0.0; + let mut s = 0.0; + for (k, &v) in data.iter().enumerate() { + t += v * (k + 1) as f64; + s += v; + } + (t, s) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Strict sequential reference — the ground truth we compare against. + fn naive_sum(data: &[f64]) -> f64 { + data.iter().sum() + } + + fn naive_wma_seed(data: &[f64]) -> (f64, f64) { + let t = data + .iter() + .enumerate() + .map(|(k, &v)| v * (k + 1) as f64) + .sum(); + let s = data.iter().sum(); + (t, s) + } + + /// Deterministic test vectors spanning the lane boundaries: empty, a + /// partial chunk (< LANES), an exact multiple, and an exact-multiple + + /// remainder. This exercises every branch of the chunked reduction. + fn cases() -> Vec> { + let big: Vec = (0..1000).map(|i| (i as f64) * 0.5 - 123.0).collect(); + vec![ + vec![], + vec![42.0], + vec![1.0, 2.0, 3.0], // < LANES + (1..=8).map(|i| i as f64).collect(), // exactly LANES + (1..=17).map(|i| i as f64).collect(), // LANES*2 + 1 + big, + ] + } + + #[test] + fn sum_matches_sequential_within_tolerance() { + for data in cases() { + let got = sum(&data); + let want = naive_sum(&data); + assert!( + (got - want).abs() <= 1e-9 * want.abs().max(1.0), + "sum mismatch: got {got}, want {want}, len {}", + data.len() + ); + } + } + + #[test] + fn wma_seed_matches_sequential_within_tolerance() { + for data in cases() { + let (t, s) = wma_seed(&data); + let (wt, ws) = naive_wma_seed(&data); + assert!( + (t - wt).abs() <= 1e-9 * wt.abs().max(1.0), + "wma t mismatch: got {t}, want {wt}, len {}", + data.len() + ); + assert!( + (s - ws).abs() <= 1e-9 * ws.abs().max(1.0), + "wma s mismatch: got {s}, want {ws}, len {}", + data.len() + ); + } + } + + #[test] + fn sum_empty_is_zero() { + assert_eq!(sum(&[]), 0.0); + } +} diff --git a/crates/ferro_ta_core/src/statistic.rs b/crates/ferro_ta_core/src/statistic.rs index 9fc2219..3ad0987 100644 --- a/crates/ferro_ta_core/src/statistic.rs +++ b/crates/ferro_ta_core/src/statistic.rs @@ -449,6 +449,36 @@ mod tests { assert!((d1 - d2).abs() < 1e-12); } + #[test] + fn dtw_nan_in_input_propagates() { + // NaN in either input must propagate to the distance (IEEE 754 semantics). + let a = vec![1.0, 2.0, f64::NAN, 4.0]; + let b = vec![1.0, 2.0, 3.0, 4.0]; + assert!(dtw_distance(&a, &b, None).is_nan()); + assert!(dtw_distance(&b, &a, None).is_nan()); + } + + #[test] + fn dtw_is_symmetric() { + let a = vec![1.0, 4.0, 2.0, 8.0, 3.0, 6.0, 5.0]; + let b = vec![2.0, 3.0, 7.0, 4.0, 5.0, 1.0, 9.0]; + let d_ab = dtw_distance(&a, &b, None); + let d_ba = dtw_distance(&b, &a, None); + assert!((d_ab - d_ba).abs() < 1e-12); + } + + #[test] + fn dtw_path_length_bounded() { + // A valid warp path has length between max(n, m) and n + m - 1. + let a: Vec = (0..7).map(|x| x as f64).collect(); + let b: Vec = (0..10).map(|x| (x as f64).sin()).collect(); + let (_, path) = dtw_path(&a, &b, None); + let n = a.len(); + let m = b.len(); + assert!(path.len() >= n.max(m)); + assert!(path.len() <= n + m - 1); + } + #[test] fn dtw_window_constrained_ge_unconstrained() { // window convention matches dtaidistance: Some(w) means |i-j| < w. diff --git a/deny.toml b/deny.toml index 01fbf26..7ef18fb 100644 --- a/deny.toml +++ b/deny.toml @@ -53,7 +53,19 @@ skip = [] db-urls = ["https://github.com/rustsec/advisory-db"] # Deny known security vulnerabilities version = 2 -ignore = [] +ignore = [ + # pyo3 0.25 advisories — fix is pyo3 >=0.29, which is a large API + # migration (IntoPy/ToPyObject were removed in 0.26). Ignored here + # because ferro-ta does NOT use either affected code path: + # * RUSTSEC-2026-0176 — OOB read in PyList/PyTuple nth/nth_back + # iterators: the crate has zero PyList/PyTuple iterator usage. + # * RUSTSEC-2026-0177 — missing Sync bound on + # PyCFunction::new_closure: the crate uses #[pyfunction], never + # new_closure. + # Tracked for removal once the pyo3 0.29 upgrade lands. + "RUSTSEC-2026-0176", + "RUSTSEC-2026-0177", +] # --------------------------------------------------------------------------- # Sources — only allow crates from crates.io and our own path deps diff --git a/docs/guides/dtw.md b/docs/guides/dtw.md new file mode 100644 index 0000000..c7a825f --- /dev/null +++ b/docs/guides/dtw.md @@ -0,0 +1,80 @@ +# Dynamic Time Warping + +ferro-ta ships three DTW entry points. Pick the one that matches your +workload — the distance-only path is measurably faster than the one that +reconstructs the warping path, and `BATCH_DTW` parallelises over rows. + +## Quick reference + +| Function | Returns | When to use | +|---|---|---| +| `DTW_DISTANCE(a, b, window=None)` | `float` | You only need the distance. Fastest. | +| `DTW(a, b, window=None)` | `(float, ndarray[N, 2])` | You need the alignment path for plotting or downstream analysis. | +| `BATCH_DTW(matrix, reference, window=None)` | `ndarray[N]` | You have N candidate series and one reference; uses rayon. | + +## Distance convention + +ferro-ta's DTW uses squared-Euclidean local cost accumulated along the +optimal path, with a single `sqrt()` applied at the end. This matches +`dtaidistance.dtw.distance()` to within floating-point tolerance (parity +tests assert numerical agreement, not bitwise identity). Example: + +```python +>>> import ferro_ta as fta +>>> fta.DTW_DISTANCE([0.0, 1.0, 2.0], [1.0, 2.0, 3.0]) +1.4142135623730951 # == sqrt(2), same as dtaidistance +``` + +If you are migrating from a library that uses absolute-difference local +cost without the final sqrt (e.g. `fastdtw`'s default), your numbers will +not line up. That is a choice ferro-ta made for parity with the +scientific-Python ecosystem. + +## Window constraint (Sakoe-Chiba band) + +Passing `window=w` constrains the DP to cells where `|i - j| < w`. This +turns the O(n·m) cost into O(n·w), which is typically a 5–20× speedup for +realistic `w`. A narrower band can only *increase* the distance, so +`window=` is safe to use whenever your series are roughly aligned. + +```python +# Unconstrained +fta.DTW_DISTANCE(a, b) + +# Constrained: warping may shift up to 5 positions +fta.DTW_DISTANCE(a, b, window=5) +``` + +## Batch usage + +`BATCH_DTW` compares each row of a 2-D matrix against one reference +series, in parallel: + +```python +import numpy as np +import ferro_ta as fta + +reference = np.random.random(500) +candidates = np.random.random((1000, 500)) + +distances = fta.BATCH_DTW(candidates, reference, window=20) +nearest = int(np.argmin(distances)) +``` + +Parallelism is via rayon; no thread-pool configuration is needed on the +Python side. For the sequence lengths ferro-ta targets (thousands of +bars, hundreds to low thousands of candidates), batch-parallel classic +DTW beats FastDTW-style approximations. + +## Edge cases + +- **Empty input:** raises `FerroTAInputError`. +- **NaN in input:** propagates to the output (matches IEEE 754). Call + `ferro_ta.core.exceptions.check_finite()` first if you want to fail + loudly instead. +- **Different-length series:** fully supported. The path array length + is bounded by `max(n, m) <= len(path) <= n + m - 1`. + +## See also + +- `tests/unit/indicators/test_statistic.py` — parity tests against `dtaidistance`. diff --git a/docs/guides/simd.md b/docs/guides/simd.md new file mode 100644 index 0000000..31fd46a --- /dev/null +++ b/docs/guides/simd.md @@ -0,0 +1,92 @@ +# SIMD acceleration + +ferro-ta accelerates hot reductions with **runtime CPU-feature dispatch** +via the [`multiversion`](https://crates.io/crates/multiversion) crate. Each +dispatched function is compiled into several variants — baseline, SSE, +AVX2/FMA, AVX-512 on x86_64; NEON on aarch64 — and the fastest one the +**current** CPU supports is chosen at load time via CPUID. + +## Why dispatch instead of `-C target-cpu` + +A static `RUSTFLAGS=-C target-cpu=x86-64-v3` build *requires* AVX2 on the +running CPU; on an older chip it crashes with an illegal instruction +(SIGILL). Runtime dispatch instead ships every code path in one binary and +picks at runtime, so a single artifact: + +- runs on **any** CPU of the target architecture (no SIGILL on pre-AVX2 + hardware), and +- still uses wide vector units where the hardware has them. + +That property is what lets the **same** wheel / Docker image / crate run +across a heterogeneous fleet. + +## When it helps + +SIMD helps indicators whose inner loop is a reduction over contiguous +`f64` data — e.g. the initial window sum that seeds SMA, the `(T, S)` seed +for WMA, and similar fixed-window reductions. It does **not** help: + +- The O(n) streaming recurrences (`window_sum += new - old`): each step + depends on the previous one, so they are inherently sequential. +- Branchy inner loops (SAR, candlestick patterns). +- Streaming classes (a single-bar update is one or two ops). + +The shared primitives live in `crates/ferro_ta_core/src/simd.rs` +(`sum`, `wma_seed`). They accumulate into independent lanes before a final +horizontal combine — that lane independence is what allows the optimizer to +vectorize each CPU-feature variant. A consequence is that results differ +from a strict left-to-right sum by a few ULPs, well inside every +indicator's documented tolerance. + +## The `simd` feature + +Dispatch is gated behind the `simd` Cargo feature, which is **on by +default**: + +```bash +# default build — runtime dispatch enabled +cargo build -p ferro_ta_core --release + +# pure-scalar build (debugging / baseline benchmarking) +cargo build -p ferro_ta_core --release --no-default-features +``` + +For Python, wheels published to PyPI are built with the default features, +so `pip install ferro-ta` ships the dispatched fast path with no action on +your part. To build a pure-scalar extension from source: + +```bash +maturin develop --release --no-default-features +``` + +## Measured speedups + +The nightly `benchmarks/bench_simd.py` job (see +`.github/workflows/nightly-bench.yml`) builds the extension twice — once +with `--no-default-features` (pure scalar) and once with `--features simd` +(dispatch) — and reports the per-indicator delta. Numbers are regenerated +on every run and vary with hardware; treat any table in a PR as a snapshot, +not a contract. The dispatched kernels here target correctness-preserving +reductions, so gains are modest on the sliding-window indicators and larger +on full-array reductions. + +## Adding a SIMD-optimized indicator + +1. Write and test the **scalar** implementation first — it is the ground + truth. +2. If the hot path is a contiguous `f64` reduction, route it through a + `crate::simd` primitive, or wrap a new helper in + `#[multiversion::multiversion(targets = "simd")]` with the loop body + accumulating into independent lanes. +3. Add a parity test comparing the dispatched result against the strict + scalar reference within tolerance (see `simd.rs` tests for the pattern). +4. Benchmark scalar vs dispatch via `bench_simd.py`. Only keep the SIMD + path if it wins — alignment and tail-handling overhead can make a naive + vectorization *lose* to scalar. + +## See also + +- `crates/ferro_ta_core/src/simd.rs` — dispatched primitives and tests. +- `benches/indicators.rs` — criterion suite. +- `crates/ferro_ta_core/Cargo.toml` `[features] simd = ["dep:multiversion"]` + — the gate (default-on). diff --git a/pyproject.toml b/pyproject.toml index 59084e3..75b575c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -84,7 +84,7 @@ omit = ["*/_ferro_ta*", "*/mcp/*"] [tool.coverage.report] show_missing = true -fail_under = 65 +fail_under = 80 exclude_lines = [ "pragma: no cover", "if TYPE_CHECKING:", diff --git a/python/ferro_ta/__init__.py b/python/ferro_ta/__init__.py index 07e384d..8fc10be 100644 --- a/python/ferro_ta/__init__.py +++ b/python/ferro_ta/__init__.py @@ -110,8 +110,14 @@ __version__ = _detect_version() # --------------------------------------------------------------------------- from ferro_ta.core.exceptions import ( # noqa: F401 FerroTAError, + FerroTaError, FerroTAInputError, FerroTAValueError, + InsufficientDataError, + InvalidInputError, + InvalidPeriodError, + LengthMismatchError, + NumericConvergenceError, ) # --------------------------------------------------------------------------- diff --git a/python/ferro_ta/core/exceptions.py b/python/ferro_ta/core/exceptions.py index d2a8618..aa7d3f3 100644 --- a/python/ferro_ta/core/exceptions.py +++ b/python/ferro_ta/core/exceptions.py @@ -131,6 +131,63 @@ class FerroTAInputError(FerroTAError, ValueError): code = "FTERR002" +# --------------------------------------------------------------------------- +# Finer-grained exception subclasses (added in 1.2.0). +# +# These are drop-in compatible with the base classes: every subclass still +# inherits from ``FerroTAError`` and ``ValueError``, so existing user code +# like ``except FerroTAValueError:`` or ``except ValueError:`` keeps working. +# The subclasses exist so users can catch *specific* failure modes without +# string-matching on the error message. +# --------------------------------------------------------------------------- + + +class InvalidPeriodError(FerroTAValueError): + """Parameter like ``timeperiod``, ``fastperiod``, ``slowperiod`` is out of range. + + Default error code: ``FTERR001``. + """ + + +class InsufficientDataError(FerroTAInputError): + """Input array is shorter than the minimum required for the indicator. + + Default error code: ``FTERR003``. + """ + + code = "FTERR003" + + +class LengthMismatchError(FerroTAInputError): + """Two or more input arrays (e.g. OHLC) have different lengths. + + Default error code: ``FTERR004``. + """ + + code = "FTERR004" + + +class NumericConvergenceError(FerroTAValueError): + """An iterative calculation failed to converge within tolerance. + + Raised by iterative pricing models (implied volatility root-finding, + Newton-Raphson, etc.) when the maximum iteration count is exhausted. + """ + + +class InvalidInputError(FerroTAInputError): + """Input contains NaN/Inf in strict mode, wrong dtype, or wrong shape. + + Default error code: ``FTERR005``. + """ + + code = "FTERR005" + + +# Public aliases that match the names documented in the README and +# CHANGELOG [Unreleased] section. +FerroTaError = FerroTAError # type: ignore[misc] + # --------------------------------------------------------------------------- # Validation helpers (called by Python wrappers) # --------------------------------------------------------------------------- @@ -154,7 +211,7 @@ def check_timeperiod(value: int, name: str = "timeperiod", minimum: int = 1) -> If ``value < minimum``. """ if value < minimum: - raise FerroTAValueError( + raise InvalidPeriodError( f"{name} must be >= {minimum}, got {value}", suggestion=f"Set {name}={minimum} or higher.", ) @@ -193,7 +250,7 @@ def check_equal_length(**arrays: object) -> None: if len(set(lengths.values())) > 1: detail = ", ".join(f"{k}={v}" for k, v in lengths.items()) - raise FerroTAInputError( + raise LengthMismatchError( f"All input arrays must have the same length. Got: {detail}", code=_CODE_LENGTH_MISMATCH, suggestion="Trim or align your arrays so that open, high, low, close, and volume all have the same number of rows.", @@ -223,7 +280,7 @@ def check_finite(arr: object, name: str = "input") -> None: a = np.asarray(arr, dtype=np.float64) if not np.all(np.isfinite(a)): - raise FerroTAInputError( + raise InvalidInputError( f"{name} contains NaN or Inf values. " "ferro_ta propagates NaN by default; call check_finite() only " "when you require all-finite inputs.", @@ -255,7 +312,7 @@ def check_min_length(arr: object, min_len: int, name: str = "input") -> None: elif hasattr(arr, "shape"): length = arr.shape[0] # type: ignore[union-attr] if length < min_len: - raise FerroTAInputError( + raise InsufficientDataError( f"{name} must have at least {min_len} elements, got {length}", code=_CODE_TOO_SHORT, suggestion=f"Provide at least {min_len} data points. Current length: {length}.", diff --git a/uv.lock b/uv.lock index ffe567a..ad6fc03 100644 --- a/uv.lock +++ b/uv.lock @@ -1272,11 +1272,11 @@ wheels = [ [[package]] name = "idna" -version = "3.11" +version = "3.18" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/6f/6d/0703ccc57f3a7233505399edb88de3cbd678da106337b9fcde432b65ed60/idna-3.11.tar.gz", hash = "sha256:795dafcc9c04ed0c1fb032c2aa73654d8e8c5023a7df64a53f39190ada629902", size = 194582, upload-time = "2025-10-12T14:55:20.501Z" } +sdist = { url = "https://files.pythonhosted.org/packages/cd/63/9496c57188a2ee585e0f1db071d75089a11e98aa86eb99d9d7618fc1edce/idna-3.18.tar.gz", hash = "sha256:ffb385a7e039654cef1ab9ef32c6fafe283c0c0467bba1d9029738ce4a14a848", size = 196711, upload-time = "2026-06-02T14:34:07.794Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/0e/61/66938bbb5fc52dbdf84594873d5b51fb1f7c7794e9c0f5bd885f30bc507b/idna-3.11-py3-none-any.whl", hash = "sha256:771a87f49d9defaf64091e6e6fe9c18d4833f140bd19464795bc32d966ca37ea", size = 71008, upload-time = "2025-10-12T14:55:18.883Z" }, + { url = "https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl", hash = "sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2", size = 65455, upload-time = "2026-06-02T14:34:06.319Z" }, ] [[package]] @@ -2906,7 +2906,7 @@ wheels = [ [[package]] name = "pytest" -version = "9.0.2" +version = "9.1.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "colorama", marker = "sys_platform == 'win32'" }, @@ -2917,9 +2917,9 @@ dependencies = [ { name = "pygments" }, { name = "tomli", marker = "python_full_version < '3.11'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/d1/db/7ef3487e0fb0049ddb5ce41d3a49c235bf9ad299b6a25d5780a89f19230f/pytest-9.0.2.tar.gz", hash = "sha256:75186651a92bd89611d1d9fc20f0b4345fd827c41ccd5c299a868a05d70edf11", size = 1568901, upload-time = "2025-12-06T21:30:51.014Z" } +sdist = { url = "https://files.pythonhosted.org/packages/e4/47/b9efed96c114afcfa3c9d3fe98a76a1d14c74a9e266d397cf6eb64be5e01/pytest-9.1.1.tar.gz", hash = "sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313", size = 1636369, upload-time = "2026-06-19T10:58:32.857Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/3b/ab/b3226f0bd7cdcf710fbede2b3548584366da3b19b5021e74f5bde2a8fa3f/pytest-9.0.2-py3-none-any.whl", hash = "sha256:711ffd45bf766d5264d487b917733b453d917afd2b0ad65223959f59089f875b", size = 374801, upload-time = "2025-12-06T21:30:49.154Z" }, + { url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" }, ] [[package]] @@ -4261,11 +4261,11 @@ wheels = [ [[package]] name = "urllib3" -version = "2.6.3" +version = "2.7.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/c7/24/5f1b3bdffd70275f6661c76461e25f024d5a38a46f04aaca912426a2b1d3/urllib3-2.6.3.tar.gz", hash = "sha256:1b62b6884944a57dbe321509ab94fd4d3b307075e0c2eae991ac71ee15ad38ed", size = 435556, upload-time = "2026-01-07T16:24:43.925Z" } +sdist = { url = "https://files.pythonhosted.org/packages/53/0c/06f8b233b8fd13b9e5ee11424ef85419ba0d8ba0b3138bf360be2ff56953/urllib3-2.7.0.tar.gz", hash = "sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c", size = 433602, upload-time = "2026-05-07T16:13:18.596Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/39/08/aaaad47bc4e9dc8c725e68f9d04865dbcb2052843ff09c97b08904852d84/urllib3-2.6.3-py3-none-any.whl", hash = "sha256:bf272323e553dfb2e87d9bfd225ca7b0f467b919d7bbd355436d3fd37cb0acd4", size = 131584, upload-time = "2026-01-07T16:24:42.685Z" }, + { url = "https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897", size = 131087, upload-time = "2026-05-07T16:13:17.151Z" }, ] [[package]]