ci: enhance CI workflows for Python, Rust, and WASM
- Updated Python CI to streamline linting, type checking, and testing processes, including the addition of a wheel build job. - Refactored Rust CI to utilize pre-built actions for cargo-deny and cargo-audit, improving dependency checks. - Optimized WASM CI by consolidating build and test steps, and ensuring proper artifact uploads for both Node.js and web packages. - Added token authentication for GitHub actions in the release workflow to enhance security.
This commit is contained in:
Pratik Bhadane
@@ -7,151 +7,153 @@ permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
# -------------------------------------------------------------------------
|
||||
# Lint — no Rust, no wheel build, very fast
|
||||
# -------------------------------------------------------------------------
|
||||
lint:
|
||||
name: Lint (ruff)
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Python 3.12
|
||||
uses: actions/setup-python@v6
|
||||
- uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: "3.12"
|
||||
- run: pip install uv
|
||||
- run: uv run --with ruff ruff check python/ tests/
|
||||
- run: uv run --with ruff ruff format --check python/ tests/
|
||||
|
||||
- name: Install uv
|
||||
run: pip install uv
|
||||
|
||||
- name: Run ruff check via uv
|
||||
run: uv run --with ruff ruff check python/ tests/
|
||||
- name: Run ruff format check via uv
|
||||
run: uv run --with ruff ruff format --check python/ tests/
|
||||
|
||||
# -------------------------------------------------------------------------
|
||||
# Type checking — needs wheel; build once with cache
|
||||
# -------------------------------------------------------------------------
|
||||
typecheck:
|
||||
name: Type checking (mypy + pyright)
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Python 3.12
|
||||
uses: actions/setup-python@v6
|
||||
- uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: "3.12"
|
||||
- uses: dtolnay/rust-toolchain@v1
|
||||
with:
|
||||
toolchain: stable
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
- run: pip install maturin numpy
|
||||
- run: maturin build --release --out dist && pip install dist/*.whl
|
||||
- run: pip install uv
|
||||
- run: uv run --with mypy --with numpy python -m mypy python/ferro_ta --ignore-missing-imports --no-error-summary
|
||||
- run: uv run --with pyright python -m pyright python/ferro_ta
|
||||
|
||||
- name: Install uv
|
||||
run: pip install uv
|
||||
|
||||
- name: Run mypy on ferro_ta via uv
|
||||
run: uv run --with mypy --with numpy python -m mypy python/ferro_ta --ignore-missing-imports --no-error-summary
|
||||
|
||||
- name: Run pyright on ferro_ta via uv
|
||||
run: uv run --with pyright python -m pyright python/ferro_ta
|
||||
|
||||
test:
|
||||
name: Test (ubuntu-latest / Python ${{ matrix.python-version }})
|
||||
# -------------------------------------------------------------------------
|
||||
# Build wheel artifact (once per run, reused by all test matrix entries)
|
||||
# -------------------------------------------------------------------------
|
||||
build-wheel:
|
||||
name: Build wheel (ubuntu / Python ${{ matrix.python-version }})
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
matrix:
|
||||
python-version: ["3.10", "3.11", "3.12", "3.13"]
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
- uses: dtolnay/rust-toolchain@v1
|
||||
with:
|
||||
toolchain: stable
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
with:
|
||||
shared-key: "wheel-${{ matrix.python-version }}"
|
||||
- run: pip install maturin numpy
|
||||
- run: maturin build --release --out dist
|
||||
- uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: wheel-${{ matrix.python-version }}
|
||||
path: dist/*.whl
|
||||
|
||||
# -------------------------------------------------------------------------
|
||||
# Test matrix — downloads pre-built wheel, no Rust compilation
|
||||
# -------------------------------------------------------------------------
|
||||
test:
|
||||
name: Test (Python ${{ matrix.python-version }})
|
||||
runs-on: ubuntu-latest
|
||||
needs: build-wheel
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
python-version: ["3.10", "3.11", "3.12", "3.13"]
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Python ${{ matrix.python-version }}
|
||||
uses: actions/setup-python@v6
|
||||
- uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
|
||||
- name: Install maturin and test dependencies
|
||||
- uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: wheel-${{ matrix.python-version }}
|
||||
path: dist
|
||||
- name: Install wheel + test deps
|
||||
run: |
|
||||
pip install maturin numpy pytest pytest-cov pandas polars hypothesis pyyaml mcp
|
||||
|
||||
- name: Build and install ferro_ta (dev mode)
|
||||
run: |
|
||||
maturin build --release --out dist
|
||||
pip install dist/*.whl
|
||||
|
||||
- name: Run unit tests with coverage
|
||||
pip install pytest pytest-cov pandas polars hypothesis pyyaml mcp scipy
|
||||
- name: Run tests with coverage
|
||||
run: pytest tests/unit/ tests/integration/ -v --cov=ferro_ta --cov-report=xml --cov-report=term-missing --cov-fail-under=65
|
||||
|
||||
- name: Check API manifest is current
|
||||
if: matrix.python-version == '3.12'
|
||||
run: python scripts/check_api_manifest.py
|
||||
|
||||
- name: Upload coverage report
|
||||
uses: actions/upload-artifact@v7
|
||||
- uses: actions/upload-artifact@v7
|
||||
if: matrix.python-version == '3.12'
|
||||
with:
|
||||
name: coverage-python-${{ matrix.python-version }}
|
||||
path: coverage.xml
|
||||
|
||||
# -------------------------------------------------------------------------
|
||||
# Benchmark vs TA-Lib (downloads wheel from build-wheel job)
|
||||
# -------------------------------------------------------------------------
|
||||
benchmark-vs-talib:
|
||||
name: Benchmark vs TA-Lib
|
||||
runs-on: ubuntu-latest
|
||||
needs: build-wheel
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Python 3.12
|
||||
uses: actions/setup-python@v6
|
||||
- uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: "3.12"
|
||||
|
||||
- name: Install TA-Lib C library (Ubuntu)
|
||||
- uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: wheel-3.12
|
||||
path: dist
|
||||
- name: Install TA-Lib C library
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y build-essential curl
|
||||
curl -sL https://sourceforge.net/projects/ta-lib/files/ta-lib/0.4.0/ta-lib-0.4.0-src.tar.gz/download -o ta-lib-0.4.0-src.tar.gz
|
||||
tar -xzf ta-lib-0.4.0-src.tar.gz
|
||||
cd ta-lib
|
||||
./configure --prefix=/usr
|
||||
make
|
||||
sudo make install
|
||||
sudo ldconfig
|
||||
|
||||
- name: Install maturin and ta-lib Python package
|
||||
run: |
|
||||
pip install maturin numpy
|
||||
pip install ta-lib
|
||||
|
||||
- name: Build and install ferro_ta
|
||||
run: |
|
||||
maturin build --release --out dist
|
||||
pip install dist/*.whl
|
||||
|
||||
- name: Run benchmark comparison
|
||||
run: |
|
||||
python benchmarks/bench_vs_talib.py --sizes 10000 100000 --json benchmark_vs_talib.json
|
||||
|
||||
- name: Enforce benchmark regression policy
|
||||
run: python3 benchmarks/check_vs_talib_regression.py --input benchmark_vs_talib.json
|
||||
|
||||
- name: Upload benchmark results
|
||||
uses: actions/upload-artifact@v7
|
||||
cd ta-lib && ./configure --prefix=/usr && make && sudo make install && sudo ldconfig
|
||||
- run: pip install dist/*.whl numpy ta-lib
|
||||
- run: python benchmarks/bench_vs_talib.py --sizes 10000 100000 --json benchmark_vs_talib.json
|
||||
- run: python3 benchmarks/check_vs_talib_regression.py --input benchmark_vs_talib.json
|
||||
- uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: benchmark-vs-talib
|
||||
path: benchmark_vs_talib.json
|
||||
|
||||
# -------------------------------------------------------------------------
|
||||
# Performance smoke (downloads wheel from build-wheel job)
|
||||
# -------------------------------------------------------------------------
|
||||
perf-smoke:
|
||||
name: Performance smoke and contracts
|
||||
runs-on: ubuntu-latest
|
||||
needs: build-wheel
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Python 3.12
|
||||
uses: actions/setup-python@v6
|
||||
- uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: "3.12"
|
||||
|
||||
- name: Install maturin and perf dependencies
|
||||
run: |
|
||||
pip install maturin numpy pytest
|
||||
|
||||
- name: Build and install ferro_ta
|
||||
run: |
|
||||
maturin build --release --out dist
|
||||
pip install dist/*.whl
|
||||
|
||||
- name: Generate reproducible perf artifacts
|
||||
run: |
|
||||
- uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: wheel-3.12
|
||||
path: dist
|
||||
- run: pip install dist/*.whl numpy pytest
|
||||
- run: |
|
||||
python benchmarks/run_perf_contract.py \
|
||||
--output-dir perf-contract \
|
||||
--skip-talib \
|
||||
@@ -161,12 +163,8 @@ jobs:
|
||||
--streaming-bars 20000 \
|
||||
--price-bars 20000 \
|
||||
--iv-bars 50000
|
||||
|
||||
- name: Enforce hotspot regression policy
|
||||
run: python benchmarks/check_hotspot_regression.py --input perf-contract/runtime_hotspots.json
|
||||
|
||||
- name: Upload perf artifacts
|
||||
uses: actions/upload-artifact@v7
|
||||
- run: python benchmarks/check_hotspot_regression.py --input perf-contract/runtime_hotspots.json
|
||||
- uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: perf-contract
|
||||
path: perf-contract/
|
||||
|
||||
@@ -7,38 +7,48 @@ permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
audit:
|
||||
name: Dependency audit (cargo + pip)
|
||||
# -------------------------------------------------------------------------
|
||||
# cargo-deny — uses the official pre-built action (no cargo install needed)
|
||||
# -------------------------------------------------------------------------
|
||||
cargo-deny:
|
||||
name: cargo deny check
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: EmbarkStudios/cargo-deny-action@v2
|
||||
|
||||
- name: Install cargo-audit and run cargo audit
|
||||
run: |
|
||||
cargo install cargo-audit
|
||||
cargo audit
|
||||
# -------------------------------------------------------------------------
|
||||
# cargo-audit — uses rustsec/audit-check pre-built action
|
||||
# -------------------------------------------------------------------------
|
||||
cargo-audit:
|
||||
name: cargo audit
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: rustsec/audit-check@v2
|
||||
with:
|
||||
token: ${{ github.token }}
|
||||
|
||||
- name: Install cargo-deny and run cargo deny check
|
||||
run: |
|
||||
cargo install cargo-deny --locked
|
||||
cargo deny check
|
||||
|
||||
- name: Set up Python 3.12
|
||||
uses: actions/setup-python@v6
|
||||
# -------------------------------------------------------------------------
|
||||
# pip-audit + uv.lock freshness check (lightweight, no Rust needed)
|
||||
# -------------------------------------------------------------------------
|
||||
pip-audit:
|
||||
name: pip-audit + uv.lock check
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: "3.12"
|
||||
- run: pip install uv
|
||||
- run: uv run --with pip-audit pip-audit --skip-editable
|
||||
- run: uv lock --check
|
||||
|
||||
- name: Install uv
|
||||
run: pip install uv
|
||||
|
||||
- name: Install pip-audit via uv and run pip-audit
|
||||
run: uv run --with pip-audit pip-audit --skip-editable
|
||||
|
||||
- name: Verify uv.lock is up-to-date
|
||||
run: uv lock --check
|
||||
|
||||
rust:
|
||||
name: Rust (fmt + clippy)
|
||||
# -------------------------------------------------------------------------
|
||||
# fmt + clippy (with Rust cache so subsequent runs skip recompilation)
|
||||
# -------------------------------------------------------------------------
|
||||
rust-lint:
|
||||
name: Rust fmt + clippy
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
@@ -46,26 +56,32 @@ jobs:
|
||||
with:
|
||||
toolchain: stable
|
||||
components: rustfmt, clippy
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
- run: cargo fmt --all -- --check
|
||||
- run: cargo clippy --release -- -D warnings
|
||||
- name: Verify benchmarks compile (ferro_ta_core)
|
||||
- name: Verify benchmarks compile
|
||||
run: cargo bench -p ferro_ta_core --no-run
|
||||
|
||||
# -------------------------------------------------------------------------
|
||||
# Build + test ferro_ta_core (with Rust cache)
|
||||
# -------------------------------------------------------------------------
|
||||
rust-core:
|
||||
name: Rust core library (ferro_ta_core)
|
||||
name: Rust core (build + test)
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: dtolnay/rust-toolchain@v1
|
||||
with:
|
||||
toolchain: stable
|
||||
- name: Build core crate
|
||||
run: cargo build -p ferro_ta_core
|
||||
- name: Test core crate
|
||||
run: cargo test -p ferro_ta_core
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
- run: cargo build -p ferro_ta_core
|
||||
- run: cargo test -p ferro_ta_core
|
||||
|
||||
# -------------------------------------------------------------------------
|
||||
# Coverage (optional, cached)
|
||||
# -------------------------------------------------------------------------
|
||||
rust-coverage:
|
||||
name: Rust coverage (tarpaulin, optional)
|
||||
name: Rust coverage (tarpaulin)
|
||||
runs-on: ubuntu-latest
|
||||
continue-on-error: true
|
||||
steps:
|
||||
@@ -73,19 +89,22 @@ jobs:
|
||||
- uses: dtolnay/rust-toolchain@v1
|
||||
with:
|
||||
toolchain: stable
|
||||
- name: Install cargo-tarpaulin
|
||||
run: cargo install cargo-tarpaulin --locked
|
||||
- name: Collect Rust coverage (ferro_ta_core)
|
||||
run: cargo tarpaulin -p ferro_ta_core --out Xml --output-dir coverage/
|
||||
- name: Upload Rust coverage artifact
|
||||
uses: actions/upload-artifact@v7
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
- uses: taiki-e/install-action@v2
|
||||
with:
|
||||
tool: cargo-tarpaulin
|
||||
- run: cargo tarpaulin -p ferro_ta_core --out Xml --output-dir coverage/
|
||||
- uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: rust-coverage
|
||||
path: coverage/
|
||||
if-no-files-found: ignore
|
||||
|
||||
# -------------------------------------------------------------------------
|
||||
# Fuzz (optional, nightly, cached)
|
||||
# -------------------------------------------------------------------------
|
||||
fuzz:
|
||||
name: Fuzz targets (short CI run, optional)
|
||||
name: Fuzz targets (short CI run)
|
||||
runs-on: ubuntu-latest
|
||||
continue-on-error: true
|
||||
steps:
|
||||
@@ -93,16 +112,17 @@ jobs:
|
||||
- uses: dtolnay/rust-toolchain@v1
|
||||
with:
|
||||
toolchain: nightly
|
||||
- name: Install cargo-fuzz
|
||||
run: cargo install cargo-fuzz --locked
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
- uses: taiki-e/install-action@v2
|
||||
with:
|
||||
tool: cargo-fuzz
|
||||
- name: Run fuzz_sma (10000 iterations)
|
||||
working-directory: fuzz
|
||||
run: cargo fuzz run fuzz_sma -- -runs=10000 -max_len=512
|
||||
- name: Run fuzz_rsi (10000 iterations)
|
||||
working-directory: fuzz
|
||||
run: cargo fuzz run fuzz_rsi -- -runs=10000 -max_len=512
|
||||
- name: Upload fuzz artifacts on crash
|
||||
uses: actions/upload-artifact@v7
|
||||
- uses: actions/upload-artifact@v7
|
||||
if: always()
|
||||
with:
|
||||
name: fuzz-artifacts
|
||||
|
||||
@@ -8,54 +8,41 @@ permissions:
|
||||
|
||||
jobs:
|
||||
wasm:
|
||||
name: WASM binding (wasm-pack test --node)
|
||||
name: WASM (test + build + bench)
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Node.js
|
||||
uses: actions/setup-node@v6
|
||||
- uses: actions/setup-node@v6
|
||||
with:
|
||||
node-version: "20"
|
||||
|
||||
- name: Install Rust (stable)
|
||||
uses: dtolnay/rust-toolchain@v1
|
||||
- uses: dtolnay/rust-toolchain@v1
|
||||
with:
|
||||
toolchain: stable
|
||||
targets: wasm32-unknown-unknown
|
||||
|
||||
- name: Install wasm-pack
|
||||
run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh
|
||||
|
||||
- name: Build and test WASM binding
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
- uses: taiki-e/install-action@v2
|
||||
with:
|
||||
tool: wasm-pack
|
||||
- name: Test WASM binding
|
||||
working-directory: wasm
|
||||
run: wasm-pack test --node
|
||||
|
||||
- name: Build WASM package (both targets)
|
||||
- name: Build WASM packages (node + web)
|
||||
working-directory: wasm
|
||||
run: npm run build
|
||||
|
||||
- name: Check API manifest is current
|
||||
run: python3 scripts/check_api_manifest.py
|
||||
|
||||
- name: Benchmark WASM package
|
||||
- name: Benchmark WASM
|
||||
working-directory: wasm
|
||||
run: node bench.js --json ../wasm_benchmark.json
|
||||
|
||||
- name: Upload WASM node package artifact
|
||||
uses: actions/upload-artifact@v7
|
||||
- uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: wasm-pkg-node
|
||||
path: wasm/node/
|
||||
|
||||
- name: Upload WASM web package artifact
|
||||
uses: actions/upload-artifact@v7
|
||||
- uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: wasm-pkg-web
|
||||
path: wasm/web/
|
||||
|
||||
- name: Upload WASM benchmark artifact
|
||||
uses: actions/upload-artifact@v7
|
||||
- uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: wasm-benchmark
|
||||
path: wasm_benchmark.json
|
||||
|
||||
@@ -20,6 +20,7 @@ jobs:
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
fetch-depth: 0
|
||||
token: ${{ secrets.GH_PAT }}
|
||||
|
||||
- name: Extract version from tag
|
||||
id: version
|
||||
@@ -58,3 +59,4 @@ jobs:
|
||||
body: ${{ steps.changelog.outputs.body }}
|
||||
draft: false
|
||||
prerelease: ${{ contains(github.ref_name, '-') }}
|
||||
token: ${{ secrets.GH_PAT }}
|
||||
|
||||
Reference in New Issue
Block a user