diff --git a/.github/workflows/ci-rust.yml b/.github/workflows/ci-rust.yml index 8a07ba0..80abd87 100644 --- a/.github/workflows/ci-rust.yml +++ b/.github/workflows/ci-rust.yml @@ -18,13 +18,20 @@ jobs: - uses: EmbarkStudios/cargo-deny-action@v2 # ------------------------------------------------------------------------- - # cargo-audit — uses rustsec/audit-check pre-built action + # cargo-audit — caches the RustSec advisory database (~400 MB git clone) + # so subsequent runs skip the initial fetch entirely (~3 min → ~15s) # ------------------------------------------------------------------------- cargo-audit: name: cargo audit runs-on: ubuntu-latest steps: - uses: actions/checkout@v6 + - name: Cache RustSec advisory database + uses: actions/cache@v4 + with: + path: ~/.cargo/advisory-db + key: rustsec-advisory-db-${{ hashFiles('Cargo.lock') }} + restore-keys: rustsec-advisory-db- - uses: rustsec/audit-check@v2 with: token: ${{ github.token }} @@ -112,16 +119,17 @@ jobs: - uses: dtolnay/rust-toolchain@v1 with: toolchain: nightly + targets: x86_64-unknown-linux-gnu - uses: Swatinem/rust-cache@v2 - uses: taiki-e/install-action@v2 with: tool: cargo-fuzz - name: Run fuzz_sma (10000 iterations) working-directory: fuzz - run: cargo fuzz run fuzz_sma -- -runs=10000 -max_len=512 + run: cargo fuzz run fuzz_sma --target x86_64-unknown-linux-gnu -- -runs=10000 -max_len=512 - name: Run fuzz_rsi (10000 iterations) working-directory: fuzz - run: cargo fuzz run fuzz_rsi -- -runs=10000 -max_len=512 + run: cargo fuzz run fuzz_rsi --target x86_64-unknown-linux-gnu -- -runs=10000 -max_len=512 - uses: actions/upload-artifact@v7 if: always() with: diff --git a/scripts/pre_push_checks.sh b/scripts/pre_push_checks.sh index 71bab4a..0417d4c 100755 --- a/scripts/pre_push_checks.sh +++ b/scripts/pre_push_checks.sh @@ -1,27 +1,40 @@ #!/usr/bin/env bash +# Pre-push CI gate — runs checks in parallel to minimise wall-clock time. +# +# Usage: +# scripts/pre_push_checks.sh # all checks +# scripts/pre_push_checks.sh rust_clippy wasm # selected checks +# scripts/pre_push_checks.sh --list +# FERRO_FAST=1 scripts/pre_push_checks.sh # skip docs + wasm bench set -euo pipefail ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" cd "$ROOT_DIR" AVAILABLE_CHECKS=( - version - changelog - rust_fmt - rust_clippy - rust_core - rust_bench - python_lint - python_typecheck - python_test - docs - wasm - manifest + version changelog manifest + rust_fmt rust_clippy rust_core rust_bench + python_lint python_typecheck python_test + docs wasm ) - DEFAULT_CHECKS=("${AVAILABLE_CHECKS[@]}") -python_env_ready=0 +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +need_cmd() { + if ! command -v "$1" >/dev/null 2>&1; then + echo "Missing required command: $1" >&2; exit 1 + fi +} + +run_cmd() { + printf ' +' + printf ' %q' "$@" + printf '\n' + "$@" +} usage() { cat <<'EOF' @@ -30,93 +43,60 @@ Usage: scripts/pre_push_checks.sh [ ...] scripts/pre_push_checks.sh --list -Runs the repo's basic local CI gate before push. By default it covers: - version changelog rust_fmt rust_clippy rust_core rust_bench - python_lint python_typecheck python_test docs wasm manifest - -Notes: - - This mirrors the required CI categories we can run locally. - - It intentionally skips the multi-Python test matrix, audit jobs, perf smoke, - and benchmark-regression jobs. +Environment: + FERRO_FAST=1 Skip docs and wasm (fastest local feedback loop) EOF } -list_checks() { - printf '%s\n' "${AVAILABLE_CHECKS[@]}" -} - -need_cmd() { - local command_name="$1" - if ! command -v "$command_name" >/dev/null 2>&1; then - echo "Missing required command: $command_name" >&2 - exit 1 - fi -} - -run_cmd() { - printf ' +' - printf ' %q' "$@" - printf '\n' - "$@" -} - -ensure_python_env() { - if [[ "$python_env_ready" -eq 1 ]]; then - return - fi - - need_cmd uv - run_cmd uv sync --extra dev --extra docs --extra mcp - run_cmd uv run --extra dev --extra docs --extra mcp maturin develop --release - python_env_ready=1 -} - -run_version() { - need_cmd python3 - run_cmd python3 scripts/bump_version.py --check -} - -run_changelog() { - need_cmd python3 - run_cmd python3 scripts/check_changelog.py -} - -run_rust_fmt() { - need_cmd cargo - run_cmd cargo fmt --all -- --check -} - -run_rust_clippy() { - need_cmd cargo - run_cmd cargo clippy --release -- -D warnings -} - -run_rust_core() { - need_cmd cargo - run_cmd cargo build -p ferro_ta_core - run_cmd cargo test -p ferro_ta_core -} - -run_rust_bench() { - need_cmd cargo - run_cmd cargo bench -p ferro_ta_core --no-run -} +# --------------------------------------------------------------------------- +# Individual check functions +# --------------------------------------------------------------------------- +run_version() { need_cmd python3; run_cmd python3 scripts/bump_version.py --check; } +run_changelog() { need_cmd python3; run_cmd python3 scripts/check_changelog.py; } +run_manifest() { need_cmd python3; run_cmd python3 scripts/check_api_manifest.py; } +run_rust_fmt() { need_cmd cargo; run_cmd cargo fmt --all -- --check; } run_python_lint() { need_cmd uv run_cmd uv run --with ruff ruff check python/ tests/ run_cmd uv run --with ruff ruff format --check python/ tests/ } +run_rust_clippy() { need_cmd cargo; run_cmd cargo clippy --release -- -D warnings; } +run_rust_core() { need_cmd cargo; run_cmd cargo build -p ferro_ta_core && run_cmd cargo test -p ferro_ta_core; } +run_rust_bench() { need_cmd cargo; run_cmd cargo bench -p ferro_ta_core --no-run; } + run_python_typecheck() { need_cmd uv - run_cmd uv run --with mypy --with numpy python -m mypy python/ferro_ta --ignore-missing-imports --no-error-summary + run_cmd uv run --with mypy --with numpy python -m mypy python/ferro_ta \ + --ignore-missing-imports --no-error-summary run_cmd uv run --with pyright python -m pyright python/ferro_ta } +# python_test and docs both need a compiled extension. +# Use a flag file so only the first concurrent caller runs maturin develop; +# subsequent callers (in parallel background jobs) wait and reuse it. +_MATURIN_LOCK="${TMPDIR:-/tmp}/ferro_ta_maturin_$$.lock" +_MATURIN_FLAG="${TMPDIR:-/tmp}/ferro_ta_maturin_$$.done" + +ensure_python_env() { + [[ -f "$_MATURIN_FLAG" ]] && return + ( + flock 9 + if [[ ! -f "$_MATURIN_FLAG" ]]; then + need_cmd uv + run_cmd uv sync --extra dev --extra docs --extra mcp + run_cmd uv run --extra dev --extra docs --extra mcp maturin develop --release + touch "$_MATURIN_FLAG" + fi + ) 9>"$_MATURIN_LOCK" +} + run_python_test() { ensure_python_env - run_cmd uv run --extra dev --extra mcp --with pytest-cov pytest tests/unit/ tests/integration/ -v --cov=ferro_ta --cov-report=term-missing --cov-fail-under=65 + run_cmd uv run --extra dev --extra mcp --with pytest-cov \ + pytest tests/unit/ tests/integration/ \ + -v --cov=ferro_ta --cov-report=term-missing --cov-fail-under=65 } run_docs() { @@ -125,69 +105,161 @@ run_docs() { } run_wasm() { - need_cmd node - need_cmd wasm-pack - local benchmark_json="../.wasm_benchmark.prepush.json" + need_cmd node; need_cmd wasm-pack ( cd wasm - trap 'rm -f "$benchmark_json"' EXIT run_cmd wasm-pack test --node run_cmd npm run build - run_cmd node bench.js --json "$benchmark_json" + if [[ "${FERRO_FAST:-0}" != "1" ]]; then + local bj="../.wasm_benchmark.prepush.json" + run_cmd node bench.js --json "$bj" + rm -f "$bj" + fi ) } -run_manifest() { - need_cmd python3 - run_cmd python3 scripts/check_api_manifest.py -} - run_check() { - local check_name="$1" - case "$check_name" in - version) run_version ;; - changelog) run_changelog ;; - rust_fmt) run_rust_fmt ;; - rust_clippy) run_rust_clippy ;; - rust_core) run_rust_core ;; - rust_bench) run_rust_bench ;; - python_lint) run_python_lint ;; + case "$1" in + version) run_version ;; + changelog) run_changelog ;; + manifest) run_manifest ;; + rust_fmt) run_rust_fmt ;; + rust_clippy) run_rust_clippy ;; + rust_core) run_rust_core ;; + rust_bench) run_rust_bench ;; + python_lint) run_python_lint ;; python_typecheck) run_python_typecheck ;; - python_test) run_python_test ;; - docs) run_docs ;; - wasm) run_wasm ;; - manifest) run_manifest ;; - *) - echo "Unknown check: $check_name" >&2 - echo "Use --list to see supported checks." >&2 - exit 1 - ;; + python_test) run_python_test ;; + docs) run_docs ;; + wasm) run_wasm ;; + *) echo "Unknown check: $1 — use --list" >&2; exit 1 ;; esac } -if [[ "${1:-}" == "--help" || "${1:-}" == "-h" ]]; then - usage - exit 0 -fi +# --------------------------------------------------------------------------- +# Parallel runner — starts all checks concurrently, collects results +# --------------------------------------------------------------------------- -if [[ "${1:-}" == "--list" ]]; then - list_checks - exit 0 -fi +run_parallel() { + local -a checks=("$@") + [[ "${#checks[@]}" -eq 0 ]] && return 0 + + local -a pids logs names + local start + start=$(date +%s) + + printf '\nStarting %d checks in parallel: %s\n' "${#checks[@]}" "${checks[*]}" + + for check in "${checks[@]}"; do + local log + log=$(mktemp /tmp/ferro_prepush_XXXXXX) + logs+=("$log") + names+=("$check") + run_check "$check" >"$log" 2>&1 & + pids+=($!) + done + + local failed=0 + local -a failed_names + printf '\n' + for i in "${!pids[@]}"; do + if wait "${pids[$i]}" 2>/dev/null; then + printf ' ✓ %s\n' "${names[$i]}" + else + printf ' ✗ %s\n' "${names[$i]}" + failed_names+=("${names[$i]}") + failed=1 + fi + done + + # Print logs for failed checks only + if [[ "$failed" -eq 1 ]]; then + for i in "${!names[@]}"; do + local name="${names[$i]}" + if [[ " ${failed_names[*]:-} " == *" $name "* ]]; then + printf '\n'; printf '━%.0s' {1..60}; printf '\nFAILED: %s\n' "$name"; printf '━%.0s' {1..60}; printf '\n' + cat "${logs[$i]}" + fi + done + fi + + for log in "${logs[@]}"; do rm -f "$log"; done + rm -f "$_MATURIN_LOCK" "$_MATURIN_FLAG" + + printf '\nElapsed: %ds\n' "$(( $(date +%s) - start ))" + return "$failed" +} + +# --------------------------------------------------------------------------- +# Argument parsing +# --------------------------------------------------------------------------- + +[[ "${1:-}" == "--help" || "${1:-}" == "-h" ]] && { usage; exit 0; } +[[ "${1:-}" == "--list" ]] && { printf '%s\n' "${AVAILABLE_CHECKS[@]}"; exit 0; } selected_checks=() if [[ "$#" -gt 0 ]]; then selected_checks=("$@") else selected_checks=("${DEFAULT_CHECKS[@]}") + if [[ "${FERRO_FAST:-0}" == "1" ]]; then + selected_checks=() + for c in "${DEFAULT_CHECKS[@]}"; do + [[ "$c" == "docs" || "$c" == "wasm" ]] && continue + selected_checks+=("$c") + done + printf 'FERRO_FAST=1: skipping docs + wasm\n' + fi fi -total_checks="${#selected_checks[@]}" -index=0 -for check_name in "${selected_checks[@]}"; do - index=$((index + 1)) - printf '\n[%d/%d] %s\n' "$index" "$total_checks" "$check_name" - run_check "$check_name" +# --------------------------------------------------------------------------- +# Execution strategy: +# Phase 1 — instant gate (sequential, fail-fast): +# version, changelog, manifest, python_lint, rust_fmt +# These are trivial to run and catch the most common mistakes early. +# If any fail here we abort immediately without waiting for slow checks. +# +# Phase 2 — everything else in parallel: +# rust_clippy, rust_core, rust_bench, python_typecheck, +# python_test, docs, wasm +# --------------------------------------------------------------------------- + +FAST_CHECKS=(version changelog manifest python_lint rust_fmt) + +phase1=() +phase2=() +for c in "${selected_checks[@]}"; do + is_fast=0 + for f in "${FAST_CHECKS[@]}"; do [[ "$c" == "$f" ]] && is_fast=1 && break; done + if [[ "$is_fast" -eq 1 ]]; then phase1+=("$c"); else phase2+=("$c"); fi done -printf '\nAll selected pre-push checks passed.\n' +# Phase 1: fast gate +if [[ "${#phase1[@]}" -gt 0 ]]; then + printf 'Phase 1 — fast gate (%d checks)\n' "${#phase1[@]}" + start1=$(date +%s) + for c in "${phase1[@]}"; do + printf ' [%s] ... ' "$c" + log=$(mktemp /tmp/ferro_prepush_XXXXXX) + if run_check "$c" >"$log" 2>&1; then + printf 'ok\n' + else + printf 'FAILED\n' + cat "$log" + rm -f "$log" + echo "" >&2 + echo "Fast gate failed on '$c' — aborting before slow checks." >&2 + exit 1 + fi + rm -f "$log" + done + printf 'Phase 1 passed (%ds)\n' "$(( $(date +%s) - start1 ))" +fi + +# Phase 2: parallel slow checks +if [[ "${#phase2[@]}" -gt 0 ]]; then + printf '\nPhase 2 — parallel slow checks\n' + run_parallel "${phase2[@]}" || exit 1 +fi + +printf '\nAll pre-push checks passed.\n'