mirror of
https://github.com/NicolasBohn/NexQuant.git
synced 2026-07-27 23:47:46 +00:00
feat: Add GitHub infrastructure, CI/CD pipelines, and examples
- Add GitHub issue templates (bug, feature, docs) - Add pull request template with closed-source checklist - Add CODEOWNERS for code review assignment - Add CI/CD workflows (ci, lint, security, docs, release) - pytest + coverage with Python 3.10/3.11 matrix - Ruff + MyPy code quality checks - Bandit + safety security scanning - Sphinx docs + GitHub Pages deployment - Automated PyPI releases on tag push - Add 6 comprehensive examples + Jupyter quickstart - 01_factor_discovery.py (LLM factor generation) - 02_factor_evolution.py (factor optimization) - 03_strategy_generation.py (IC-weighted combination) - 04_backtest_simple.py (strategy backtesting) - 05_model_training.py (XGBoost/LSTM training) - 06_rl_trading_agent.py (PPO/DQN/A2C agents) - notebooks/quickstart.ipynb (interactive tutorial) - Restructure .gitignore with explicit closed-source sections - Add CI/coverage/license badges to README - Complete CLI docstrings for all 9 commands - Add data_config.yaml for quant loop configuration
This commit is contained in:
TPTBusiness
@@ -0,0 +1,84 @@
|
||||
name: CI
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ main, develop ]
|
||||
paths-ignore:
|
||||
- '**.md'
|
||||
- 'docs/**'
|
||||
- 'LICENSE'
|
||||
pull_request:
|
||||
branches: [ main ]
|
||||
paths-ignore:
|
||||
- '**.md'
|
||||
- 'docs/**'
|
||||
- 'LICENSE'
|
||||
|
||||
env:
|
||||
PYTHONUNBUFFERED: "1"
|
||||
|
||||
jobs:
|
||||
test:
|
||||
name: Test (Python ${{ matrix.python-version }}, ${{ matrix.os }})
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
python-version: ["3.10", "3.11"]
|
||||
os: [ubuntu-latest]
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python ${{ matrix.python-version }}
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
|
||||
- name: Cache pip dependencies
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: ~/.cache/pip
|
||||
key: ${{ runner.os }}-py${{ matrix.python-version }}-pip-${{ hashFiles('**/requirements.txt', '**/pyproject.toml') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-py${{ matrix.python-version }}-pip-
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install -e ".[test]"
|
||||
pip install -r requirements.txt
|
||||
|
||||
- name: List installed packages
|
||||
run: pip list
|
||||
|
||||
- name: Run tests with coverage
|
||||
run: |
|
||||
pytest test/ \
|
||||
-v \
|
||||
--cov=rdagent \
|
||||
--cov-report=xml \
|
||||
--cov-report=html \
|
||||
--cov-report=term-missing \
|
||||
--durations=10 \
|
||||
-x
|
||||
|
||||
- name: Upload coverage to Codecov
|
||||
if: matrix.python-version == '3.10' && github.ref == 'refs/heads/main'
|
||||
uses: codecov/codecov-action@v4
|
||||
with:
|
||||
file: ./coverage.xml
|
||||
flags: unittests
|
||||
name: codecov-umbrella
|
||||
fail_ci_if_error: false
|
||||
env:
|
||||
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
|
||||
|
||||
- name: Upload coverage HTML report
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: coverage-html-report-py${{ matrix.python-version }}
|
||||
path: htmlcov/
|
||||
retention-days: 7
|
||||
@@ -0,0 +1,83 @@
|
||||
name: Documentation
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ main ]
|
||||
paths:
|
||||
- 'docs/**'
|
||||
- 'README.md'
|
||||
- '**/*.rst'
|
||||
- '.github/workflows/docs.yml'
|
||||
pull_request:
|
||||
branches: [ main ]
|
||||
paths:
|
||||
- 'docs/**'
|
||||
- 'README.md'
|
||||
- '**/*.rst'
|
||||
|
||||
jobs:
|
||||
docs:
|
||||
name: Build Documentation
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.10"
|
||||
|
||||
- name: Cache pip dependencies
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: ~/.cache/pip
|
||||
key: ${{ runner.os }}-pip-docs-${{ hashFiles('**/pyproject.toml') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-pip-docs-
|
||||
|
||||
- name: Install docs dependencies
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install -e ".[docs]"
|
||||
|
||||
- name: Build Sphinx documentation
|
||||
run: |
|
||||
cd docs
|
||||
make clean
|
||||
make html SPHINXOPTS="-W --keep-going" || {
|
||||
echo "::error::Sphinx build failed with warnings"
|
||||
exit 1
|
||||
}
|
||||
|
||||
- name: Check for broken links
|
||||
run: |
|
||||
cd docs
|
||||
make linkcheck || {
|
||||
echo "::warning::Some links are broken (non-blocking)"
|
||||
exit 0
|
||||
}
|
||||
|
||||
- name: Upload docs artifact
|
||||
if: github.ref == 'refs/heads/main'
|
||||
uses: actions/upload-pages-artifact@v3
|
||||
with:
|
||||
path: docs/_build/html
|
||||
|
||||
deploy:
|
||||
name: Deploy to GitHub Pages
|
||||
needs: docs
|
||||
if: github.ref == 'refs/heads/main'
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
pages: write
|
||||
id-token: write
|
||||
environment:
|
||||
name: github-pages
|
||||
url: ${{ steps.deployment.outputs.page_url }}
|
||||
|
||||
steps:
|
||||
- name: Deploy to GitHub Pages
|
||||
id: deployment
|
||||
uses: actions/deploy-pages@v4
|
||||
@@ -0,0 +1,81 @@
|
||||
name: Code Quality
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ main, develop ]
|
||||
pull_request:
|
||||
branches: [ main ]
|
||||
|
||||
jobs:
|
||||
lint:
|
||||
name: Lint & Format
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.10"
|
||||
|
||||
- name: Cache pip dependencies
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: ~/.cache/pip
|
||||
key: ${{ runner.os }}-pip-lint-${{ hashFiles('**/pyproject.toml') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-pip-lint-
|
||||
|
||||
- name: Install lint dependencies
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install ruff mypy
|
||||
|
||||
- name: Run Ruff (linter)
|
||||
run: |
|
||||
echo "=== Running Ruff Linter ==="
|
||||
ruff check . --statistics || {
|
||||
echo "::error::Ruff linter found issues. Run: ruff check . --fix"
|
||||
exit 1
|
||||
}
|
||||
|
||||
- name: Run Ruff (formatter)
|
||||
run: |
|
||||
echo "=== Running Ruff Formatter ==="
|
||||
ruff format --check . || {
|
||||
echo "::error::Ruff formatter found issues. Run: ruff format ."
|
||||
exit 1
|
||||
}
|
||||
|
||||
- name: Run MyPy (type checker)
|
||||
run: |
|
||||
echo "=== Running MyPy Type Checker ==="
|
||||
mypy rdagent/ \
|
||||
--ignore-missing-imports \
|
||||
--no-strict-optional \
|
||||
--follow-imports=skip \
|
||||
--warn-return-any || {
|
||||
echo "::warning::MyPy found type issues (non-blocking)"
|
||||
# Non-blocking: MyPy warnings don't fail the build
|
||||
exit 0
|
||||
}
|
||||
|
||||
- name: Check for trailing whitespace
|
||||
run: |
|
||||
echo "=== Checking for trailing whitespace ==="
|
||||
if grep -rIn '[[:space:]]$' --include='*.py' --include='*.md' --include='*.rst' . | grep -v '.git'; then
|
||||
echo "::error::Found trailing whitespace. Please remove it."
|
||||
exit 1
|
||||
fi
|
||||
echo "✓ No trailing whitespace found"
|
||||
|
||||
- name: Check for merge conflicts
|
||||
run: |
|
||||
echo "=== Checking for merge conflict markers ==="
|
||||
if grep -rn '<<<<<<< HEAD\|=======\|>>>>>>>' --include='*.py' --include='*.md' . | grep -v '.git'; then
|
||||
echo "::error::Found merge conflict markers. Please resolve them."
|
||||
exit 1
|
||||
fi
|
||||
echo "✓ No merge conflict markers found"
|
||||
@@ -0,0 +1,97 @@
|
||||
name: Release
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- 'v*'
|
||||
- 'V*'
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
packages: write
|
||||
|
||||
jobs:
|
||||
release:
|
||||
name: Create Release
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.10"
|
||||
|
||||
- name: Install build tools
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install build twine
|
||||
|
||||
- name: Build package
|
||||
run: python -m build
|
||||
|
||||
- name: Check package
|
||||
run: twine check dist/*
|
||||
|
||||
- name: Verify no closed-source assets
|
||||
run: |
|
||||
echo "=== Verifying Release Package ==="
|
||||
|
||||
# Extract and check contents
|
||||
python -c "
|
||||
import tarfile
|
||||
import sys
|
||||
|
||||
with tarfile.open('dist/' + [f for f in __import__('os').listdir('dist') if f.endswith('.tar.gz')][0], 'r:gz') as tar:
|
||||
names = tar.getnames()
|
||||
closed_patterns = ['git_ignore_folder', 'results/', '.env', 'models/local', 'prompts/local']
|
||||
|
||||
found = False
|
||||
for name in names:
|
||||
for pattern in closed_patterns:
|
||||
if pattern in name:
|
||||
print(f'ERROR: Found closed-source asset: {name}')
|
||||
found = True
|
||||
|
||||
if found:
|
||||
sys.exit(1)
|
||||
print('✓ No closed-source assets in package')
|
||||
"
|
||||
|
||||
- name: Generate release notes
|
||||
id: release_notes
|
||||
run: |
|
||||
# Try to find changelog for this version
|
||||
VERSION=${GITHUB_REF#refs/tags/}
|
||||
CHANGELOG_FILE="changelog/${VERSION}.md"
|
||||
|
||||
if [ -f "$CHANGELOG_FILE" ]; then
|
||||
echo "body_path=$CHANGELOG_FILE" >> $GITHUB_OUTPUT
|
||||
elif [ -f "CHANGELOG.md" ]; then
|
||||
# Extract section for this version from main changelog
|
||||
echo "body_path=CHANGELOG.md" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "body_path=" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
|
||||
- name: Create GitHub Release
|
||||
uses: softprops/action-gh-release@v2
|
||||
with:
|
||||
body_path: ${{ steps.release_notes.outputs.body_path || '' }}
|
||||
files: dist/*
|
||||
draft: false
|
||||
prerelease: false
|
||||
generate_release_notes: true
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Publish to PyPI
|
||||
if: startsWith(github.ref, 'refs/tags/v')
|
||||
env:
|
||||
TWINE_USERNAME: __token__
|
||||
TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}
|
||||
run: twine upload dist/*
|
||||
@@ -0,0 +1,135 @@
|
||||
name: Security Scan
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ main ]
|
||||
pull_request:
|
||||
branches: [ main ]
|
||||
schedule:
|
||||
# Weekly on Monday at 6:00 UTC
|
||||
- cron: '0 6 * * 1'
|
||||
|
||||
jobs:
|
||||
security:
|
||||
name: Security Analysis
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.10"
|
||||
|
||||
- name: Cache pip dependencies
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: ~/.cache/pip
|
||||
key: ${{ runner.os }}-pip-security-${{ hashFiles('**/requirements.txt') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-pip-security-
|
||||
|
||||
- name: Install security tools
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install bandit safety
|
||||
|
||||
- name: Run Bandit (code security)
|
||||
run: |
|
||||
echo "=== Running Bandit Security Scan ==="
|
||||
bandit \
|
||||
-c .bandit.yml \
|
||||
-r rdagent/ \
|
||||
-f json \
|
||||
-o bandit-report.json \
|
||||
--exit-zero || true
|
||||
|
||||
# Show summary
|
||||
bandit -c .bandit.yml -r rdagent/ -ll || true
|
||||
|
||||
- name: Upload Bandit report
|
||||
uses: actions/upload-artifact@v4
|
||||
if: always()
|
||||
with:
|
||||
name: bandit-security-report
|
||||
path: bandit-report.json
|
||||
retention-days: 30
|
||||
|
||||
- name: Check dependencies for vulnerabilities
|
||||
run: |
|
||||
echo "=== Checking Dependencies for Vulnerabilities ==="
|
||||
safety check --json || {
|
||||
echo "::warning::Some dependencies have known vulnerabilities"
|
||||
echo "Please review and update dependencies."
|
||||
exit 0 # Non-blocking
|
||||
}
|
||||
|
||||
- name: Check for exposed secrets
|
||||
run: |
|
||||
echo "=== Scanning for Exposed Secrets ==="
|
||||
|
||||
# Check for common secret patterns
|
||||
PATTERNS=(
|
||||
"api_key\s*=\s*['\"][^'\"]+['\"]"
|
||||
"secret\s*=\s*['\"][^'\"]+['\"]"
|
||||
"password\s*=\s*['\"][^'\"]+['\"]"
|
||||
"token\s*=\s*['\"][^'\"]+['\"]"
|
||||
"PRIVATE.KEY"
|
||||
"BEGIN RSA PRIVATE KEY"
|
||||
)
|
||||
|
||||
FOUND_SECRETS=0
|
||||
for pattern in "${PATTERNS[@]}"; do
|
||||
if grep -rInE "$pattern" --include='*.py' --include='*.yml' --include='*.yaml' --include='*.json' . | \
|
||||
grep -v '.git' | \
|
||||
grep -v 'test/' | \
|
||||
grep -v 'example' | \
|
||||
grep -v '# ' | \
|
||||
grep -v 'os.environ' | \
|
||||
grep -v 'getenv' | \
|
||||
grep -v 'argparse'; then
|
||||
FOUND_SECRETS=1
|
||||
fi
|
||||
done
|
||||
|
||||
if [ $FOUND_SECRETS -eq 1 ]; then
|
||||
echo "::error::Potential secrets exposure detected!"
|
||||
echo "Please review the output above and remove any hardcoded credentials."
|
||||
echo "Use environment variables or .env files instead."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "✓ No exposed secrets found"
|
||||
|
||||
- name: Verify closed-source files not committed
|
||||
run: |
|
||||
echo "=== Verifying No Closed-Source Assets Committed ==="
|
||||
|
||||
CLOSED_PATTERNS=(
|
||||
"git_ignore_folder/"
|
||||
"results/"
|
||||
".env"
|
||||
"models/local/"
|
||||
"prompts/local/"
|
||||
"rdagent/scenarios/qlib/local/"
|
||||
"*.db"
|
||||
"*.log"
|
||||
)
|
||||
|
||||
FOUND_CLOSED=0
|
||||
for pattern in "${CLOSED_PATTERNS[@]}"; do
|
||||
if git ls-files | grep -q "$pattern"; then
|
||||
echo "::error::Found closed-source asset: $pattern"
|
||||
FOUND_CLOSED=1
|
||||
fi
|
||||
done
|
||||
|
||||
if [ $FOUND_CLOSED -eq 1 ]; then
|
||||
echo "CRITICAL: Closed-source assets must not be committed to the repository!"
|
||||
echo "Please remove them and add to .gitignore if needed."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "✓ No closed-source assets found"
|
||||
Reference in New Issue
Block a user