diff --git a/.agents/skills/local-qa/SKILL.md b/.agents/skills/local-qa/SKILL.md new file mode 100644 index 0000000..7c0dd86 --- /dev/null +++ b/.agents/skills/local-qa/SKILL.md @@ -0,0 +1,25 @@ +# local-qa + +Run local QA checks (format, lint, test) on the repository. + +## When to use + +After making changes to repository files, run `scripts/qa.sh` to validate formatting, linting, and tests. + +## Steps + +1. Execute `scripts/qa.sh` and capture the results. +2. Report successes, failures, warnings, and any modified files. + +## If tools are missing + +Install them following this priority order: + +1. Project package managers (`uv`, `poetry`, npm scripts) +2. System package managers (`brew`, `apt`) +3. Language-specific installers (`pipx`, `pip`, `npm`, `go install`) + +## Constraints + +- Only execute QA and tool-installation commands. +- If installation fails or requires unavailable privileges, report the attempt and exact failure, then stop. diff --git a/.agents/skills/local-qa/scripts/qa.sh b/.agents/skills/local-qa/scripts/qa.sh new file mode 100755 index 0000000..585728e --- /dev/null +++ b/.agents/skills/local-qa/scripts/qa.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +set -euox pipefail +cd "$(git rev-parse --show-toplevel)" + +# Python +uv run ruff format . +uv run ruff check --fix . +uv run pyright . +uv run pytest + +# Markdown +npx -y prettier --write './**/*.md' + +# GitHub Actions +zizmor --fix=safe .github/workflows +git ls-files -z -- '.github/workflows/*.yml' | xargs -0 -t actionlint +git ls-files -z -- '.github/workflows/*.yml' | xargs -0 -t yamllint -d '{"extends": "relaxed", "rules": {"line-length": "disable"}}' +checkov --framework=all --output=github_failed_only --directory=. diff --git a/.claude/agents/codex.md b/.claude/agents/codex.md new file mode 100644 index 0000000..8f353f4 --- /dev/null +++ b/.claude/agents/codex.md @@ -0,0 +1,46 @@ +# Codex Agent + +Specialized Claude agent for autonomous development work using OpenAI's Codex CLI. + +## Modes + +### Ask Mode + +Read-only code analysis: answer questions about implementation, architecture, and debugging with specific file references and code examples. + +### Exec Mode + +Generate and modify code: create new components, refactor existing code, fix bugs, and write tests while maintaining quality standards. + +### Review Mode + +Comprehensive code review: identify security vulnerabilities, bugs, performance issues, and quality improvements without making changes. + +### Search Mode + +Research current documentation, best practices, solutions, and technology comparisons using web resources. + +## Core Requirements + +- Prioritize Codex CLI as the primary execution engine. +- Ask, Review, and Search modes are read-only; only Exec mode modifies code. +- All answers require verification: + - Ask mode must confirm file paths exist. + - Exec mode requires test passage and linting. + - Review mode needs severity prioritization. + - Search mode demands sourced citations. + +## Workflow + +1. Understand the task. +2. Gather context from the codebase. +3. Execute via Codex CLI with specific parameters. +4. Verify results. +5. Communicate findings with appropriate structure and detail. + +## Constraints + +- No hardcoded secrets. +- Thorough testing. +- Specific file references. +- Honest communication about limitations. diff --git a/.claude/settings.json b/.claude/settings.json new file mode 100644 index 0000000..1c24547 --- /dev/null +++ b/.claude/settings.json @@ -0,0 +1,18 @@ +{ + "hooks": { + "Stop": [ + { + "matcher": "", + "hooks": [ + { + "type": "command", + "command": ".claude/skills/local-qa/scripts/qa.sh" + } + ] + } + ] + }, + "enabledPlugins": { + "code-simplifier@claude-plugins-official": true + } +} diff --git a/.claude/skills b/.claude/skills new file mode 120000 index 0000000..2b7a412 --- /dev/null +++ b/.claude/skills @@ -0,0 +1 @@ +../.agents/skills \ No newline at end of file diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 0000000..1fbf672 --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1,3 @@ +--- +github: + - dceoy diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..2bf387f --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,17 @@ +--- +version: 2 +updates: + - package-ecosystem: github-actions + directory: / + schedule: + interval: daily + cooldown: + default-days: 7 + open-pull-requests-limit: 10 + - package-ecosystem: pip + directory: / + schedule: + interval: daily + cooldown: + default-days: 7 + open-pull-requests-limit: 10 diff --git a/.github/renovate.json b/.github/renovate.json new file mode 100644 index 0000000..aa6e565 --- /dev/null +++ b/.github/renovate.json @@ -0,0 +1,13 @@ +{ + "$schema": "https://docs.renovatebot.com/renovate-schema.json", + "extends": ["config:recommended"], + "minimumReleaseAge": "7 days", + "packageRules": [ + { + "description": "Automatically merge minor and patch-level updates", + "matchUpdateTypes": ["minor", "patch", "digest"], + "automerge": true, + "automergeType": "branch" + } + ] +} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..15b5c8b --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,98 @@ +--- +name: CI/CD +on: + push: + branches: + - main + pull_request: + branches: + - main + types: + - opened + - synchronize + - reopened + workflow_dispatch: # checkov:skip=CKV_GHA_7:workflow_dispatch inputs are required for manual runs + inputs: + workflow: + required: true + type: choice + options: + - lint-and-test + - docs-deploy + - release + description: Choose the workflow to run + default: lint-and-test +permissions: + contents: read +defaults: + run: + shell: bash -euo pipefail {0} + working-directory: . +jobs: + python-lint-and-scan: + if: > + github.event_name == 'push' + || github.event_name == 'pull_request' + || (github.event_name == 'workflow_dispatch' && inputs.workflow == 'lint-and-test') + permissions: + contents: read + uses: dceoy/gh-actions-for-devops/.github/workflows/python-package-lint-and-scan.yml@main # zizmor: ignore[unpinned-uses] + with: + package-path: . + runs-on: windows-latest + python-test: + if: > + github.event_name == 'push' + || github.event_name == 'pull_request' + || (github.event_name == 'workflow_dispatch' && inputs.workflow == 'lint-and-test') + permissions: + contents: read + uses: dceoy/gh-actions-for-devops/.github/workflows/python-package-test.yml@main # zizmor: ignore[unpinned-uses] + with: + package-path: . + runs-on: windows-latest + python-docs-deploy: + if: > + github.event_name == 'push' + || ( + github.event_name == 'workflow_dispatch' + && (inputs.workflow == 'docs-deploy' || inputs.workflow == 'release') + ) + permissions: + contents: write + uses: dceoy/gh-actions-for-devops/.github/workflows/python-package-mkdocs-gh-deploy.yml@main # zizmor: ignore[unpinned-uses] + with: + package-path: . + runs-on: ubuntu-slim + secrets: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + python-package-release: + if: > + github.event_name == 'push' + || ( + github.event_name == 'workflow_dispatch' + && (inputs.workflow == 'release' || inputs.workflow == 'lint-and-test') + ) + permissions: + contents: write + id-token: write + uses: dceoy/gh-actions-for-devops/.github/workflows/python-package-release-on-pypi-and-github.yml@main # zizmor: ignore[unpinned-uses] + with: + package-path: . + create-releases: ${{ github.event_name == 'workflow_dispatch' && inputs.workflow == 'release' }} + secrets: + PYPI_API_TOKEN: ${{ secrets.PYPI_API_TOKEN }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + dependabot-auto-merge: + if: > + github.event_name == 'pull_request' && github.actor == 'dependabot[bot]' + needs: + - python-lint-and-scan + - python-test + uses: dceoy/gh-actions-for-devops/.github/workflows/dependabot-auto-merge.yml@main # zizmor: ignore[unpinned-uses] + permissions: + contents: write + pull-requests: write + actions: read + with: + unconditional: true diff --git a/.github/workflows/claude-code.yml b/.github/workflows/claude-code.yml new file mode 100644 index 0000000..40bc1e6 --- /dev/null +++ b/.github/workflows/claude-code.yml @@ -0,0 +1,57 @@ +--- +name: Claude Code review and mention bot +on: + pull_request: + branches: + - main + types: + - opened + - ready_for_review + issue_comment: + types: + - created + pull_request_review_comment: + types: + - created + issues: + types: + - opened + - assigned + pull_request_review: + types: + - submitted +permissions: + contents: read +jobs: + claude-code-review: + if: > + github.event_name == 'pull_request' + && github.event.pull_request.draft == false + && (! startsWith(github.head_ref, 'dependabot/')) + && (! startsWith(github.head_ref, 'renovate/')) + permissions: + contents: read + pull-requests: write + issues: write + id-token: write + actions: read + uses: dceoy/gh-actions-for-devops/.github/workflows/claude-code-review.yml@main # zizmor: ignore[unpinned-uses] + secrets: + CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} # zizmor: ignore[secrets-outside-env] + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + claude-code-bot: + if: > + (github.event_name == 'issue_comment' && contains(github.event.comment.body, '@claude')) + || (github.event_name == 'pull_request_review_comment' && contains(github.event.comment.body, '@claude')) + || (github.event_name == 'pull_request_review' && contains(github.event.review.body, '@claude')) + || (github.event_name == 'issues' && (contains(github.event.issue.body, '@claude') || contains(github.event.issue.title, '@claude'))) + permissions: + contents: read + pull-requests: write + issues: write + id-token: write + actions: read + uses: dceoy/gh-actions-for-devops/.github/workflows/claude-code-bot.yml@main # zizmor: ignore[unpinned-uses] + secrets: + CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} # zizmor: ignore[secrets-outside-env] + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..8852bd9 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,79 @@ +# Repository Guidelines + +## Commands + +### Development Setup + +```bash +uv sync +``` + +### Code Quality and Documentation + +**Important**: Run these before committing or creating a PR. + +1. **format, lint, and test**: Use `local-qa` skill. +2. **Documentation build** (if any public API changes): `uv run mkdocs build` + +## Architecture + +### Key Dependencies + +- **pdmt5**: Pandas-based data handler for MetaTrader 5 (core library) +- **typer**: CLI framework for building command-line interfaces +- **click**: Parameter type customization for CLI options +- **pandas**: Core data manipulation and analysis + +### Package Structure + +- `mt5cli/`: Main package directory + - `__init__.py`: Package initialization and exports (`detect_format`, `export_dataframe`) + - `cli.py`: CLI application with typer-based commands for data export + - `__main__.py`: Entry point for `python -m mt5cli` +- `tests/`: Comprehensive test suite (pytest-based) + - `test_cli.py`: Tests for CLI commands, parameter types, and export functions +- `docs/`: MkDocs documentation with API reference + - `docs/index.md`: Main documentation + - `docs/api/`: Auto-generated API documentation for all modules +- Modern Python packaging with `pyproject.toml` and uv dependency management + +### Quality Standards + +- Type hints required (pyright strict mode) +- Comprehensive linting with 35+ rule categories (ruff) +- Test coverage tracking with 100% (pytest-cov) +- Parametrized tests for input/result matrices using `pytest.mark.parametrize` (pytest) +- Test doubles (mocks, stubs) using `pytest_mock` for external dependencies (pytest-mock) +- Pydantic models for data validation and configuration + +### Documentation workflow + +1. Add Google-style docstrings to functions/classes +2. Local preview: `uv run mkdocs serve` +3. Build: `uv run mkdocs build` +4. Deploy: `uv run mkdocs gh-deploy` + +## Commit & Pull Request Guidelines + +- Run QA checks using `local-qa` skill before committing or creating a PR. +- Branch names use appropriate prefixes on creation (e.g., `feature/...`, `bugfix/...`, `refactor/...`, `docs/...`, `chore/...`). +- When instructed to create a PR, create it as a draft with appropriate labels by default. + +## Code Design Principles + +Always prefer the simplest design that works. + +- **KISS**: Choose straightforward solutions and avoid unnecessary abstraction. +- **DRY**: Remove duplication when it improves clarity and maintainability. +- **YAGNI**: Do not add features, hooks, or flexibility until they are needed. +- **SOLID/Clean Code**: Apply these as tools, only when they keep the design simpler and easier to change. + +## Development Methodology + +Keep delivery incremental, test-backed, and easy to review. + +- Make small, safe, reversible changes. +- Prefer `Red -> Green -> Refactor`. +- Do not mix feature work and refactoring in the same commit. +- Refactor when it improves clarity or removes real duplication (Rule of Three). +- Keep tests fast, focused, and self-validating. diff --git a/CLAUDE.md b/CLAUDE.md new file mode 120000 index 0000000..47dc3e3 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1 @@ +AGENTS.md \ No newline at end of file diff --git a/README.md b/README.md index d9031a2..12e1d39 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,85 @@ # mt5cli -Command-line tool for MetaTrader 5 + +[![CI/CD](https://github.com/dceoy/mt5cli/actions/workflows/ci.yml/badge.svg)](https://github.com/dceoy/mt5cli/actions/workflows/ci.yml) + +Command-line tool for exporting MetaTrader 5 data to CSV, JSON, Parquet, and SQLite3. + +Built on top of [pdmt5](https://github.com/dceoy/pdmt5), a pandas-based data handler for MetaTrader 5. + +## Features + +- **Multi-format export**: CSV, JSON, Parquet, and SQLite3 output formats +- **Auto-detection**: Format detection from file extensions +- **Comprehensive data access**: Rates, ticks, account info, symbols, orders, positions, and trading history +- **Flexible timeframes**: Named timeframes (M1, H1, D1, etc.) and numeric values +- **Connection management**: Optional credentials, server, and timeout configuration + +## Installation + +```bash +pip install -U mt5cli MetaTrader5 +``` + +## Usage + +```bash +# Export account information to CSV +mt5cli -o account.csv account-info + +# Export EURUSD M1 rates to Parquet +mt5cli -o rates.parquet rates-from --symbol EURUSD --timeframe M1 \ + --date-from 2024-01-01 --count 1000 + +# Export ticks to JSON +mt5cli -o ticks.json ticks-from --symbol EURUSD \ + --date-from 2024-01-01 --count 500 --flags ALL + +# Export symbols to SQLite3 with custom table name +mt5cli -o data.db --table symbols symbols --group "*USD*" + +# Export with connection credentials +mt5cli --login 12345 --password mypass --server MyBroker-Demo \ + -o positions.csv positions +``` + +Run as a Python module: + +```bash +python -m mt5cli -o account.csv account-info +``` + +## Commands + +| Command | Description | +|---------|-------------| +| `rates-from` | Export rates from a start date | +| `rates-from-pos` | Export rates from a start position | +| `rates-range` | Export rates for a date range | +| `ticks-from` | Export ticks from a start date | +| `ticks-range` | Export ticks for a date range | +| `account-info` | Export account information | +| `terminal-info` | Export terminal information | +| `symbols` | Export symbol list | +| `symbol-info` | Export symbol details | +| `orders` | Export active orders | +| `positions` | Export open positions | +| `history-orders` | Export historical orders | +| `history-deals` | Export historical deals | + +## Requirements + +- Python 3.11+ +- Windows OS (MetaTrader 5 requirement) +- MetaTrader 5 platform installed + +## Development + +```bash +git clone https://github.com/dceoy/mt5cli.git +cd mt5cli +uv sync +``` + +## License + +[MIT](LICENSE) diff --git a/docs/api/cli.md b/docs/api/cli.md new file mode 100644 index 0000000..89b885b --- /dev/null +++ b/docs/api/cli.md @@ -0,0 +1,3 @@ +# CLI Module + +::: mt5cli.cli diff --git a/docs/api/index.md b/docs/api/index.md new file mode 100644 index 0000000..8013438 --- /dev/null +++ b/docs/api/index.md @@ -0,0 +1,63 @@ +# API Reference + +This section contains the complete API documentation for mt5cli. + +## Modules + +The mt5cli package consists of the following modules: + +### [CLI](cli.md) + +Command-line interface module providing typer-based commands for exporting MetaTrader 5 data to CSV, JSON, Parquet, and SQLite3 formats. + +## Architecture Overview + +The package follows a simple architecture built on top of pdmt5: + +1. **CLI Layer** (`cli.py`): Typer application with subcommands for each data type, custom Click parameter types for datetime/timeframe/tick flags parsing, and format detection/export utilities. +2. **Data Layer** (via `pdmt5`): Uses `Mt5DataClient` and `Mt5Config` from the pdmt5 package for all MetaTrader 5 data access. + +## Usage Guidelines + +All modules follow these conventions: + +- **Type Safety**: All functions include comprehensive type hints +- **Error Handling**: User-friendly error messages via typer +- **Documentation**: Google-style docstrings with examples +- **Validation**: Custom Click parameter types for input validation + +## Quick Start + +```bash +# Export account information to CSV +mt5cli -o account.csv account-info + +# Export EURUSD H1 rates to Parquet +mt5cli -o rates.parquet rates-from --symbol EURUSD --timeframe H1 \ + --date-from 2024-01-01 --count 1000 + +# Export ticks to JSON +mt5cli -o ticks.json ticks-from --symbol EURUSD \ + --date-from 2024-01-01 --count 500 --flags ALL + +# Export to SQLite3 with custom table name +mt5cli -o data.db --table symbols symbols --group "*USD*" +``` + +## Python API + +```python +from mt5cli import detect_format, export_dataframe +import pandas as pd + +# Detect output format from file extension +fmt = detect_format(Path("output.parquet")) # Returns "parquet" + +# Export a DataFrame +df = pd.DataFrame({"symbol": ["EURUSD"], "bid": [1.1234]}) +export_dataframe(df, Path("output.csv"), "csv") +``` + +## Examples + +See individual module pages for detailed usage examples and code samples. diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000..bba731f --- /dev/null +++ b/docs/index.md @@ -0,0 +1,117 @@ +# mt5cli + +Command-line tool for MetaTrader 5 data export. + +## Overview + +mt5cli is a CLI application that exports MetaTrader 5 trading data to multiple file formats. It is built on top of [pdmt5](https://github.com/dceoy/pdmt5), a pandas-based data handler for MetaTrader 5. + +## Features + +- **Multi-format export**: CSV, JSON, Parquet, and SQLite3 output formats +- **Auto-detection**: Format detection from file extensions +- **Comprehensive data access**: Rates, ticks, account info, symbols, orders, positions, and trading history +- **Flexible timeframes**: Named timeframes (M1, H1, D1, etc.) and numeric values +- **Connection management**: Optional credentials, server, and timeout configuration + +## Installation + +```bash +pip install mt5cli +``` + +## Quick Start + +```bash +# Export account information to CSV +mt5cli -o account.csv account-info + +# Export EURUSD M1 rates to Parquet +mt5cli -o rates.parquet rates-from --symbol EURUSD --timeframe M1 \ + --date-from 2024-01-01 --count 1000 + +# Export ticks to JSON +mt5cli -o ticks.json ticks-from --symbol EURUSD \ + --date-from 2024-01-01 --count 500 --flags ALL + +# Export symbols to SQLite3 with custom table name +mt5cli -o data.db --table symbols symbols --group "*USD*" + +# Export with connection credentials +mt5cli --login 12345 --password mypass --server MyBroker-Demo \ + -o positions.csv positions +``` + +## Commands + +### Rates + +| Command | Description | +|---------|-------------| +| `rates-from` | Export rates from a start date | +| `rates-from-pos` | Export rates from a start position | +| `rates-range` | Export rates for a date range | + +### Ticks + +| Command | Description | +|---------|-------------| +| `ticks-from` | Export ticks from a start date | +| `ticks-range` | Export ticks for a date range | + +### Information + +| Command | Description | +|---------|-------------| +| `account-info` | Export account information | +| `terminal-info` | Export terminal information | +| `symbols` | Export symbol list | +| `symbol-info` | Export symbol details | + +### Trading + +| Command | Description | +|---------|-------------| +| `orders` | Export active orders | +| `positions` | Export open positions | +| `history-orders` | Export historical orders | +| `history-deals` | Export historical deals | + +## Global Options + +| Option | Description | +|--------|-------------| +| `-o, --output` | Output file path (required) | +| `-f, --format` | Output format (auto-detected from extension if omitted) | +| `--table` | Table name for SQLite3 output (default: "data") | +| `--login` | Trading account login | +| `--password` | Trading account password | +| `--server` | Trading server name | +| `--path` | Path to MetaTrader5 terminal EXE file | +| `--timeout` | Connection timeout in milliseconds | +| `--log-level` | Logging level (DEBUG, INFO, WARNING, ERROR) | + +## Requirements + +- Python 3.11+ +- Windows OS (MetaTrader 5 requirement) +- MetaTrader 5 platform + +## API Reference + +Browse the API documentation for detailed module information: + +- [CLI Module](api/cli.md) - CLI application with export commands and utility functions + +## Development + +This project follows strict code quality standards: + +- Type hints required (strict mode) +- Comprehensive linting with Ruff +- Test coverage tracking +- Google-style docstrings + +## License + +MIT License - see [LICENSE](https://github.com/dceoy/mt5cli/blob/main/LICENSE) file for details. diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 0000000..6a128b6 --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,71 @@ +site_name: mt5cli API Documentation +site_description: Command-line tool for MetaTrader 5 +site_author: dceoy +site_url: https://github.com/dceoy/mt5cli + +repo_name: dceoy/mt5cli +repo_url: https://github.com/dceoy/mt5cli + +theme: + name: material + palette: + - scheme: default + primary: blue + accent: blue + toggle: + icon: material/brightness-7 + name: Switch to dark mode + - scheme: slate + primary: blue + accent: blue + toggle: + icon: material/brightness-4 + name: Switch to light mode + features: + - content.code.annotate + - content.code.copy + - navigation.indexes + - navigation.sections + - navigation.tabs + - navigation.top + - search.highlight + - search.share + - search.suggest + - toc.follow + +plugins: + - search + - mkdocstrings: + handlers: + python: + paths: [.] + options: + show_source: true + show_root_heading: true + show_root_toc_entry: true + docstring_style: google + docstring_section_style: table + separate_signature: true + show_signature_annotations: true + signature_crossrefs: true + merge_init_into_class: true + show_if_no_docstring: true + +nav: + - Home: index.md + - API Reference: + - Overview: api/index.md + - CLI: api/cli.md + +markdown_extensions: + - admonition + - pymdownx.details + - pymdownx.superfences + - pymdownx.highlight: + anchor_linenums: true + - pymdownx.inlinehilite + - pymdownx.snippets + - pymdownx.tabbed: + alternate_style: true + - toc: + permalink: true