diff --git a/README.md b/README.md index 4569720..1271ccd 100644 --- a/README.md +++ b/README.md @@ -50,30 +50,45 @@ python -m mt5cli -o account.csv account-info ## Commands -| Command | Description | -| ------------------ | ----------------------------------------------------------- | -| `rates-from` | Export rates from a start date | -| `rates-from-pos` | Export rates from a start position | -| `rates-range` | Export rates for a date range | -| `ticks-from` | Export ticks from a start date | -| `ticks-range` | Export ticks for a date range | -| `account-info` | Export account information | -| `terminal-info` | Export terminal information | -| `version` | Export MetaTrader 5 version information | -| `last-error` | Export the last error information | -| `symbols` | Export symbol list | -| `symbol-info` | Export symbol details | -| `symbol-info-tick` | Export the last tick for a symbol | -| `market-book` | Export market depth (order book) | -| `orders` | Export active orders | -| `positions` | Export open positions | -| `history-orders` | Export historical orders | -| `history-deals` | Export historical deals | -| `order-check` | Check funds sufficiency for a trade request | -| `order-send` | Send a trade request to the trade server (`--yes` required) | +| Command | Description | +| ------------------ | ------------------------------------------------------------------------------------------------------------ | +| `rates-from` | Export rates from a start date | +| `rates-from-pos` | Export rates from a start position | +| `rates-range` | Export rates for a date range | +| `ticks-from` | Export ticks from a start date | +| `ticks-range` | Export ticks for a date range | +| `account-info` | Export account information | +| `terminal-info` | Export terminal information | +| `version` | Export MetaTrader 5 version information | +| `last-error` | Export the last error information | +| `symbols` | Export symbol list | +| `symbol-info` | Export symbol details | +| `symbol-info-tick` | Export the last tick for a symbol | +| `market-book` | Export market depth (order book) | +| `orders` | Export active orders | +| `positions` | Export open positions | +| `history-orders` | Export historical orders | +| `history-deals` | Export historical deals | +| `order-check` | Check funds sufficiency for a trade request | +| `order-send` | Send a trade request to the trade server (`--yes` required) | +| `collect-history` | Bundle rates, ticks, history-orders, and history-deals for one or more symbols into a single SQLite database | Use `order-check` to validate a request payload before running `order-send --yes`. +### `collect-history` + +Collect several historical datasets per symbol into one SQLite database in a single MT5 session. Pick datasets with repeatable `--dataset` (default: all four), choose conflict behavior with `--if-exists append|replace|fail` (default: `fail`), and optionally derive `cash_events` / `positions_reconstructed` views from `history_deals` via `--with-views`. + +```bash +mt5cli -o history.db collect-history \ + --symbol EURUSD --symbol GBPUSD \ + --date-from 2024-01-01 --date-to 2024-02-01 \ + --dataset rates --dataset history-deals \ + --timeframe M1 --flags ALL --if-exists append --with-views +``` + +History orders and deals are fetched per symbol and concatenated, so the symbol filter is applied consistently across all datasets. The `cash_events` view is derived from symbol-filtered `history_deals`, so account-level cash events with empty or non-matching symbols may be excluded. The `rates` table records the requested `timeframe` so appended runs at different timeframes remain distinguishable. The `positions_reconstructed` view aggregates trade deals by `position_id`, excludes positions without closing deals, and uses volume-weighted open/close prices; reversal deals (`DEAL_ENTRY_INOUT`) are reported via `volume_reversal` / `reversal_count` columns and do not contribute to the weighted prices. + ## Requirements - Python 3.11+ diff --git a/docs/index.md b/docs/index.md index 9c2cdc4..605bff8 100644 --- a/docs/index.md +++ b/docs/index.md @@ -85,6 +85,35 @@ mt5cli --login 12345 --password mypass --server MyBroker-Demo \ Use `order-check` to validate a request payload before running `order-send --yes`. +### Bulk Collection + +| Command | Description | +| ----------------- | -------------------------------------------------------------------------------------------------------------------------------------------------- | +| `collect-history` | Collect rates, ticks, history-orders, and history-deals for one or more symbols into a single SQLite database (optional cash-event/position views) | + +```bash +mt5cli -o history.db collect-history \ + --symbol EURUSD --symbol GBPUSD \ + --date-from 2024-01-01 --date-to 2024-02-01 \ + --dataset rates --dataset history-deals \ + --timeframe M1 --flags ALL --if-exists append --with-views +``` + +`collect-history` options: + +| Option | Default | Description | +| -------------- | ---------- | --------------------------------------------------------------------------------------------- | +| `--symbol/-s` | _required_ | Symbol to collect (repeat for multiple). | +| `--date-from` | _required_ | Start date in ISO 8601. | +| `--date-to` | _required_ | End date in ISO 8601. | +| `--dataset` | all four | Repeatable: `rates`, `ticks`, `history-orders`, `history-deals`. | +| `--timeframe` | `M1` | Rates timeframe; recorded in a `timeframe` column on the `rates` table. | +| `--flags` | `ALL` | Tick copy flags forwarded to `copy_ticks_range`. | +| `--if-exists` | `fail` | `append`, `replace`, or `fail` when a target table already exists. | +| `--with-views` | off | Add `cash_events` and `positions_reconstructed` views (requires the `history-deals` dataset). | + +History orders and deals are fetched per symbol and concatenated, so the symbol filter is applied consistently across all datasets. The `cash_events` view is derived from symbol-filtered `history_deals`, so account-level cash events with empty or non-matching symbols may be excluded. The `positions_reconstructed` view excludes positions with no closing deal, uses volume-weighted open/close prices, and reports reversal deals (`DEAL_ENTRY_INOUT`) via `volume_reversal` / `reversal_count`. + ## Global Options | Option | Description | diff --git a/mt5cli/cli.py b/mt5cli/cli.py index d76cf17..24a24e1 100644 --- a/mt5cli/cli.py +++ b/mt5cli/cli.py @@ -56,6 +56,19 @@ TICK_FLAG_MAP: dict[str, int] = { "TRADE": 4, } +_TRADE_DEAL_TYPES: tuple[int, int] = (0, 1) +_TRADE_DEAL_TYPES_SQL = f"({', '.join(str(value) for value in _TRADE_DEAL_TYPES)})" +_POSITIONS_VIEW_REQUIRED_COLUMNS: frozenset[str] = frozenset({ + "position_id", + "symbol", + "time", + "type", + "entry", + "volume", + "price", + "profit", +}) + _FORMAT_EXTENSIONS: dict[str, str] = { ".csv": "csv", ".json": "json", @@ -89,6 +102,31 @@ class LogLevel(StrEnum): ERROR = "ERROR" +class Dataset(StrEnum): + """Datasets supported by the ``collect-history`` command.""" + + rates = "rates" + ticks = "ticks" + history_orders = "history-orders" + history_deals = "history-deals" + + +class IfExists(StrEnum): + """SQLite table conflict behavior for the ``collect-history`` command.""" + + APPEND = "append" + REPLACE = "replace" + FAIL = "fail" + + +_DATASET_TABLE_NAMES: dict[Dataset, str] = { + Dataset.rates: "rates", + Dataset.ticks: "ticks", + Dataset.history_orders: "history_orders", + Dataset.history_deals: "history_deals", +} + + # --------------------------------------------------------------------------- # Click parameter types # --------------------------------------------------------------------------- @@ -894,6 +932,521 @@ def order_send( ) +def _create_cash_events_view( + conn: sqlite3.Connection, + deals_columns: set[str], +) -> bool: + """Create the cash_events SQLite view derived from history_deals. + + Args: + conn: Open SQLite connection. + deals_columns: Column names present in the history_deals table. + + Returns: + True if the view was created, False if required columns are missing. + """ + if "type" not in deals_columns: + logger.warning("Skipping cash_events view: history_deals.type is missing") + return False + conn.execute("DROP VIEW IF EXISTS cash_events") + conn.execute( + "CREATE VIEW cash_events AS" # noqa: S608 + f" SELECT * FROM history_deals WHERE type NOT IN {_TRADE_DEAL_TYPES_SQL}", + ) + return True + + +def _create_positions_reconstructed_view( + conn: sqlite3.Connection, + deals_columns: set[str], +) -> bool: + """Create the positions_reconstructed SQLite view derived from history_deals. + + The view aggregates trade deals (``type IN (0, 1)``) by ``position_id`` and + excludes positions that have no closing deal (``entry IN (1, 3)``), so + still-open positions and reversal-only fragments are filtered out. + + Open/close prices are volume-weighted averages over the corresponding + entry deals. Reversal deals (``DEAL_ENTRY_INOUT = 2``) are reported via + ``volume_reversal`` and ``reversal_count``; they do not contribute to the + open or close volume/price weights because a single reversal deal mixes a + close of the existing direction with the open of the new direction. + + Args: + conn: Open SQLite connection. + deals_columns: Column names present in the history_deals table. + + Returns: + True if the view was created, False if required columns are missing. + """ + if not _POSITIONS_VIEW_REQUIRED_COLUMNS.issubset(deals_columns): + missing = ", ".join(sorted(_POSITIONS_VIEW_REQUIRED_COLUMNS - deals_columns)) + logger.warning( + "Skipping positions_reconstructed view: history_deals missing columns: %s", + missing, + ) + return False + conn.execute("DROP VIEW IF EXISTS positions_reconstructed") + conn.execute( + "CREATE VIEW positions_reconstructed AS" # noqa: S608 + " SELECT" + " position_id," + " symbol," + " MIN(CASE WHEN entry = 0 THEN time END) AS open_time," + " MAX(CASE WHEN entry IN (1, 2, 3) THEN time END) AS close_time," + " MIN(CASE WHEN entry = 0 THEN type END) AS direction," + " SUM(CASE WHEN entry = 0 THEN volume ELSE 0 END) AS volume_open," + " SUM(CASE WHEN entry IN (1, 3) THEN volume ELSE 0 END) AS volume_close," + " SUM(CASE WHEN entry = 2 THEN volume ELSE 0 END) AS volume_reversal," + " CASE" + " WHEN SUM(CASE WHEN entry = 0 THEN volume ELSE 0 END) > 0" + " THEN SUM(CASE WHEN entry = 0 THEN price * volume ELSE 0 END)" + " / SUM(CASE WHEN entry = 0 THEN volume ELSE 0 END)" + " END AS open_price," + " CASE" + " WHEN SUM(CASE WHEN entry IN (1, 3) THEN volume ELSE 0 END) > 0" + " THEN SUM(CASE WHEN entry IN (1, 3) THEN price * volume ELSE 0 END)" + " / SUM(CASE WHEN entry IN (1, 3) THEN volume ELSE 0 END)" + " END AS close_price," + " SUM(profit) AS total_profit," + " SUM(CASE WHEN entry = 2 THEN 1 ELSE 0 END) AS reversal_count," + " COUNT(*) AS deals_count" + " FROM history_deals" + f" WHERE type IN {_TRADE_DEAL_TYPES_SQL} AND position_id != 0" + " GROUP BY position_id, symbol" + " HAVING SUM(CASE WHEN entry IN (1, 3) THEN 1 ELSE 0 END) > 0", + ) + return True + + +def _write_frame_to_sqlite( + conn: sqlite3.Connection, + frame: pd.DataFrame, + table_name: str, + if_exists: IfExists, +) -> bool: + """Write a non-empty-schema frame to SQLite. + + Args: + conn: Open SQLite connection. + frame: DataFrame to write. + table_name: Target SQLite table name. + if_exists: Table conflict behavior. + + Returns: + True if a table was written, False if the frame had no columns. + """ + if len(frame.columns) == 0: + logger.warning("Skipping %s: dataset returned no columns", table_name) + return False + frame.to_sql( # type: ignore[reportUnknownMemberType] + table_name, + conn, + if_exists=if_exists.value, + index=False, + chunksize=50_000, + method="multi", + ) + return True + + +def _create_collect_history_indexes( + conn: sqlite3.Connection, + written_columns: dict[Dataset, set[str]], +) -> None: + """Create useful indexes for collected history tables when present.""" + if {"symbol", "time"}.issubset(written_columns.get(Dataset.rates, set())): + conn.execute( + "CREATE INDEX IF NOT EXISTS idx_rates_symbol_time ON rates(symbol, time)", + ) + if {"symbol", "time"}.issubset(written_columns.get(Dataset.ticks, set())): + conn.execute( + "CREATE INDEX IF NOT EXISTS idx_ticks_symbol_time ON ticks(symbol, time)", + ) + if {"position_id", "symbol"}.issubset( + written_columns.get(Dataset.history_deals, set()) + ): + conn.execute( + "CREATE INDEX IF NOT EXISTS idx_history_deals_position_symbol" + " ON history_deals(position_id, symbol)", + ) + + +def _record_written_columns( + written_columns: dict[Dataset, set[str]], + dataset: Dataset, + frame: pd.DataFrame, +) -> None: + """Remember columns for datasets written during streaming collection.""" + columns = set(frame.columns) + if dataset in written_columns: + written_columns[dataset].update(columns) + else: + written_columns[dataset] = columns + + +def _write_streamed_frame( + conn: sqlite3.Connection, + frame: pd.DataFrame, + dataset: Dataset, + table_exists: bool, + if_exists: IfExists, + written_columns: dict[Dataset, set[str]], +) -> bool: + """Write one streamed dataset frame and track table state. + + Args: + conn: Open SQLite connection. + frame: DataFrame to write. + dataset: Dataset being written. + table_exists: Whether this dataset table has already been written. + if_exists: Initial table conflict behavior. + written_columns: Mutable map of columns written by dataset. + + Returns: + True if the dataset table exists after this write attempt. + """ + write_mode = IfExists.APPEND if table_exists else if_exists + if _write_frame_to_sqlite( + conn, + frame, + _DATASET_TABLE_NAMES[dataset], + write_mode, + ): + _record_written_columns(written_columns, dataset, frame) + return True + return table_exists + + +def _write_rates_dataset( + conn: sqlite3.Connection, + client: Mt5DataClient, + symbols: list[str], + timeframe: int, + date_from: datetime, + date_to: datetime, + if_exists: IfExists, + written_columns: dict[Dataset, set[str]], +) -> bool: + """Stream rates frames into SQLite. + + Args: + conn: Open SQLite connection. + client: Connected MT5 data client. + symbols: Symbols to collect. + timeframe: Rates timeframe integer. + date_from: Start date. + date_to: End date. + if_exists: Initial table conflict behavior. + written_columns: Mutable map of columns written by dataset. + + Returns: + True if the rates table was written. + """ + table_exists = False + for sym in symbols: + frame = client.copy_rates_range_as_df( + symbol=sym, + timeframe=timeframe, + date_from=date_from, + date_to=date_to, + ) + frame.insert(0, "symbol", sym) + frame.insert(1, "timeframe", timeframe) + table_exists = _write_streamed_frame( + conn, + frame, + Dataset.rates, + table_exists, + if_exists, + written_columns, + ) + return table_exists + + +def _write_ticks_dataset( + conn: sqlite3.Connection, + client: Mt5DataClient, + symbols: list[str], + flags: int, + date_from: datetime, + date_to: datetime, + if_exists: IfExists, + written_columns: dict[Dataset, set[str]], +) -> bool: + """Stream ticks frames into SQLite. + + Args: + conn: Open SQLite connection. + client: Connected MT5 data client. + symbols: Symbols to collect. + flags: Tick copy flags integer. + date_from: Start date. + date_to: End date. + if_exists: Initial table conflict behavior. + written_columns: Mutable map of columns written by dataset. + + Returns: + True if the ticks table was written. + """ + table_exists = False + for sym in symbols: + frame = client.copy_ticks_range_as_df( + symbol=sym, + date_from=date_from, + date_to=date_to, + flags=flags, + ) + frame.insert(0, "symbol", sym) + table_exists = _write_streamed_frame( + conn, + frame, + Dataset.ticks, + table_exists, + if_exists, + written_columns, + ) + return table_exists + + +def _write_history_dataset( + conn: sqlite3.Connection, + fetch: Callable[..., pd.DataFrame], + dataset: Dataset, + symbols: list[str], + date_from: datetime, + date_to: datetime, + if_exists: IfExists, + written_columns: dict[Dataset, set[str]], +) -> bool: + """Stream a history dataset into SQLite with exact symbol filtering. + + Args: + conn: Open SQLite connection. + fetch: Bound history_orders_get_as_df / history_deals_get_as_df method. + dataset: History dataset being written. + symbols: Symbols to collect. + date_from: Start date. + date_to: End date. + if_exists: Initial table conflict behavior. + written_columns: Mutable map of columns written by dataset. + + Returns: + True if the history table was written. + """ + table_exists = False + for sym in symbols: + frame = fetch(date_from=date_from, date_to=date_to, symbol=sym) + if "symbol" in frame.columns: + frame = frame[frame["symbol"] == sym] + table_exists = _write_streamed_frame( + conn, + frame, + dataset, + table_exists, + if_exists, + written_columns, + ) + return table_exists + + +def _write_collected_datasets( + conn: sqlite3.Connection, + client: Mt5DataClient, + symbols: list[str], + datasets: set[Dataset], + timeframe: int, + flags: int, + date_from: datetime, + date_to: datetime, + if_exists: IfExists, +) -> tuple[set[Dataset], dict[Dataset, set[str]]]: + """Collect selected datasets and stream each symbol frame into SQLite. + + Args: + conn: Open SQLite connection. + client: Connected MT5 data client. + symbols: Symbols to collect. + datasets: Selected datasets to write. + timeframe: Rates timeframe integer. + flags: Tick copy flags integer. + date_from: Start date. + date_to: End date. + if_exists: Initial table conflict behavior. + + Returns: + Written datasets and their columns. + """ + written_columns: dict[Dataset, set[str]] = {} + written_tables: set[Dataset] = set() + if Dataset.rates in datasets and _write_rates_dataset( + conn, + client, + symbols, + timeframe, + date_from, + date_to, + if_exists, + written_columns, + ): + written_tables.add(Dataset.rates) + if Dataset.ticks in datasets and _write_ticks_dataset( + conn, + client, + symbols, + flags, + date_from, + date_to, + if_exists, + written_columns, + ): + written_tables.add(Dataset.ticks) + if Dataset.history_orders in datasets and _write_history_dataset( + conn, + client.history_orders_get_as_df, + Dataset.history_orders, + symbols, + date_from, + date_to, + if_exists, + written_columns, + ): + written_tables.add(Dataset.history_orders) + if Dataset.history_deals in datasets and _write_history_dataset( + conn, + client.history_deals_get_as_df, + Dataset.history_deals, + symbols, + date_from, + date_to, + if_exists, + written_columns, + ): + written_tables.add(Dataset.history_deals) + return written_tables, written_columns + + +@app.command() +def collect_history( + ctx: typer.Context, + symbol: Annotated[ + list[str], + typer.Option( + "--symbol", + "-s", + help="Symbol to collect (repeat for multiple symbols).", + ), + ], + date_from: Annotated[ + datetime, + typer.Option(click_type=DATETIME_TYPE, help="Start date."), + ], + date_to: Annotated[ + datetime, + typer.Option(click_type=DATETIME_TYPE, help="End date."), + ], + dataset: Annotated[ + list[Dataset] | None, + typer.Option( + "--dataset", + help=( + "Dataset to include (repeat for multiple)." + " Defaults to all: rates, ticks, history-orders, history-deals." + ), + ), + ] = None, + timeframe: Annotated[ + int, + typer.Option( + click_type=TIMEFRAME_TYPE, + help="Rates timeframe (e.g., M1, H1, D1).", + ), + ] = 1, + flags: Annotated[ + int, + typer.Option( + click_type=TICK_FLAGS_TYPE, + help="Tick copy flags (ALL, INFO, TRADE, or integer).", + ), + ] = 1, + if_exists: Annotated[ + IfExists, + typer.Option( + "--if-exists", + help="Behavior when a target table already exists.", + ), + ] = IfExists.FAIL, + with_views: Annotated[ + bool, + typer.Option( + "--with-views", + help=( + "Add cash_events and positions_reconstructed SQLite views" + " derived from history_deals." + ), + ), + ] = False, +) -> None: + """Collect historical datasets into a single SQLite database. + + Tables written depend on ``--dataset``: ``rates``, ``ticks``, + ``history_orders``, ``history_deals``. History datasets are fetched per + symbol and concatenated. Rates rows carry the requested ``timeframe`` so + appended runs at different timeframes remain distinguishable. + + With ``--with-views`` (requires the ``history-deals`` dataset), optional + views ``cash_events`` and ``positions_reconstructed`` are derived from + ``history_deals`` when the required columns are present. + + Raises: + typer.BadParameter: If the output format is not SQLite3. + """ + export_ctx = _get_export_context(ctx) + if export_ctx.output_format != "sqlite3": + msg = ( + "collect-history requires SQLite3 output." + " Use a .db/.sqlite/.sqlite3 extension or --format sqlite3." + ) + raise typer.BadParameter(msg) + datasets = set(dataset) if dataset else set(Dataset) + client = Mt5DataClient(config=export_ctx.config) + client.initialize_and_login_mt5() + try: + with sqlite3.connect(export_ctx.output) as conn: + conn.execute("PRAGMA journal_mode=WAL") + conn.execute("PRAGMA synchronous=NORMAL") + written_tables, written_columns = _write_collected_datasets( + conn, + client, + symbol, + datasets, + timeframe, + flags, + date_from, + date_to, + if_exists, + ) + _create_collect_history_indexes(conn, written_columns) + if with_views and Dataset.history_deals in written_tables: + _create_cash_events_view(conn, written_columns[Dataset.history_deals]) + _create_positions_reconstructed_view( + conn, + written_columns[Dataset.history_deals], + ) + elif with_views: + logger.warning( + "--with-views ignored: history_deals table was not written" + ) + logger.info( + "Collected %s for %d symbol(s) into %s", + ", ".join(sorted(ds.value for ds in datasets)), + len(symbol), + export_ctx.output, + ) + finally: + client.shutdown() + + def main() -> None: """Run the mt5cli CLI.""" app() diff --git a/pyproject.toml b/pyproject.toml index 15d8f6e..3ecd22b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "mt5cli" -version = "0.2.0" +version = "0.3.0" description = "Command-line tool for MetaTrader 5" authors = [{name = "dceoy", email = "dceoy@users.noreply.github.com"}] maintainers = [{name = "dceoy", email = "dceoy@users.noreply.github.com"}] diff --git a/skills/mt5cli/SKILL.md b/skills/mt5cli/SKILL.md index 83f26e3..5882671 100644 --- a/skills/mt5cli/SKILL.md +++ b/skills/mt5cli/SKILL.md @@ -50,21 +50,22 @@ Global options MUST precede the subcommand. ## Commands -| Command | Required options | Optional options | -| ---------------- | ----------------------------------------------------- | --------------------------------------------------------------------------- | -| `rates-from` | `--symbol`, `--timeframe`, `--date-from`, `--count` | — | -| `rates-from-pos` | `--symbol`, `--timeframe`, `--start-pos`, `--count` | — | -| `rates-range` | `--symbol`, `--timeframe`, `--date-from`, `--date-to` | — | -| `ticks-from` | `--symbol`, `--date-from`, `--count`, `--flags` | — | -| `ticks-range` | `--symbol`, `--date-from`, `--date-to`, `--flags` | — | -| `account-info` | — | — | -| `terminal-info` | — | — | -| `symbols` | — | `--group` (e.g., `*USD*`) | -| `symbol-info` | `--symbol` | — | -| `orders` | — | `--symbol`, `--group`, `--ticket` | -| `positions` | — | `--symbol`, `--group`, `--ticket` | -| `history-orders` | — | `--date-from`, `--date-to`, `--group`, `--symbol`, `--ticket`, `--position` | -| `history-deals` | — | `--date-from`, `--date-to`, `--group`, `--symbol`, `--ticket`, `--position` | +| Command | Required options | Optional options | +| ----------------- | ----------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `rates-from` | `--symbol`, `--timeframe`, `--date-from`, `--count` | — | +| `rates-from-pos` | `--symbol`, `--timeframe`, `--start-pos`, `--count` | — | +| `rates-range` | `--symbol`, `--timeframe`, `--date-from`, `--date-to` | — | +| `ticks-from` | `--symbol`, `--date-from`, `--count`, `--flags` | — | +| `ticks-range` | `--symbol`, `--date-from`, `--date-to`, `--flags` | — | +| `account-info` | — | — | +| `terminal-info` | — | — | +| `symbols` | — | `--group` (e.g., `*USD*`) | +| `symbol-info` | `--symbol` | — | +| `orders` | — | `--symbol`, `--group`, `--ticket` | +| `positions` | — | `--symbol`, `--group`, `--ticket` | +| `history-orders` | — | `--date-from`, `--date-to`, `--group`, `--symbol`, `--ticket`, `--position` | +| `history-deals` | — | `--date-from`, `--date-to`, `--group`, `--symbol`, `--ticket`, `--position` | +| `collect-history` | `--symbol` (repeatable), `--date-from`, `--date-to` | `--dataset` (repeatable; rates/ticks/history-orders/history-deals; default all), `--timeframe` (M1; recorded on rates), `--flags` (ALL), `--if-exists` (append/replace/fail; default fail), `--with-views` (SQLite3 output only) | ## Examples @@ -85,6 +86,14 @@ mt5cli -o data.db --table symbols symbols --group "*USD*" # Historical deals filtered by symbol (using an already-logged-in MT5 terminal). mt5cli -o deals.csv history-deals --symbol EURUSD --date-from 2024-01-01 + +# Bundle selected historical datasets into one SQLite db, appending to any +# existing tables, plus cash_events and positions_reconstructed views. +mt5cli -o history.db collect-history \ + --symbol EURUSD --symbol GBPUSD \ + --date-from 2024-01-01 --date-to 2024-02-01 \ + --dataset rates --dataset history-deals \ + --timeframe M1 --flags ALL --if-exists append --with-views ``` ## Guidelines diff --git a/tests/test_cli.py b/tests/test_cli.py index bd28cd4..70251e2 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -3,6 +3,7 @@ from __future__ import annotations import json +import logging import re import sqlite3 from datetime import UTC, datetime @@ -1013,6 +1014,614 @@ class TestCallback: # --------------------------------------------------------------------------- +_DEALS_FIXTURE: dict[str, list[object]] = { + "ticket": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], + "position_id": [100, 100, 100, 0, 200, 200, 300, 400, 400, 500, 500, 600, 600, 600], + "symbol": [ + "EURUSD", + "EURUSD", + "EURUSD", + "", + "EURUSD", + "EURUSD", + "GBPUSD", + "GBPUSD", + "GBPUSD", + "EURUSD", + "EURUSD", + "GBPUSD", + "GBPUSD", + "GBPUSD", + ], + "time": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], + # type: 0=BUY, 1=SELL, 2=BALANCE + "type": [0, 0, 1, 2, 0, 1, 0, 0, 2, 0, 1, 0, 1, 1], + # entry: 0=IN, 1=OUT, 2=INOUT (reversal), 3=OUT_BY + "entry": [0, 0, 1, 0, 0, 1, 0, 0, 2, 0, 3, 0, 2, 1], + "volume": [1.0, 3.0, 4.0, 0.0, 2.0, 2.0, 5.0, 1.0, 1.0, 2.0, 2.0, 3.0, 1.0, 3.0], + "price": [ + 1.10, + 1.20, + 1.50, + 0.0, + 2.00, + 2.20, + 1.30, + 1.30, + 1.40, + 1.00, + 1.05, + 1.10, + 9.99, + 1.40, + ], + "profit": [0.0, 0.0, 10.0, 5.0, 0.0, 8.0, 0.0, 0.0, -1.0, 0.0, 3.0, 0.0, -2.0, 7.0], +} + + +def _build_history_client(mocker: MockerFixture) -> MagicMock: + """Build a mocked Mt5DataClient with per-symbol history results.""" + client = MagicMock() + + def _rates(**kwargs: object) -> pd.DataFrame: + return pd.DataFrame({ + "time": [1], + "open": [1.0], + "symbol_arg": [kwargs.get("symbol")], + }) + + def _ticks(**kwargs: object) -> pd.DataFrame: + return pd.DataFrame({ + "time": [1], + "bid": [1.0], + "symbol_arg": [kwargs.get("symbol")], + }) + + client.copy_rates_range_as_df.side_effect = _rates + client.copy_ticks_range_as_df.side_effect = _ticks + + def _orders(**kwargs: object) -> pd.DataFrame: + return pd.DataFrame({"ticket": [10], "symbol": [kwargs.get("symbol")]}) + + def _deals(**kwargs: object) -> pd.DataFrame: + sym = kwargs.get("symbol") + df = pd.DataFrame(_DEALS_FIXTURE) + return df[df["symbol"] == sym].reset_index(drop=True) + + client.history_orders_get_as_df.side_effect = _orders + client.history_deals_get_as_df.side_effect = _deals + mocker.patch("mt5cli.cli.Mt5DataClient", return_value=client) + return client + + +class TestCollectHistory: + """Tests for the collect-history command.""" + + @pytest.fixture + def history_client(self, mocker: MockerFixture) -> MagicMock: + """Create a mocked Mt5DataClient with history-style DataFrames.""" + return _build_history_client(mocker) + + def test_collect_history_writes_all_tables( + self, + tmp_path: Path, + history_client: MagicMock, + ) -> None: + """Test that collect-history writes rates, ticks, and history tables.""" + output = tmp_path / "history.db" + result = runner.invoke( + app, + [ + "-o", + str(output), + "collect-history", + "--symbol", + "EURUSD", + "--symbol", + "GBPUSD", + "--date-from", + "2024-01-01", + "--date-to", + "2024-02-01", + ], + ) + assert result.exit_code == 0, result.output + assert history_client.copy_rates_range_as_df.call_count == 2 + assert history_client.copy_ticks_range_as_df.call_count == 2 + history_client.copy_ticks_range_as_df.assert_any_call( + symbol="EURUSD", + date_from=datetime(2024, 1, 1, tzinfo=UTC), + date_to=datetime(2024, 2, 1, tzinfo=UTC), + flags=1, + ) + with sqlite3.connect(output) as conn: + tables = { + row[0] + for row in conn.execute( + "SELECT name FROM sqlite_master WHERE type='table'", + ).fetchall() + } + assert {"rates", "ticks", "history_orders", "history_deals"} <= tables + + def test_collect_history_history_fetched_per_symbol( + self, + tmp_path: Path, + history_client: MagicMock, + ) -> None: + """Test that history-orders and history-deals are fetched per symbol.""" + output = tmp_path / "history.db" + result = runner.invoke( + app, + [ + "-o", + str(output), + "collect-history", + "--symbol", + "EURUSD", + "--symbol", + "GBPUSD", + "--date-from", + "2024-01-01", + "--date-to", + "2024-02-01", + ], + ) + assert result.exit_code == 0, result.output + assert history_client.history_orders_get_as_df.call_count == 2 + assert history_client.history_deals_get_as_df.call_count == 2 + history_client.history_orders_get_as_df.assert_any_call( + date_from=datetime(2024, 1, 1, tzinfo=UTC), + date_to=datetime(2024, 2, 1, tzinfo=UTC), + symbol="EURUSD", + ) + history_client.history_deals_get_as_df.assert_any_call( + date_from=datetime(2024, 1, 1, tzinfo=UTC), + date_to=datetime(2024, 2, 1, tzinfo=UTC), + symbol="GBPUSD", + ) + + @pytest.mark.parametrize( + ("selected", "expected_tables", "excluded_calls"), + [ + ( + ["rates", "history-deals"], + {"rates", "history_deals"}, + ("copy_ticks_range_as_df", "history_orders_get_as_df"), + ), + ( + ["ticks", "history-orders"], + {"ticks", "history_orders"}, + ("copy_rates_range_as_df", "history_deals_get_as_df"), + ), + ], + ) + def test_collect_history_dataset_selection( + self, + tmp_path: Path, + history_client: MagicMock, + selected: list[str], + expected_tables: set[str], + excluded_calls: tuple[str, ...], + ) -> None: + """Test that --dataset limits which datasets are fetched and written.""" + output = tmp_path / "history.db" + args = [ + "-o", + str(output), + "collect-history", + "--symbol", + "EURUSD", + "--date-from", + "2024-01-01", + "--date-to", + "2024-02-01", + ] + for name in selected: + args.extend(["--dataset", name]) + result = runner.invoke(app, args) + assert result.exit_code == 0, result.output + for name in excluded_calls: + getattr(history_client, name).assert_not_called() + with sqlite3.connect(output) as conn: + tables = { + row[0] + for row in conn.execute( + "SELECT name FROM sqlite_master WHERE type='table'", + ).fetchall() + } + assert expected_tables <= tables + assert tables.isdisjoint( + {"rates", "ticks", "history_orders", "history_deals"} - expected_tables + ) + + def test_collect_history_rates_table_has_timeframe( + self, + tmp_path: Path, + history_client: MagicMock, # noqa: ARG002 + ) -> None: + """Test that the rates table carries the requested timeframe value.""" + output = tmp_path / "history.db" + result = runner.invoke( + app, + [ + "-o", + str(output), + "collect-history", + "--symbol", + "EURUSD", + "--date-from", + "2024-01-01", + "--date-to", + "2024-02-01", + "--timeframe", + "H1", + "--dataset", + "rates", + ], + ) + assert result.exit_code == 0, result.output + with sqlite3.connect(output) as conn: + rows = conn.execute( + "SELECT DISTINCT timeframe FROM rates", + ).fetchall() + assert rows == [(16385,)] + + def test_collect_history_if_exists_append( + self, + tmp_path: Path, + history_client: MagicMock, # noqa: ARG002 + ) -> None: + """Test that --if-exists=append accumulates rows across runs.""" + output = tmp_path / "history.db" + common = [ + "-o", + str(output), + "collect-history", + "--symbol", + "EURUSD", + "--date-from", + "2024-01-01", + "--date-to", + "2024-02-01", + "--dataset", + "rates", + ] + first = runner.invoke(app, common) + second = runner.invoke(app, [*common, "--if-exists", "append"]) + assert first.exit_code == 0, first.output + assert second.exit_code == 0, second.output + with sqlite3.connect(output) as conn: + (count,) = conn.execute("SELECT COUNT(*) FROM rates").fetchone() + assert count == 2 + + def test_collect_history_if_exists_fail( + self, + tmp_path: Path, + history_client: MagicMock, # noqa: ARG002 + ) -> None: + """Test that --if-exists=fail rejects writing into an existing table.""" + output = tmp_path / "history.db" + common = [ + "-o", + str(output), + "collect-history", + "--symbol", + "EURUSD", + "--date-from", + "2024-01-01", + "--date-to", + "2024-02-01", + "--dataset", + "rates", + ] + first = runner.invoke(app, common) + second = runner.invoke(app, [*common, "--if-exists", "fail"]) + assert first.exit_code == 0, first.output + assert second.exit_code != 0 + + def test_collect_history_ticks_default_flags_all( + self, + tmp_path: Path, + history_client: MagicMock, + ) -> None: + """Test that --flags defaults to ALL for ticks.""" + output = tmp_path / "history.db" + result = runner.invoke( + app, + [ + "-o", + str(output), + "collect-history", + "--symbol", + "EURUSD", + "--date-from", + "2024-01-01", + "--date-to", + "2024-02-01", + ], + ) + assert result.exit_code == 0, result.output + history_client.copy_ticks_range_as_df.assert_called_once_with( + symbol="EURUSD", + date_from=datetime(2024, 1, 1, tzinfo=UTC), + date_to=datetime(2024, 2, 1, tzinfo=UTC), + flags=1, + ) + + def test_collect_history_with_views( + self, + tmp_path: Path, + history_client: MagicMock, # noqa: ARG002 + ) -> None: + """Test that --with-views creates cash_events and positions views.""" + output = tmp_path / "history.db" + result = runner.invoke( + app, + [ + "-o", + str(output), + "collect-history", + "--symbol", + "EURUSD", + "--symbol", + "GBPUSD", + "--date-from", + "2024-01-01", + "--date-to", + "2024-02-01", + "--with-views", + ], + ) + assert result.exit_code == 0, result.output + with sqlite3.connect(output) as conn: + views = { + row[0] + for row in conn.execute( + "SELECT name FROM sqlite_master WHERE type='view'", + ).fetchall() + } + cash = conn.execute("SELECT type FROM cash_events").fetchall() + positions = { + row[0]: row + for row in conn.execute( + "SELECT position_id, volume_open, volume_close," + " volume_reversal, open_price, close_price, reversal_count" + " FROM positions_reconstructed", + ).fetchall() + } + assert {"cash_events", "positions_reconstructed"} <= views + assert all(row[0] not in {0, 1} for row in cash) + # Position 100 (BUY 1@1.10 + BUY 3@1.20 then SELL 4@1.50) is closed. + # Position 200 (BUY 2@2.00 then SELL 2@2.20) is closed. + # Position 300 (open-only) and 400 (reversal-only) are excluded. + assert set(positions) == {100, 200, 500, 600} + pos_100 = positions[100] + tol = 1e-9 + assert abs(pos_100[1] - 4.0) < tol # volume_open + assert abs(pos_100[2] - 4.0) < tol # volume_close + assert abs(pos_100[3] - 0.0) < tol # volume_reversal + # Volume-weighted open: (1*1.10 + 3*1.20) / 4 = 1.175 + assert abs(pos_100[4] - 1.175) < tol + # Volume-weighted close: (4*1.50) / 4 = 1.50 + assert abs(pos_100[5] - 1.50) < tol + assert pos_100[6] == 0 # reversal_count + pos_500 = positions[500] + assert abs(pos_500[2] - 2.0) < tol # OUT_BY contributes to close volume + assert abs(pos_500[5] - 1.05) < tol + pos_600 = positions[600] + assert abs(pos_600[1] - 3.0) < tol + assert abs(pos_600[2] - 3.0) < tol + assert abs(pos_600[3] - 1.0) < tol + assert abs(pos_600[4] - 1.10) < tol + assert abs(pos_600[5] - 1.40) < tol + assert pos_600[6] == 1 + + def test_collect_history_filters_history_symbols_exactly( + self, + tmp_path: Path, + mocker: MockerFixture, + ) -> None: + """Test that history wildcard results are filtered to exact symbols.""" + client = MagicMock() + client.history_orders_get_as_df.return_value = pd.DataFrame({ + "ticket": [1, 2], + "symbol": ["EURUSD", "EURUSDm"], + }) + client.history_deals_get_as_df.return_value = pd.DataFrame({ + "ticket": [3, 4], + "symbol": ["EURUSD", "EURUSDm"], + }) + mocker.patch("mt5cli.cli.Mt5DataClient", return_value=client) + output = tmp_path / "history.db" + result = runner.invoke( + app, + [ + "-o", + str(output), + "collect-history", + "--symbol", + "EURUSD", + "--date-from", + "2024-01-01", + "--date-to", + "2024-02-01", + "--dataset", + "history-orders", + "--dataset", + "history-deals", + ], + ) + assert result.exit_code == 0, result.output + with sqlite3.connect(output) as conn: + order_symbols = conn.execute( + "SELECT DISTINCT symbol FROM history_orders", + ).fetchall() + deal_symbols = conn.execute( + "SELECT DISTINCT symbol FROM history_deals", + ).fetchall() + assert order_symbols == [("EURUSD",)] + assert deal_symbols == [("EURUSD",)] + + def test_collect_history_requires_sqlite_format( + self, + tmp_path: Path, + history_client: MagicMock, # noqa: ARG002 + ) -> None: + """Test that non-SQLite output is rejected.""" + output = tmp_path / "history.csv" + result = runner.invoke( + app, + [ + "-o", + str(output), + "collect-history", + "--symbol", + "EURUSD", + "--date-from", + "2024-01-01", + "--date-to", + "2024-02-01", + ], + ) + assert result.exit_code != 0 + assert "requires SQLite3" in normalize_cli_output(result.output) + + def test_collect_history_requires_symbol( + self, + tmp_path: Path, + history_client: MagicMock, # noqa: ARG002 + ) -> None: + """Test that at least one --symbol is required.""" + output = tmp_path / "history.db" + result = runner.invoke( + app, + [ + "-o", + str(output), + "collect-history", + "--date-from", + "2024-01-01", + "--date-to", + "2024-02-01", + ], + ) + assert result.exit_code != 0 + + def test_collect_history_views_skipped_when_columns_missing( + self, + tmp_path: Path, + mocker: MockerFixture, + caplog: pytest.LogCaptureFixture, + ) -> None: + """Test that views are not created when required columns are missing.""" + client = MagicMock() + client.copy_rates_range_as_df.return_value = pd.DataFrame({"x": [1]}) + client.copy_ticks_range_as_df.return_value = pd.DataFrame({"x": [1]}) + client.history_orders_get_as_df.return_value = pd.DataFrame({"x": [1]}) + client.history_deals_get_as_df.return_value = pd.DataFrame({"x": [1]}) + mocker.patch("mt5cli.cli.Mt5DataClient", return_value=client) + output = tmp_path / "history.db" + with caplog.at_level(logging.WARNING, logger="mt5cli.cli"): + result = runner.invoke( + app, + [ + "-o", + str(output), + "collect-history", + "--symbol", + "EURUSD", + "--date-from", + "2024-01-01", + "--date-to", + "2024-02-01", + "--with-views", + ], + ) + assert result.exit_code == 0, result.output + with sqlite3.connect(output) as conn: + views = { + row[0] + for row in conn.execute( + "SELECT name FROM sqlite_master WHERE type='view'", + ).fetchall() + } + assert "cash_events" not in views + assert "positions_reconstructed" not in views + assert "Skipping cash_events view" in caplog.text + assert "Skipping positions_reconstructed view" in caplog.text + + def test_collect_history_skips_empty_history_without_columns( + self, + tmp_path: Path, + mocker: MockerFixture, + ) -> None: + """Test that empty no-column history results do not fail collection.""" + client = MagicMock() + client.copy_rates_range_as_df.return_value = pd.DataFrame({"time": [1]}) + client.history_deals_get_as_df.return_value = pd.DataFrame() + mocker.patch("mt5cli.cli.Mt5DataClient", return_value=client) + output = tmp_path / "history.db" + result = runner.invoke( + app, + [ + "-o", + str(output), + "collect-history", + "--symbol", + "EURUSD", + "--date-from", + "2024-01-01", + "--date-to", + "2024-02-01", + "--dataset", + "rates", + "--dataset", + "history-deals", + ], + ) + assert result.exit_code == 0, result.output + with sqlite3.connect(output) as conn: + tables = { + row[0] + for row in conn.execute( + "SELECT name FROM sqlite_master WHERE type='table'", + ).fetchall() + } + assert "rates" in tables + assert "history_deals" not in tables + + def test_collect_history_warns_when_views_requested_without_deals( + self, + tmp_path: Path, + history_client: MagicMock, # noqa: ARG002 + caplog: pytest.LogCaptureFixture, + ) -> None: + """Test that --with-views warns when history_deals is not written.""" + output = tmp_path / "history.db" + with caplog.at_level(logging.WARNING, logger="mt5cli.cli"): + result = runner.invoke( + app, + [ + "-o", + str(output), + "collect-history", + "--symbol", + "EURUSD", + "--date-from", + "2024-01-01", + "--date-to", + "2024-02-01", + "--dataset", + "rates", + "--with-views", + ], + ) + assert result.exit_code == 0, result.output + assert ( + "--with-views ignored: history_deals table was not written" in caplog.text + ) + + class TestMain: """Tests for the main entry point.""" diff --git a/uv.lock b/uv.lock index 2eb4673..4cc83de 100644 --- a/uv.lock +++ b/uv.lock @@ -487,7 +487,7 @@ wheels = [ [[package]] name = "mt5cli" -version = "0.2.0" +version = "0.3.0" source = { editable = "." } dependencies = [ { name = "click" },