From 74e3754a80a47fa68791f054d104e1ceafb6ae02 Mon Sep 17 00:00:00 2001 From: Daichi Narushima <1938249+dceoy@users.noreply.github.com> Date: Sat, 4 Jul 2026 14:19:56 +0900 Subject: [PATCH] Fix mt5cli issues #92-#95 and #97 (#102) * feat: add MT5 order metadata, coverage report, and env-backed CLI config * fix: align review-driven trading and history contracts * Bump pdmt5 to 1.1.0 * test: stabilize history gaps CLI assertion * fix: address review follow-ups for gaps and filling mode --- README.md | 81 ++++++---- docs/api/public-contract.md | 14 +- docs/index.md | 17 +- mt5cli/__init__.py | 4 + mt5cli/cli.py | 179 +++++++++++++++++++-- mt5cli/contract.py | 2 + mt5cli/history.py | 168 +++++++++++++++++++- mt5cli/sdk.py | 2 + mt5cli/trading.py | 122 ++++++++++++++- pyproject.toml | 2 +- skills/mt5cli/SKILL.md | 11 +- tests/test_cli.py | 299 ++++++++++++++++++++++++++++++++++-- tests/test_history.py | 294 +++++++++++++++++++++++++++++++++++ tests/test_trading.py | 247 +++++++++++++++++++++++++++++ uv.lock | 10 +- 15 files changed, 1372 insertions(+), 80 deletions(-) diff --git a/README.md b/README.md index 353bb8d..43a475a 100644 --- a/README.md +++ b/README.md @@ -144,9 +144,10 @@ mt5cli -o ticks.json ticks-from --symbol EURUSD \ # Export symbols to SQLite3 with custom table name mt5cli -o data.db --table symbols symbols --group "*USD*" -# Export with connection credentials -mt5cli --login 12345 --password mypass --server MyBroker-Demo \ - -o positions.csv positions +# Export with connection credentials from env or placeholders +MT5_LOGIN=12345 MT5_PASSWORD=secret MT5_SERVER=MyBroker-Demo \ + mt5cli -o positions.csv positions +MT5_PATH="/path/to/terminal64.exe" mt5cli -o positions.csv positions ``` Run as a Python module: @@ -157,40 +158,54 @@ python -m mt5cli -o account.csv account-info ## Commands -| Command | Description | -| ---------------------- | ------------------------------------------------------------------------------------------------------------------------------------------- | -| `rates-from` | Export rates from a start date | -| `rates-from-pos` | Export rates from a start position | -| `latest-rates` | Export latest rates from a start position | -| `rates-range` | Export rates for a date range | -| `ticks-from` | Export ticks from a start date | -| `ticks-range` | Export ticks for a date range | -| `ticks-recent` | Export ticks from a recent trailing window | -| `account-info` | Export account information | -| `terminal-info` | Export terminal information | -| `version` | Export MetaTrader 5 version information | -| `last-error` | Export the last error information | -| `symbols` | Export symbol list | -| `symbol-info` | Export symbol details | -| `symbol-info-tick` | Export the last tick for a symbol | -| `minimum-margins` | Export minimum-volume buy and sell margin requirements | -| `market-book` | Export market depth (order book) | -| `orders` | Export active orders | -| `positions` | Export open positions | -| `history-orders` | Export historical orders | -| `history-deals` | Export historical deals | -| `recent-history-deals` | Export historical deals from a recent trailing window | -| `mt5-summary` | Export terminal/account status summary | -| `order-check` | Check funds sufficiency for a trade request | -| `order-send` | Send a raw trade request to the trade server (`--yes` required; expert path) | -| `close-positions` | Close open positions by `--symbol` or `--ticket` (`--yes` required for live; `--dry-run` available) | -| `collect-history` | Collect rates, history-orders, and history-deals for one or more symbols into a single SQLite database (ticks opt-in via `--dataset ticks`) | -| `grafana-schema` | Create or refresh Grafana-ready views and indexes in an existing SQLite database (idempotent, no MT5 connection) | -| `snapshot` | Snapshot current account, position, order, and terminal state into SQLite for live Grafana dashboards | +| Command | Description | +| ---------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------- | +| `rates-from` | Export rates from a start date | +| `rates-from-pos` | Export rates from a start position | +| `latest-rates` | Export latest rates from a start position | +| `rates-range` | Export rates for a date range | +| `ticks-from` | Export ticks from a start date | +| `ticks-range` | Export ticks for a date range | +| `ticks-recent` | Export ticks from a recent trailing window | +| `account-info` | Export account information | +| `terminal-info` | Export terminal information | +| `version` | Export MetaTrader 5 version information | +| `last-error` | Export the last error information | +| `symbols` | Export symbol list | +| `symbol-info` | Export symbol details | +| `symbol-info-tick` | Export the last tick for a symbol | +| `minimum-margins` | Export minimum-volume buy and sell margin requirements | +| `market-book` | Export market depth (order book) | +| `orders` | Export active orders | +| `positions` | Export open positions | +| `history-orders` | Export historical orders | +| `history-deals` | Export historical deals | +| `recent-history-deals` | Export historical deals from a recent trailing window | +| `mt5-summary` | Export terminal/account status summary | +| `order-check` | Check funds sufficiency for a trade request | +| `order-send` | Send a raw trade request to the trade server (`--yes` required; expert path) | +| `close-positions` | Close open positions by `--symbol` or `--ticket` (`--yes` required for live; `--dry-run` available; optional `--deviation` / `--comment` / `--magic`) | +| `collect-history` | Collect rates, history-orders, and history-deals for one or more symbols into a single SQLite database (ticks opt-in via `--dataset ticks`) | +| `history-gaps` | Export a SQLite-only one-row-per-gap report from managed rate compatibility views without connecting to MT5 | +| `grafana-schema` | Create or refresh Grafana-ready views and indexes in an existing SQLite database (idempotent, no MT5 connection) | +| `snapshot` | Snapshot current account, position, order, and terminal state into SQLite for live Grafana dashboards | Use `order-check` to validate a request payload before running `order-send --yes`. `close-positions` is the safer high-level alternative that builds correct close requests automatically. At least one `--symbol` or `--ticket` must be provided. +CLI connection flags fall back to `MT5_LOGIN`, `MT5_PASSWORD`, `MT5_SERVER`, +and `MT5_PATH` when unset, and explicit CLI values still win. + +### `history-gaps` + +Inspect collected SQLite rate views offline and export one row per detected gap. +For managed compatibility views, the command infers bar spacing from the view +name. Use `--granularity-seconds` for custom tables or views. + +```bash +mt5cli -o gaps.json history-gaps --sqlite3 history.db +mt5cli -o eurusd.csv history-gaps --sqlite3 history.db --table rate_EURUSD__M1_1 +``` ### `collect-history` diff --git a/docs/api/public-contract.md b/docs/api/public-contract.md index b053559..5905335 100644 --- a/docs/api/public-contract.md +++ b/docs/api/public-contract.md @@ -70,6 +70,7 @@ timestamp normalization in downstream apps. | Symbol | Role | | ----------------------------------------------------------------- | -------------------------------------------------------------------------------------------- | | `collect_history` | One-shot date-range export into SQLite | +| `report_rate_gaps` | SQLite-only one-row-per-gap report for a rate table or compatibility view | | `update_history`, `update_history_with_config` | Incremental append from `MAX(time)` cursors | | `ThrottledHistoryUpdater` | Minimum interval between successful incremental updates; optional `update_backend` injection | | `RateTarget`, `build_rate_targets` | Neutral `(symbol, timeframe)` series descriptors | @@ -98,6 +99,7 @@ strategy entries, exits, Kelly sizing, or signal logic. | `calculate_symbol_group_margin_ratio` | Estimated symbol-group margin/equity with optional exposure | | `determine_order_limits` | SL/TP price levels from ratios | | `calculate_trailing_stop_updates` | Per-ticket generic trailing stop-loss update plan | +| `resolve_broker_filling_mode` | Broker-supported filling-mode selection helper | | `ensure_symbol_selected` | Select/verify Market Watch visibility | | `fetch_recent_history_deals_for_trading_client` | Recent deal history from a connected trading client | | `place_market_order`, `close_open_positions`, `update_sltp_for_open_positions`, `update_trailing_stop_loss_for_open_positions` | Order execution helpers (`dry_run` supported) | @@ -211,6 +213,9 @@ The Typer application in `mt5cli.cli` exposes file-export commands documented in - Require `-o/--output` and write CSV, JSON, Parquet, or SQLite. - Accept global MT5 connection options (`--login`, `--password`, `--server`, `--path`, `--timeout`). +- Resolve unset CLI connection options from `MT5_LOGIN`, `MT5_PASSWORD`, + `MT5_SERVER`, and `MT5_PATH`, and expand `${ENV_VAR}` placeholders in CLI + string fields before building the MT5 config. - Delegate to the same Python APIs described here; they are not duplicated business logic. @@ -228,7 +233,14 @@ constructed request payload. `close-positions` is the safer high-level helper that closes open positions by `--symbol` or `--ticket` using `close_open_positions()`. Both `order-send --yes` and `close-positions --yes` are live execution paths. `close-positions --dry-run` previews close orders -without placing them and does not require `--yes`. +without placing them and does not require `--yes`. `close-positions` also +accepts optional `--deviation`, `--comment`, and `--magic`; `--magic` scopes +the selected open positions fail-closed when position magic metadata is absent. + +`history-gaps` reads an existing SQLite history database and exports one row +per detected gap from managed rate compatibility views. It never initializes +MT5. Pass `--granularity-seconds` for custom tables or views whose bar spacing +cannot be inferred from the name. ## Internal helpers (not stable) diff --git a/docs/index.md b/docs/index.md index dee25d7..56481af 100644 --- a/docs/index.md +++ b/docs/index.md @@ -109,8 +109,10 @@ mt5cli -o ticks.json ticks-from --symbol EURUSD \ # Export symbols to SQLite3 with custom table name mt5cli -o data.db --table symbols symbols --group "*USD*" -# Export with connection credentials -mt5cli --login 12345 --password mypass --server MyBroker-Demo \ +# Export with connection credentials from env or placeholders +MT5_LOGIN=12345 MT5_PASSWORD=secret MT5_SERVER=MyBroker-Demo \ + mt5cli -o positions.csv positions +mt5cli --login '${MT5_LOGIN}' --password '${MT5_PASSWORD}' --server '${MT5_SERVER}' \ -o positions.csv positions ``` @@ -164,10 +166,10 @@ mt5cli --login 12345 --password mypass --server MyBroker-Demo \ These commands send requests to the live trade server and can place or close real trades. Both require `--yes` for live execution. -| Command | Description | -| ----------------- | ---------------------------------------------------------------------------------------------------- | -| `order-send` | Send a **raw** trade request directly to MT5 (`--yes` required; expert path — no extra validation) | -| `close-positions` | Close open positions by `--symbol` or `--ticket` (`--yes` required for live; `--dry-run` to preview) | +| Command | Description | +| ----------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `order-send` | Send a **raw** trade request directly to MT5 (`--yes` required; expert path — no extra validation) | +| `close-positions` | Close open positions by `--symbol` or `--ticket` (`--yes` required for live; `--dry-run` to preview; optional `--deviation` / `--comment` / `--magic`) | Use `order-check` (Trading State) to validate funds before running `order-send --yes`. `close-positions` is the safer high-level alternative that builds correct close @@ -179,6 +181,7 @@ applications should prefer dedicated closing helpers or their own risk controls. | Command | Description | | ----------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `collect-history` | Collect rates, history-orders, and history-deals (ticks opt-in via `--dataset ticks`) for one or more symbols into a single SQLite database (optional cash-event/position views) | +| `history-gaps` | Export a SQLite-only one-row-per-gap report from managed rate compatibility views without connecting to MT5 | ```bash mt5cli -o history.db collect-history \ @@ -213,7 +216,7 @@ See the [History schema diagram](api/history.md#entity-relationship-diagram) for | `-f, --format` | Output format (auto-detected from extension if omitted) | | `--table` | Table name for SQLite3 output (default: "data") | | `--login` | Trading account login | -| `--password` | Trading account password | +| `--password` | Trading account password (`MT5_PASSWORD`) | | `--server` | Trading server name | | `--path` | Path to MetaTrader5 terminal EXE file | | `--timeout` | Connection timeout in milliseconds | diff --git a/mt5cli/__init__.py b/mt5cli/__init__.py index f5a822d..b715f26 100644 --- a/mt5cli/__init__.py +++ b/mt5cli/__init__.py @@ -22,6 +22,7 @@ from .history import ( drop_forming_rate_bar, load_rate_series_by_granularity, load_rate_series_from_sqlite, + report_rate_gaps, ) from .sdk import ( AccountSpec, @@ -76,6 +77,7 @@ from .trading import ( mt5_trading_session, normalize_order_volume, place_market_order, + resolve_broker_filling_mode, update_sltp_for_open_positions, update_trailing_stop_loss_for_open_positions, ) @@ -140,8 +142,10 @@ __all__ = [ "mt5_trading_session", "normalize_order_volume", "place_market_order", + "report_rate_gaps", "resolve_account_spec", "resolve_account_specs", + "resolve_broker_filling_mode", "update_history", "update_history_with_config", "update_observability", diff --git a/mt5cli/cli.py b/mt5cli/cli.py index 13b69e5..fe89de2 100644 --- a/mt5cli/cli.py +++ b/mt5cli/cli.py @@ -4,6 +4,9 @@ from __future__ import annotations import json import logging +import os +import re +import sqlite3 from dataclasses import dataclass from datetime import datetime # noqa: TC003 from pathlib import Path # noqa: TC003 @@ -11,10 +14,10 @@ from typing import TYPE_CHECKING, Annotated, Any, cast import pandas as pd import typer -from pdmt5 import Mt5Config from . import sdk from .client import MT5Client +from .history import report_rate_gaps, resolve_granularity_name from .trading import OrderExecutionResult, close_open_positions, create_trading_client from .utils import ( DATETIME_TYPE, @@ -32,6 +35,8 @@ from .utils import ( if TYPE_CHECKING: from collections.abc import Callable + from pdmt5 import Mt5Config + logger = logging.getLogger(__name__) # --------------------------------------------------------------------------- @@ -66,12 +71,55 @@ app = typer.Typer( _REQUEST_OPTION_HELP = ( "Order request as a JSON object string, or '@path' to load JSON from a file." ) +_CLI_ENV_DEFAULTS: dict[str, str] = { + "path": "MT5_PATH", + "login": "MT5_LOGIN", + "password": "MT5_PASSWORD", + "server": "MT5_SERVER", +} +_RATE_VIEW_NAME_RE = re.compile( + r"^rate_(?P.+)__(?:(?P[A-Z0-9]+)_)?(?P\d+)$", +) def _get_export_context(ctx: typer.Context) -> _ExportContext: return cast("_ExportContext", ctx.obj) +def _resolve_cli_option(value: str | None, env_name: str) -> str | None: + return value if value is not None else os.environ.get(env_name) + + +def _timeframe_interval_seconds(timeframe: int) -> int | None: + granularity = resolve_granularity_name(timeframe) + units = { + "M": 60, + "H": 3600, + "D": 86400, + "W": 604800, + } + for prefix, seconds in units.items(): + suffix = granularity.removeprefix(prefix) + if granularity.startswith(prefix) and suffix.isdigit(): + return int(suffix) * seconds + return None + + +def _infer_gap_table_granularity_seconds(table: str) -> int | None: + if (match := _RATE_VIEW_NAME_RE.fullmatch(table)) is None: + return None + return _timeframe_interval_seconds(int(match.group("timeframe"))) + + +def _default_gap_tables(conn: sqlite3.Connection) -> list[str]: + rows = conn.execute( + "SELECT name FROM sqlite_master" + " WHERE type IN ('table', 'view') AND name GLOB 'rate_*__*'" + " ORDER BY name", + ).fetchall() + return [str(row[0]) for row in rows] + + def _execute_export( ctx: typer.Context, fetch_fn: Callable[[], pd.DataFrame], @@ -132,20 +180,36 @@ def _callback( # pyright: ignore[reportUnusedFunction] typer.Option(help="Table name for SQLite3 output."), ] = "data", login: Annotated[ - int | None, - typer.Option(help="Trading account login."), + str | None, + typer.Option( + help="Trading account login.", + envvar=_CLI_ENV_DEFAULTS["login"], + show_envvar=True, + ), ] = None, password: Annotated[ str | None, - typer.Option(help="Trading account password."), + typer.Option( + help="Trading account password.", + envvar=_CLI_ENV_DEFAULTS["password"], + show_envvar=True, + ), ] = None, server: Annotated[ str | None, - typer.Option(help="Trading server name."), + typer.Option( + help="Trading server name.", + envvar=_CLI_ENV_DEFAULTS["server"], + show_envvar=True, + ), ] = None, path: Annotated[ str | None, - typer.Option(help="Path to MetaTrader5 terminal EXE file."), + typer.Option( + help="Path to MetaTrader5 terminal EXE file.", + envvar=_CLI_ENV_DEFAULTS["path"], + show_envvar=True, + ), ] = None, timeout: Annotated[ int | None, @@ -169,17 +233,22 @@ def _callback( # pyright: ignore[reportUnusedFunction] ) except ValueError as exc: raise typer.BadParameter(str(exc)) from exc + try: + config = sdk.build_config( + path=_resolve_cli_option(path, _CLI_ENV_DEFAULTS["path"]), + login=_resolve_cli_option(login, _CLI_ENV_DEFAULTS["login"]), + password=_resolve_cli_option(password, _CLI_ENV_DEFAULTS["password"]), + server=_resolve_cli_option(server, _CLI_ENV_DEFAULTS["server"]), + timeout=timeout, + allow_whole_dollar_env=True, + ) + except ValueError as exc: + raise typer.BadParameter(str(exc)) from exc ctx.obj = _ExportContext( output=output, output_format=output_format, table=table, - config=Mt5Config( - path=path, - login=login, - password=password, - server=server, - timeout=timeout, - ), + config=config, ) @@ -658,6 +727,20 @@ def close_positions( help="Position ticket to close (repeat for multiple tickets).", ), ] = None, + deviation: Annotated[ + int | None, + typer.Option(help="Optional slippage/deviation for each close request."), + ] = None, + comment: Annotated[ + str | None, + typer.Option(help="Optional comment attached to each close request."), + ] = None, + magic: Annotated[ + int | None, + typer.Option( + help="Optional magic tag for close requests and position filtering.", + ), + ] = None, dry_run: Annotated[ bool, typer.Option("--dry-run", help="Preview close orders without executing them."), @@ -694,6 +777,9 @@ def close_positions( client, symbols=list(symbol) if symbol else None, tickets=list(ticket) if ticket else None, + deviation=deviation, + comment=comment, + magic=magic, dry_run=dry_run, ) finally: @@ -702,6 +788,73 @@ def close_positions( _execute_export(ctx, lambda: df) +@app.command("history-gaps", rich_help_panel="Collection") +def history_gaps( + ctx: typer.Context, + sqlite3_path: Annotated[ + Path, + typer.Option( + "--sqlite3", + help="Source SQLite history database to analyze.", + ), + ], + table: Annotated[ + list[str] | None, + typer.Option( + "--table", + help="Rate table or compatibility view to inspect (repeat for multiple).", + ), + ] = None, + granularity_seconds: Annotated[ + int | None, + typer.Option(help="Explicit bar interval in seconds for custom tables/views."), + ] = None, + min_gap_intervals: Annotated[ + int, + typer.Option(help="Minimum missing-bar count required to emit a gap row."), + ] = 1, +) -> None: + """Export SQLite rate gaps without connecting to MT5. + + Raises: + typer.BadParameter: If no compatible rate view is available and no + explicit table is provided, or if granularity inference fails. + """ + with sqlite3.connect(sqlite3_path) as conn: + tables = list(table) if table else _default_gap_tables(conn) + if not tables: + msg = ( + "No managed rate compatibility views found; pass --table for a rate " + "table or view." + ) + raise typer.BadParameter(msg, param_hint="--table") + frames: list[pd.DataFrame] = [] + for table_name in tables: + interval_seconds = ( + granularity_seconds or _infer_gap_table_granularity_seconds(table_name) + ) + if interval_seconds is None: + msg = ( + f"Could not infer granularity for {table_name!r}; pass " + "--granularity-seconds." + ) + raise typer.BadParameter(msg, param_hint="--granularity-seconds") + frames.append( + report_rate_gaps( + conn, + table_name, + granularity_seconds=interval_seconds, + min_gap_intervals=min_gap_intervals, + ) + ) + df = ( + pd.concat(frames, ignore_index=True) + if frames + else pd.DataFrame(columns=["table"]) + ) + _execute_export(ctx, lambda: df) + + @app.command(rich_help_panel="Collection") def collect_history( ctx: typer.Context, diff --git a/mt5cli/contract.py b/mt5cli/contract.py index b688b6f..c78f494 100644 --- a/mt5cli/contract.py +++ b/mt5cli/contract.py @@ -59,6 +59,8 @@ STABLE_SDK_EXPORTS: frozenset[str] = frozenset({ "mt5_trading_session", "normalize_order_volume", "place_market_order", + "report_rate_gaps", + "resolve_broker_filling_mode", "resolve_account_spec", "resolve_account_specs", "update_history", diff --git a/mt5cli/history.py b/mt5cli/history.py index 9e61e8a..9efd911 100644 --- a/mt5cli/history.py +++ b/mt5cli/history.py @@ -3,11 +3,12 @@ from __future__ import annotations import logging +import re import sqlite3 from dataclasses import dataclass -from datetime import UTC, datetime +from datetime import UTC, datetime, timedelta from pathlib import Path -from typing import TYPE_CHECKING, Literal, cast, overload +from typing import TYPE_CHECKING, Literal, SupportsInt, cast, overload import pandas as pd from pdmt5 import get_timeframe_name as _get_timeframe_name @@ -63,6 +64,20 @@ _POSITIONS_VIEW_REQUIRED_COLUMNS: frozenset[str] = frozenset({ "price", "profit", }) +_RATE_GAP_COLUMNS: tuple[str, ...] = ( + "table", + "symbol", + "timeframe", + "granularity", + "granularity_seconds", + "gap_start", + "gap_end", + "missing_intervals", +) +_RATE_VIEW_NAME_RE = re.compile( + r"^rate_(?P.+)__(?:(?P[A-Z0-9]+)_)?(?P\d+)$", +) +_MIN_TIMESTAMPS_FOR_GAPS = 2 def quote_sqlite_identifier(identifier: str) -> str: @@ -231,6 +246,155 @@ def _open_existing_sqlite_database( return conn, True +def _empty_rate_gap_report() -> pd.DataFrame: + return pd.DataFrame(columns=_RATE_GAP_COLUMNS) + + +def _coerce_optional_int(value: object) -> int | None: + if value is None or isinstance(value, bool): + return None + if isinstance(value, str): + text = value.strip() + if text.lstrip("+-").isdigit(): + return int(text) + return None + if not hasattr(value, "__int__"): + return None + try: + return int(cast("SupportsInt", value)) + except (TypeError, ValueError): + return None + + +def _rate_gap_metadata( + table: str, + frame: pd.DataFrame, + *, + granularity_seconds: int, +) -> dict[str, object]: + symbol: str | None = None + timeframe: int | None = None + granularity: str | None = None + + if "symbol" in frame.columns: + symbols = {str(value) for value in frame["symbol"].dropna().unique()} + if len(symbols) == 1: + symbol = next(iter(symbols)) + if "timeframe" in frame.columns: + timeframes = { + coerced + for value in frame["timeframe"].dropna().unique() + if (coerced := _coerce_optional_int(value)) is not None + } + if len(timeframes) == 1: + timeframe = next(iter(timeframes)) + + if timeframe is None and (match := _RATE_VIEW_NAME_RE.fullmatch(table)) is not None: + symbol = symbol or match.group("symbol") + timeframe = int(match.group("timeframe")) + granularity = match.group("granularity") or resolve_granularity_name(timeframe) + + if timeframe is not None and granularity is None: + granularity = resolve_granularity_name(timeframe) + + return { + "table": table, + "symbol": symbol, + "timeframe": timeframe, + "granularity": granularity, + "granularity_seconds": granularity_seconds, + } + + +def _iter_rate_gap_groups(frame: pd.DataFrame) -> list[pd.DataFrame]: + series_columns = [ + column for column in ("symbol", "timeframe") if column in frame.columns + ] + if not series_columns: + return [frame] + return [ + group for _, group in frame.groupby(series_columns, dropna=False, sort=False) + ] + + +def report_rate_gaps( + conn: sqlite3.Connection, + table: str, + *, + granularity_seconds: int, + min_gap_intervals: int = 1, +) -> pd.DataFrame: + """Return one row per detected gap from a SQLite rate table or view. + + Raises: + ValueError: If the table name, schema, timestamps, or gap parameters + are invalid. + """ + table_name = _validate_rate_load_request(table, count=None) + if granularity_seconds <= 0: + msg = "granularity_seconds must be positive." + raise ValueError(msg) + if min_gap_intervals <= 0: + msg = "min_gap_intervals must be positive." + raise ValueError(msg) + + columns = get_table_columns(conn, table_name) + _ensure_rate_columns(columns, table_name) + quoted_table = quote_sqlite_identifier(table_name) + frame = cast( + "pd.DataFrame", + pd.read_sql_query( # type: ignore[reportUnknownMemberType] + f"SELECT * FROM {quoted_table} ORDER BY time ASC", # noqa: S608 + conn, + ), + ) + if frame.empty: + return _empty_rate_gap_report() + + parsed_times = frame["time"].map(parse_sqlite_timestamp) + if parsed_times.isna().any(): + msg = f"SQLite table or view {table_name!r} contains unparsable time values." + raise ValueError(msg) + + series_frame = frame.copy() + series_frame["time"] = parsed_times + rows: list[dict[str, object]] = [] + for group in _iter_rate_gap_groups(series_frame): + unique_times = group["time"].drop_duplicates().sort_values(ignore_index=True) + if len(unique_times) < _MIN_TIMESTAMPS_FOR_GAPS: + continue + + metadata = _rate_gap_metadata( + table_name, + group, + granularity_seconds=granularity_seconds, + ) + deltas = unique_times.diff().dropna() + for index, delta in enumerate(deltas, start=1): + delta_seconds = int(delta.total_seconds()) + missing_intervals = max( + ((delta_seconds + (granularity_seconds - 1)) // granularity_seconds) + - 1, + 0, + ) + if missing_intervals < min_gap_intervals: + continue + previous_time = unique_times.iloc[index - 1] + next_time = unique_times.iloc[index] + rows.append({ + **metadata, + "gap_start": ( + previous_time.to_pydatetime() + + timedelta(seconds=granularity_seconds) + ), + "gap_end": ( + next_time.to_pydatetime() - timedelta(seconds=granularity_seconds) + ), + "missing_intervals": missing_intervals, + }) + return pd.DataFrame(rows, columns=_RATE_GAP_COLUMNS) + + def _validate_rate_load_request(table: str, count: int | None) -> str: table_name = _require_non_empty_identifier(table, "table or view") if count is not None and count <= 0: diff --git a/mt5cli/sdk.py b/mt5cli/sdk.py index ca65bca..f78ba23 100644 --- a/mt5cli/sdk.py +++ b/mt5cli/sdk.py @@ -33,6 +33,7 @@ from .history import ( create_history_indexes, create_positions_reconstructed_view, drop_forming_rate_bar, + report_rate_gaps, resolve_granularity_name, resolve_history_datasets, resolve_history_tick_flags, @@ -150,6 +151,7 @@ __all__ = [ "positions", "recent_history_deals", "recent_ticks", + "report_rate_gaps", "resolve_account_spec", "resolve_account_specs", "substitute_env_placeholders", diff --git a/mt5cli/trading.py b/mt5cli/trading.py index 0e47b7a..fd7d400 100644 --- a/mt5cli/trading.py +++ b/mt5cli/trading.py @@ -199,6 +199,7 @@ _SYMBOL_SNAPSHOT_FIELDS = ( "trade_tick_value", "trade_stops_level", "filling_mode", + "trade_exemode", ) _TICK_SNAPSHOT_FIELDS = ("symbol", "time", "bid", "ask", "last", "volume") POSITION_COLUMNS = ( @@ -214,6 +215,7 @@ POSITION_COLUMNS = ( "profit", "swap", "comment", + "magic", ) __all__ = [ @@ -255,6 +257,7 @@ __all__ = [ "mt5_trading_session", "normalize_order_volume", "place_market_order", + "resolve_broker_filling_mode", "update_sltp_for_open_positions", "update_trailing_stop_loss_for_open_positions", ] @@ -658,19 +661,24 @@ def create_trading_client( def detect_position_side( client: _Mt5ClientProtocol, symbol: str, + *, + magic: int | None = None, ) -> PositionSide | None: """Detect the net open position side for a symbol. Args: client: Connected MT5 client instance. symbol: Symbol to inspect. + magic: Optional magic number filter applied fail-closed. Returns: ``"long"`` when there are buy positions and no sell positions, ``"short"`` when there are sell positions and no buy positions, or ``None`` when no positions or mixed exposure exists. """ - positions = get_positions_frame(client, symbol=symbol) + positions = _filter_positions( + get_positions_frame(client, symbol=symbol), magic=magic + ) if positions.empty: return None @@ -734,6 +742,85 @@ def get_positions_frame( return frame +def _supported_filling_modes( + client: _Mt5ClientProtocol, + *, + symbol: str, + preferred_default: OrderFillingMode, +) -> set[str] | None: + snapshot = get_symbol_snapshot(client, symbol) + filling_mode = _optional_int(snapshot.get("filling_mode")) + trade_exemode = _optional_int(snapshot.get("trade_exemode")) + if filling_mode is None and trade_exemode is None: + _logger.debug( + "Filling-mode metadata unavailable for %s; keeping preferred mode %s.", + symbol, + preferred_default, + ) + return None + + supported: set[str] = set() + if filling_mode is not None: + fok_flag = getattr(client.mt5, "SYMBOL_FILLING_FOK", None) + ioc_flag = getattr(client.mt5, "SYMBOL_FILLING_IOC", None) + if isinstance(fok_flag, int) and filling_mode & fok_flag: + supported.add("FOK") + if isinstance(ioc_flag, int) and filling_mode & ioc_flag: + supported.add("IOC") + market_execution = getattr(client.mt5, "SYMBOL_TRADE_EXECUTION_MARKET", None) + if trade_exemode is not None and not ( + isinstance(market_execution, int) and trade_exemode == market_execution + ): + supported.add("RETURN") + if supported: + return supported + + _logger.debug( + "Filling-mode metadata was unparseable for %s; keeping preferred mode %s.", + symbol, + preferred_default, + ) + return None + + +def resolve_broker_filling_mode( + client: _Mt5ClientProtocol, + *, + symbol: str, + preferred_modes: Sequence[OrderFillingMode] = ("IOC", "FOK", "RETURN"), + default_mode: OrderFillingMode = "IOC", +) -> OrderFillingMode: + """Return the first broker-supported filling mode from a preferred order. + + Raises: + ValueError: If any preferred or default mode name is unsupported. + """ + preferred = [mode.upper() for mode in preferred_modes] + for mode in [*preferred, default_mode]: + if mode not in _ORDER_FILLING_MODES: + msg = f"Unsupported order_filling mode: {mode!r}." + raise ValueError(msg) + + preferred_default = cast( + "OrderFillingMode", preferred[0] if preferred else default_mode + ) + supported = _supported_filling_modes( + client, + symbol=symbol, + preferred_default=preferred_default, + ) + if supported is None: + return preferred_default + + for mode in preferred: + if mode in supported: + return cast("OrderFillingMode", mode) + fallback_mode = next( + mode for mode in (default_mode, "IOC", "FOK", "RETURN") if mode in supported + ) + return cast("OrderFillingMode", fallback_mode) + + def _order_side_from_position_type( client: _Mt5ClientProtocol, position_type: object, @@ -1334,7 +1421,7 @@ def determine_order_limits( } -def place_market_order( +def place_market_order( # noqa: C901, PLR0913 client: _Mt5ClientProtocol, *, symbol: str, @@ -1345,6 +1432,9 @@ def place_market_order( sl: float | None = None, tp: float | None = None, position: int | None = None, + deviation: int | None = None, + comment: str | None = None, + magic: int | None = None, dry_run: bool = False, ) -> OrderExecutionResult: """Place one normalized market order or return a dry-run result. @@ -1398,6 +1488,12 @@ def place_market_order( request["tp"] = tp if position is not None: request["position"] = position + if deviation is not None: + request["deviation"] = deviation + if comment is not None: + request["comment"] = comment + if magic is not None: + request["magic"] = magic if dry_run: return { "status": "dry_run", @@ -1432,6 +1528,7 @@ def _filter_positions( *, symbols: str | list[str] | None = None, tickets: list[int] | None = None, + magic: int | None = None, ) -> pd.DataFrame: frame = positions if symbols is not None: @@ -1439,6 +1536,10 @@ def _filter_positions( frame = frame.loc[frame["symbol"].isin(symbol_set)] if tickets is not None: frame = frame.loc[frame["ticket"].isin(tickets)] + if magic is not None: + if "magic" not in frame.columns: + return frame.iloc[0:0].copy() + frame = frame.loc[frame["magic"] == magic] return frame @@ -1447,6 +1548,9 @@ def close_open_positions( *, symbols: str | list[str] | None = None, tickets: list[int] | None = None, + deviation: int | None = None, + comment: str | None = None, + magic: int | None = None, dry_run: bool = False, ) -> list[OrderExecutionResult]: """Close matching open positions. @@ -1458,6 +1562,7 @@ def close_open_positions( get_positions_frame(client), symbols=symbols, tickets=tickets, + magic=magic, ) results: list[OrderExecutionResult] = [] for row in positions.to_dict("records"): @@ -1469,6 +1574,9 @@ def close_open_positions( volume=float(row["volume"]), order_side=side, position=int(row["ticket"]), + deviation=deviation, + comment=comment, + magic=magic, dry_run=dry_run, ) results.append(result) @@ -1523,6 +1631,7 @@ def calculate_trailing_stop_updates( *, symbol: str, trailing_stop_ratio: float, + magic: int | None = None, ) -> dict[int, float]: """Return per-ticket trailing stop-loss updates for open symbol positions. @@ -1533,7 +1642,9 @@ def calculate_trailing_stop_updates( missing side-specific tick price are skipped. """ _require_protective_ratio(trailing_stop_ratio, "trailing_stop_ratio") - positions = get_positions_frame(client, symbol=symbol) + positions = _filter_positions( + get_positions_frame(client, symbol=symbol), magic=magic + ) if positions.empty: return {} tick = get_tick_snapshot(client, symbol) @@ -1568,6 +1679,7 @@ def update_trailing_stop_loss_for_open_positions( *, symbol: str, trailing_stop_ratio: float, + magic: int | None = None, dry_run: bool = False, ) -> list[OrderExecutionResult]: """Update open positions whose trailing stop loss should move favorably. @@ -1579,6 +1691,7 @@ def update_trailing_stop_loss_for_open_positions( client, symbol=symbol, trailing_stop_ratio=trailing_stop_ratio, + magic=magic, ) results: list[OrderExecutionResult] = [] for ticket, stop_loss in updates.items(): @@ -1588,6 +1701,7 @@ def update_trailing_stop_loss_for_open_positions( symbol=symbol, tickets=[ticket], stop_loss=stop_loss, + magic=magic, dry_run=dry_run, ), ) @@ -1599,6 +1713,7 @@ def update_sltp_for_open_positions( *, symbol: str | None = None, tickets: list[int] | None = None, + magic: int | None = None, stop_loss: float | None = None, take_profit: float | None = None, dry_run: bool = False, @@ -1612,6 +1727,7 @@ def update_sltp_for_open_positions( get_positions_frame(client), symbols=symbol, tickets=tickets, + magic=magic, ) results: list[OrderExecutionResult] = [] for row in positions.to_dict("records"): diff --git a/pyproject.toml b/pyproject.toml index 622430b..41c76af 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ license-files = ["LICENSE"] readme = "README.md" requires-python = ">= 3.11, < 3.14" dependencies = [ - "pdmt5>=1.0.4", + "pdmt5 >= 1.1.0", "pandas >= 2.2.2", "pydantic >= 2.13.4", "click >= 8.1.0", diff --git a/skills/mt5cli/SKILL.md b/skills/mt5cli/SKILL.md index 5882671..cccd8af 100644 --- a/skills/mt5cli/SKILL.md +++ b/skills/mt5cli/SKILL.md @@ -32,10 +32,10 @@ Global options MUST precede the subcommand. | `-o, --output PATH` | Output file path (required). | | `-f, --format FORMAT` | `csv`, `json`, `parquet`, or `sqlite3` (auto from extension). | | `--table NAME` | Table name for SQLite3 output (default: `data`). | -| `--login INT` | MT5 trading account login. | -| `--password TEXT` | MT5 trading account password. | -| `--server TEXT` | MT5 trading server name. | -| `--path TEXT` | Path to MetaTrader 5 terminal EXE. | +| `--login INT` | MT5 trading account login (`MT5_LOGIN`). | +| `--password TEXT` | MT5 trading account password (`MT5_PASSWORD`). | +| `--server TEXT` | MT5 trading server name (`MT5_SERVER`). | +| `--path TEXT` | Path to MetaTrader 5 terminal EXE (`MT5_PATH`). | | `--timeout INT` | Connection timeout in milliseconds. | | `--log-level LEVEL` | `DEBUG`, `INFO`, `WARNING` (default), `ERROR`. | @@ -106,7 +106,8 @@ mt5cli -o history.db collect-history \ local MT5 terminal is already logged in. - Avoid passing `--password` on the command line in shared or logged environments — it is visible in `ps`, shell history, and CI logs. Prefer - logging in through the MT5 terminal first, then omit credentials here. + `MT5_PASSWORD`/`MT5_LOGIN`/`MT5_SERVER`/`MT5_PATH` environment variables or a + pre-authenticated local terminal session. - Reach for `--log-level DEBUG` when a command fails silently — MT5 connection errors surface there. - If the user asks to run from source in this repo, prefix with `uv run` diff --git a/tests/test_cli.py b/tests/test_cli.py index 734911e..a10518e 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -21,7 +21,9 @@ if TYPE_CHECKING: from mt5cli.cli import ( _execute_export, # type: ignore[reportPrivateUsage] _ExportContext, # type: ignore[reportPrivateUsage] + _infer_gap_table_granularity_seconds, # type: ignore[reportPrivateUsage] _sdk_client, # type: ignore[reportPrivateUsage] + _timeframe_interval_seconds, # type: ignore[reportPrivateUsage] app, main, ) @@ -631,8 +633,8 @@ class TestClosePositions: """Patch create_trading_client and return a mock trading client.""" client = _build_mock_trading_client() client.positions_get_as_df.return_value = pd.DataFrame([ - {"ticket": 1, "symbol": "JP225", "type": 0, "volume": 1.0}, - {"ticket": 2, "symbol": "EURUSD", "type": 1, "volume": 0.5}, + {"ticket": 1, "symbol": "JP225", "type": 0, "volume": 1.0, "magic": 7}, + {"ticket": 2, "symbol": "EURUSD", "type": 1, "volume": 0.5, "magic": 9}, ]) client.symbol_info_tick_as_dict.return_value = {"ask": 1.2, "bid": 1.1} mocker.patch("mt5cli.cli.create_trading_client", return_value=client) @@ -762,6 +764,40 @@ class TestClosePositions: assert data[0]["dry_run"] is True assert data[0]["order_side"] == "SELL" + def test_close_positions_dry_run_forwards_request_fields( + self, + tmp_path: Path, + trading_client: MagicMock, + ) -> None: + """Dry-run close export preserves deviation/comment/magic passthrough.""" + output = tmp_path / "close.json" + result = runner.invoke( + app, + [ + "-o", + str(output), + "close-positions", + "--symbol", + "JP225", + "--deviation", + "7", + "--comment", + "close-me", + "--magic", + "7", + "--dry-run", + ], + ) + assert result.exit_code == 0, result.output + trading_client.order_send.assert_not_called() + data = json.loads(output.read_text()) + assert len(data) == 1 + assert data[0]["symbol"] == "JP225" + request = json.loads(data[0]["request"]) + assert request["deviation"] == 7 + assert request["comment"] == "close-me" + assert request["magic"] == 7 + def test_order_send_unchanged( self, tmp_path: Path, @@ -847,11 +883,10 @@ class TestCallback: """Test that connection arguments reach Mt5Config.""" mock_client = MagicMock() mock_client.account_info_as_df.return_value = pd.DataFrame({"a": [1]}) - mocker.patch( + mt5_client = mocker.patch( "mt5cli.sdk.Mt5DataClient", return_value=mock_client, ) - mock_config = mocker.patch("mt5cli.cli.Mt5Config") output = tmp_path / "out.csv" result = runner.invoke( app, @@ -868,13 +903,132 @@ class TestCallback: ], ) assert result.exit_code == 0, result.output - mock_config.assert_called_once_with( - path=None, - login=123, - password="pw", - server="srv", - timeout=None, + config = mt5_client.call_args.kwargs["config"] + assert config.path is None + assert config.login == 123 + assert config.password is not None + assert config.password.get_secret_value() == "pw" + assert config.server == "srv" + assert config.timeout is None + + @pytest.mark.parametrize( + ("env", "extra_args", "expected"), + [ + pytest.param( + { + "MT5_LOGIN": "456", + "MT5_PASSWORD": "env-pass", + "MT5_SERVER": "Env-Server", + }, + [], + {"login": 456, "password": "env-pass", "server": "Env-Server"}, + id="env-defaults", + ), + pytest.param( + { + "MT5_LOGIN": "456", + "MT5_PASSWORD": "env-pass", + "MT5_SERVER": "Env-Server", + }, + ["--login", "123", "--password", "cli-pass", "--server", "Cli-Server"], + {"login": 123, "password": "cli-pass", "server": "Cli-Server"}, + id="cli-overrides-env", + ), + ], + ) + def test_connection_args_resolve_env_and_precedence( + self, + tmp_path: Path, + mocker: MockerFixture, + monkeypatch: pytest.MonkeyPatch, + env: dict[str, str], + extra_args: list[str], + expected: dict[str, object], + ) -> None: + """CLI args fall back to env vars and preserve explicit precedence.""" + for name, value in env.items(): + monkeypatch.setenv(name, value) + mock_client = MagicMock() + mock_client.account_info_as_df.return_value = pd.DataFrame({"a": [1]}) + mt5_client = mocker.patch( + "mt5cli.sdk.Mt5DataClient", + return_value=mock_client, ) + output = tmp_path / "out.csv" + + result = runner.invoke(app, [*extra_args, "-o", str(output), "account-info"]) + + assert result.exit_code == 0, result.output + config = mt5_client.call_args.kwargs["config"] + assert config.login == expected["login"] + assert config.password is not None + assert config.password.get_secret_value() == expected["password"] + assert config.server == expected["server"] + + def test_help_documents_mt5_env_vars(self) -> None: + """Top-level help output should expose the supported MT5 env vars.""" + result = runner.invoke(app, ["--help"]) + + assert result.exit_code == 0, result.output + normalized = normalize_cli_output(result.output) + for env_name in ("MT5_LOGIN", "MT5_PASSWORD", "MT5_SERVER", "MT5_PATH"): + assert env_name in normalized + + def test_gap_granularity_helpers_cover_unknown_cases(self) -> None: + """Gap-table granularity helpers should fail cleanly for unknown inputs.""" + assert _timeframe_interval_seconds(49153) is None + assert _infer_gap_table_granularity_seconds("custom_rates") is None + + @pytest.mark.parametrize( + ("args", "env", "exit_code", "match"), + [ + pytest.param( + ["--login", "${CLI_MT5_LOGIN}", "--server", "${CLI_MT5_SERVER}"], + {"CLI_MT5_LOGIN": "789", "CLI_MT5_SERVER": "Placeholder-Server"}, + 0, + None, + id="placeholder-expansion", + ), + pytest.param( + ["--password", "${CLI_MT5_MISSING}"], + {}, + 2, + "Environment variable 'CLI_MT5_MISSING' is not set.", + id="missing-placeholder", + ), + ], + ) + def test_cli_placeholder_resolution( + self, + tmp_path: Path, + mocker: MockerFixture, + monkeypatch: pytest.MonkeyPatch, + args: list[str], + env: dict[str, str], + exit_code: int, + match: str | None, + ) -> None: + """CLI config fields support SDK-style ${ENV_VAR} placeholders.""" + for name, value in env.items(): + monkeypatch.setenv(name, value) + mock_client = MagicMock() + mock_client.account_info_as_df.return_value = pd.DataFrame({"a": [1]}) + mt5_client = mocker.patch( + "mt5cli.sdk.Mt5DataClient", + return_value=mock_client, + ) + output = tmp_path / "out.csv" + + result = runner.invoke(app, [*args, "-o", str(output), "account-info"]) + + assert result.exit_code == exit_code, result.output + if exit_code == 0: + config = mt5_client.call_args.kwargs["config"] + assert config.login == 789 + assert config.server == "Placeholder-Server" + else: + assert match is not None + assert match in normalize_cli_output(result.output) def test_explicit_format( self, @@ -1526,6 +1680,131 @@ class TestCollectHistory: ) +class TestHistoryGapsCommand: + """Tests for the history-gaps CLI command.""" + + @pytest.mark.parametrize( + ("extra_args", "expected_tables", "expected_rows"), + [ + pytest.param([], {"rate_EURUSD__M1_1", "rate_GBPUSD__M1_1"}, 2, id="all"), + pytest.param( + ["--table", "rate_EURUSD__M1_1"], + {"rate_EURUSD__M1_1"}, + 1, + id="explicit-table", + ), + ], + ) + def test_history_gaps_exports_sqlite_report_without_mt5( + self, + tmp_path: Path, + mock_client: MagicMock, + extra_args: list[str], + expected_tables: set[str], + expected_rows: int, + ) -> None: + """history-gaps reads SQLite only and exports one row per gap.""" + database = tmp_path / "history.db" + output = tmp_path / "gaps.json" + with sqlite3.connect(database) as conn: + conn.execute( + "CREATE TABLE rates(" + "symbol TEXT, timeframe INTEGER, time TEXT, close REAL" + ")", + ) + conn.executemany( + "INSERT INTO rates(symbol, timeframe, time, close) VALUES (?, ?, ?, ?)", + [ + ("EURUSD", 1, "2024-01-01T00:00:00+00:00", 1.0), + ("EURUSD", 1, "2024-01-01T00:02:00+00:00", 1.1), + ("GBPUSD", 1, "2024-01-01T00:00:00+00:00", 1.2), + ("GBPUSD", 1, "2024-01-01T00:02:00+00:00", 1.3), + ], + ) + conn.execute( + 'CREATE VIEW "rate_EURUSD__M1_1" AS ' + "SELECT time, close FROM rates " + "WHERE symbol = 'EURUSD' AND timeframe = 1", + ) + conn.execute( + 'CREATE VIEW "rate_GBPUSD__M1_1" AS ' + "SELECT time, close FROM rates " + "WHERE symbol = 'GBPUSD' AND timeframe = 1", + ) + + result = runner.invoke( + app, + [ + "-o", + str(output), + "history-gaps", + "--sqlite3", + str(database), + *extra_args, + ], + ) + + assert result.exit_code == 0, result.output + data = json.loads(output.read_text()) + assert len(data) == expected_rows + assert {row["table"] for row in data} == expected_tables + mock_client.initialize_and_login_mt5.assert_not_called() + + def test_history_gaps_requires_compatible_default_views( + self, + tmp_path: Path, + ) -> None: + """Without --table, history-gaps should reject DBs with no managed views.""" + database = tmp_path / "empty.db" + output = tmp_path / "gaps.json" + with sqlite3.connect(database): + pass + + result = runner.invoke( + app, + ["-o", str(output), "history-gaps", "--sqlite3", str(database)], + ) + + assert result.exit_code != 0 + assert "No managed rate compatibility views found" in result.output + + def test_history_gaps_requires_granularity_for_custom_tables( + self, + tmp_path: Path, + ) -> None: + """Custom tables need an explicit granularity when no view naming exists.""" + database = tmp_path / "custom.db" + output = tmp_path / "gaps.json" + with sqlite3.connect(database) as conn: + conn.execute("CREATE TABLE custom_rates(time TEXT, close REAL)") + conn.executemany( + "INSERT INTO custom_rates(time, close) VALUES (?, ?)", + [ + ("2024-01-01T00:00:00+00:00", 1.0), + ("2024-01-01T00:02:00+00:00", 1.1), + ], + ) + + result = runner.invoke( + app, + [ + "-o", + str(output), + "history-gaps", + "--sqlite3", + str(database), + "--table", + "custom_rates", + ], + ) + + assert result.exit_code != 0 + output = normalize_cli_output(result.output) + assert "Could not infer granularity" in output + assert "'custom_rates'" in output + assert re.search(r"--granularity-\s*seconds", output) is not None + + class TestGrafanaSchemaCommand: """Tests for the grafana-schema CLI command.""" diff --git a/tests/test_history.py b/tests/test_history.py index a0361b6..9dfc298 100644 --- a/tests/test_history.py +++ b/tests/test_history.py @@ -48,6 +48,7 @@ from mt5cli.history import ( parse_sqlite_timestamp, quote_sqlite_identifier, record_written_columns, + report_rate_gaps, resolve_granularity_name, resolve_history_datasets, resolve_history_tick_flags, @@ -3075,6 +3076,299 @@ class TestRateSourceHelpers: assert set(result) == {("EURUSD", "M1"), ("EURUSD", "H1")} + def test_report_rate_gaps_reports_one_row_per_gap(self, tmp_path: Path) -> None: + """Gap reports emit one row for each detected missing interval run.""" + db_path = tmp_path / "gaps.db" + with sqlite3.connect(db_path) as conn: + conn.execute( + "CREATE TABLE rates(" + "symbol TEXT, timeframe INTEGER, time TEXT, close REAL" + ")", + ) + conn.executemany( + "INSERT INTO rates(symbol, timeframe, time, close) VALUES (?, ?, ?, ?)", + [ + ("EURUSD", 1, "2024-01-01T00:00:00+00:00", 1.0), + ("EURUSD", 1, "2024-01-01T00:01:00+00:00", 1.1), + ("EURUSD", 1, "2024-01-01T00:03:00+00:00", 1.2), + ], + ) + conn.execute( + 'CREATE VIEW "rate_EURUSD__M1_1" AS ' + "SELECT time, close FROM rates " + "WHERE symbol = 'EURUSD' AND timeframe = 1", + ) + result = report_rate_gaps( + conn, + "rate_EURUSD__M1_1", + granularity_seconds=60, + ) + + records = cast("list[dict[str, object]]", result.to_dict("records")) + assert records == [ + { + "table": "rate_EURUSD__M1_1", + "symbol": "EURUSD", + "timeframe": 1, + "granularity": "M1", + "granularity_seconds": 60, + "gap_start": datetime(2024, 1, 1, 0, 2, tzinfo=UTC), + "gap_end": datetime(2024, 1, 1, 0, 2, tzinfo=UTC), + "missing_intervals": 1, + }, + ] + + def test_report_rate_gaps_parses_numeric_sqlite_times_as_epoch_seconds( + self, + tmp_path: Path, + ) -> None: + """Numeric SQLite timestamps must be interpreted as Unix seconds.""" + db_path = tmp_path / "numeric-gaps.db" + with sqlite3.connect(db_path) as conn: + conn.execute("CREATE TABLE custom_rates(time INTEGER, close REAL)") + conn.executemany( + "INSERT INTO custom_rates(time, close) VALUES (?, ?)", + [ + (1704067200, 1.0), + (1704067320, 1.1), + ], + ) + result = report_rate_gaps( + conn, + "custom_rates", + granularity_seconds=60, + ) + + assert len(result) == 1 + assert result.iloc[0]["missing_intervals"] == 1 + + def test_report_rate_gaps_empty_schema_for_zero_gap_tables( + self, + tmp_path: Path, + ) -> None: + """Tables without gaps return the stable empty result schema.""" + db_path = tmp_path / "no-gaps.db" + with sqlite3.connect(db_path) as conn: + conn.execute("CREATE TABLE custom_rates(time TEXT, close REAL)") + conn.executemany( + "INSERT INTO custom_rates(time, close) VALUES (?, ?)", + [ + ("2024-01-01T00:00:00+00:00", 1.0), + ("2024-01-01T00:01:00+00:00", 1.1), + ], + ) + result = report_rate_gaps( + conn, + "custom_rates", + granularity_seconds=60, + ) + + assert list(result.columns) == [ + "table", + "symbol", + "timeframe", + "granularity", + "granularity_seconds", + "gap_start", + "gap_end", + "missing_intervals", + ] + assert result.empty + + def test_report_rate_gaps_filters_by_min_gap_intervals( + self, + tmp_path: Path, + ) -> None: + """Small gaps are filtered out when min_gap_intervals is raised.""" + db_path = tmp_path / "filtered-gaps.db" + with sqlite3.connect(db_path) as conn: + conn.execute("CREATE TABLE custom_rates(time TEXT, close REAL)") + conn.executemany( + "INSERT INTO custom_rates(time, close) VALUES (?, ?)", + [ + ("2024-01-01T00:00:00+00:00", 1.0), + ("2024-01-01T00:02:00+00:00", 1.1), + ], + ) + result = report_rate_gaps( + conn, + "custom_rates", + granularity_seconds=60, + min_gap_intervals=2, + ) + + assert result.empty + + def test_report_rate_gaps_computes_gaps_per_series_key( + self, + tmp_path: Path, + ) -> None: + """Managed rates tables must detect gaps within each symbol/timeframe series.""" + db_path = tmp_path / "series-gaps.db" + with sqlite3.connect(db_path) as conn: + conn.execute( + "CREATE TABLE rates(" + "symbol TEXT, timeframe INTEGER, time TEXT, close REAL" + ")", + ) + conn.executemany( + "INSERT INTO rates(symbol, timeframe, time, close) VALUES (?, ?, ?, ?)", + [ + ("EURUSD", 1, "2024-01-01T00:00:00+00:00", 1.0), + ("GBPUSD", 1, "2024-01-01T00:01:00+00:00", 1.1), + ("EURUSD", 1, "2024-01-01T00:02:00+00:00", 1.2), + ], + ) + + result = report_rate_gaps( + conn, + "rates", + granularity_seconds=60, + ) + + records = cast("list[dict[str, object]]", result.to_dict("records")) + assert records == [ + { + "table": "rates", + "symbol": "EURUSD", + "timeframe": 1, + "granularity": "M1", + "granularity_seconds": 60, + "gap_start": datetime(2024, 1, 1, 0, 1, tzinfo=UTC), + "gap_end": datetime(2024, 1, 1, 0, 1, tzinfo=UTC), + "missing_intervals": 1, + }, + ] + + def test_rate_gap_private_helpers_and_validation(self) -> None: + """Private helpers should preserve schema and reject invalid inputs.""" + empty = history._empty_rate_gap_report() # type: ignore[attr-defined] + assert list(empty.columns) == [ + "table", + "symbol", + "timeframe", + "granularity", + "granularity_seconds", + "gap_start", + "gap_end", + "missing_intervals", + ] + + class _BadInt: + def __int__(self) -> int: + msg = "bad-int" + raise ValueError(msg) + + assert history._coerce_optional_int(None) is None # type: ignore[attr-defined] + false_value: object = False + assert history._coerce_optional_int(false_value) is None # type: ignore[attr-defined] + assert history._coerce_optional_int(" +7 ") == 7 # type: ignore[attr-defined] + assert history._coerce_optional_int("bad") is None # type: ignore[attr-defined] + assert history._coerce_optional_int(object()) is None # type: ignore[attr-defined] + assert history._coerce_optional_int(_BadInt()) is None # type: ignore[attr-defined] + + metadata = history._rate_gap_metadata( # type: ignore[attr-defined] + "custom_rates", + pd.DataFrame({ + "symbol": ["EURUSD", "GBPUSD"], + "timeframe": [1, 1], + }), + granularity_seconds=60, + ) + assert metadata["symbol"] is None + assert metadata["timeframe"] == 1 + assert metadata["granularity"] == "M1" + + fallback_metadata = history._rate_gap_metadata( # type: ignore[attr-defined] + "rate_USDJPY__M1_1", + pd.DataFrame({"time": []}), + granularity_seconds=60, + ) + assert fallback_metadata["symbol"] == "USDJPY" + assert fallback_metadata["timeframe"] == 1 + assert fallback_metadata["granularity"] == "M1" + + unique_symbol_metadata = history._rate_gap_metadata( # type: ignore[attr-defined] + "custom_rates", + pd.DataFrame({"symbol": ["EURUSD"], "timeframe": [1]}), + granularity_seconds=60, + ) + assert unique_symbol_metadata["symbol"] == "EURUSD" + + multi_timeframe_metadata = history._rate_gap_metadata( # type: ignore[attr-defined] + "custom_rates", + pd.DataFrame({"timeframe": [1, 5]}), + granularity_seconds=60, + ) + assert multi_timeframe_metadata["timeframe"] is None + assert multi_timeframe_metadata["granularity"] is None + + @pytest.mark.parametrize( + ("granularity_seconds", "min_gap_intervals", "match"), + [ + pytest.param( + 0, 1, "granularity_seconds must be positive", id="bad-seconds" + ), + pytest.param(60, 0, "min_gap_intervals must be positive", id="bad-min-gap"), + ], + ) + def test_report_rate_gaps_rejects_invalid_parameters( + self, + tmp_path: Path, + granularity_seconds: int, + min_gap_intervals: int, + match: str, + ) -> None: + """Gap reports reject non-positive granularity and min-gap values.""" + db_path = tmp_path / "invalid-gaps.db" + with sqlite3.connect(db_path) as conn: + conn.execute("CREATE TABLE custom_rates(time TEXT, close REAL)") + conn.execute( + "INSERT INTO custom_rates(time, close) VALUES (?, ?)", + ("2024-01-01T00:00:00+00:00", 1.0), + ) + with pytest.raises(ValueError, match=match): + report_rate_gaps( + conn, + "custom_rates", + granularity_seconds=granularity_seconds, + min_gap_intervals=min_gap_intervals, + ) + + def test_report_rate_gaps_rejects_unparseable_timestamps( + self, + tmp_path: Path, + ) -> None: + """Unparseable table times should fail clearly.""" + db_path = tmp_path / "bad-times.db" + with sqlite3.connect(db_path) as conn: + conn.execute("CREATE TABLE custom_rates(time TEXT, close REAL)") + conn.execute( + "INSERT INTO custom_rates(time, close) VALUES (?, ?)", + ("bad-time", 1.0), + ) + with pytest.raises(ValueError, match="contains unparsable time values"): + report_rate_gaps( + conn, + "custom_rates", + granularity_seconds=60, + ) + + def test_report_rate_gaps_empty_and_single_row_sources_return_empty( + self, + tmp_path: Path, + ) -> None: + """Empty or single-row sources cannot produce gap rows.""" + db_path = tmp_path / "too-short.db" + with sqlite3.connect(db_path) as conn: + conn.execute("CREATE TABLE custom_rates(time TEXT, close REAL)") + assert report_rate_gaps(conn, "custom_rates", granularity_seconds=60).empty + conn.execute( + "INSERT INTO custom_rates(time, close) VALUES (?, ?)", + ("2024-01-01T00:00:00+00:00", 1.0), + ) + assert report_rate_gaps(conn, "custom_rates", granularity_seconds=60).empty + def test_load_rate_series_by_granularity_explicit_tables( self, tmp_path: Path, diff --git a/tests/test_trading.py b/tests/test_trading.py index b786e65..87bc96c 100644 --- a/tests/test_trading.py +++ b/tests/test_trading.py @@ -23,6 +23,7 @@ from mt5cli.trading import ( OrderLimits, OrderSide, ProjectionMode, + _filter_positions, # type: ignore[reportPrivateUsage] _Mt5ClientProtocol, # type: ignore[reportPrivateUsage] calculate_account_projected_margin_ratio, calculate_margin_and_volume, @@ -52,6 +53,7 @@ from mt5cli.trading import ( mt5_trading_session, normalize_order_volume, place_market_order, + resolve_broker_filling_mode, update_sltp_for_open_positions, update_trailing_stop_loss_for_open_positions, ) @@ -66,7 +68,11 @@ def _mock_trade_client() -> MagicMock: client.mt5.TRADE_ACTION_DEAL = 20 client.mt5.TRADE_ACTION_SLTP = 21 client.mt5.ORDER_FILLING_IOC = 30 + client.mt5.SYMBOL_FILLING_FOK = 1 + client.mt5.SYMBOL_FILLING_IOC = 2 client.mt5.ORDER_TIME_GTC = 40 + client.mt5.SYMBOL_TRADE_EXECUTION_MARKET = 3 + client.mt5.SYMBOL_TRADE_EXECUTION_REQUEST = 4 client.mt5.TRADE_RETCODE_PLACED = 10008 client.mt5.TRADE_RETCODE_DONE = 10009 client.mt5.TRADE_RETCODE_DONE_PARTIAL = 10010 @@ -115,6 +121,34 @@ class TestDetectPositionSide: assert detect_position_side(client, "EURUSD") == expected + def test_detect_position_side_filters_by_magic(self) -> None: + """Magic-scoped side detection ignores foreign positions.""" + client = MagicMock() + client.mt5.POSITION_TYPE_BUY = 0 + client.mt5.POSITION_TYPE_SELL = 1 + client.positions_get_as_df.return_value = pd.DataFrame( + [ + {"type": 0, "volume": 0.3, "magic": 7}, + {"type": 1, "volume": 0.2, "magic": 9}, + ], + ) + + assert detect_position_side(client, "EURUSD", magic=7) == "long" + assert detect_position_side(client, "EURUSD", magic=9) == "short" + + def test_detect_position_side_magic_is_fail_closed_without_magic_column( + self, + ) -> None: + """Magic-scoped side detection returns None without magic metadata.""" + client = MagicMock() + client.mt5.POSITION_TYPE_BUY = 0 + client.mt5.POSITION_TYPE_SELL = 1 + client.positions_get_as_df.return_value = pd.DataFrame( + [{"type": 0, "volume": 0.3}], + ) + + assert detect_position_side(client, "EURUSD", magic=7) is None + class TestCalculateMarginAndVolume: """Tests for calculate_margin_and_volume.""" @@ -2102,6 +2136,28 @@ class TestVolumeAndExecution: _assert_close(_request_from_result(result)["sl"], 1.0) _assert_close(_request_from_result(result)["tp"], 1.4) + def test_place_market_order_supports_optional_deviation_comment_and_magic( + self, + ) -> None: + """Test optional request metadata is preserved for market orders.""" + client = _mock_trade_client() + client.symbol_info_tick_as_dict.return_value = {"ask": 1.2, "bid": 1.1} + + result = place_market_order( + client, + symbol="EURUSD", + volume=0.1, + order_side="BUY", + deviation=7, + comment="close-me", + magic=42, + dry_run=True, + ) + + assert _request_from_result(result)["deviation"] == 7 + assert _request_from_result(result)["comment"] == "close-me" + assert _request_from_result(result)["magic"] == 42 + @pytest.mark.parametrize( ("mode_kwarg", "match"), [ @@ -2252,6 +2308,153 @@ class TestVolumeAndExecution: assert result["retcode"] == expected_retcode assert result["status"] == "failed" + @pytest.mark.parametrize( + ("symbol_info", "preferred_modes", "default_mode", "expected"), + [ + ( + {"filling_mode": 2, "trade_exemode": 3}, + ("IOC", "FOK"), + "IOC", + "IOC", + ), + ( + {"filling_mode": 1, "trade_exemode": 3}, + ("IOC", "FOK"), + "IOC", + "FOK", + ), + ( + {"filling_mode": 0, "trade_exemode": 4}, + ("RETURN", "IOC"), + "IOC", + "RETURN", + ), + ( + {"filling_mode": None, "trade_exemode": None}, + ("RETURN", "FOK"), + "IOC", + "RETURN", + ), + ( + {"filling_mode": 2, "trade_exemode": 3}, + ("FOK",), + "IOC", + "IOC", + ), + ( + {"filling_mode": 2, "trade_exemode": 3}, + ("FOK",), + "RETURN", + "IOC", + ), + ], + ids=[ + "ioc", + "fok-fallback", + "return", + "default-fallback", + "supported-default-fallback", + "ignore-unsupported-default", + ], + ) + def test_resolve_broker_filling_mode( + self, + symbol_info: dict[str, object], + preferred_modes: tuple[str, ...], + default_mode: str, + expected: str, + ) -> None: + """Test filling-mode resolution prefers supported modes then falls back.""" + client = _mock_trade_client() + client.symbol_info_as_dict.return_value = symbol_info + + result = resolve_broker_filling_mode( + client, + symbol="EURUSD", + preferred_modes=cast("Any", preferred_modes), + default_mode=cast("Any", default_mode), + ) + + assert result == expected + + @pytest.mark.parametrize( + ("preferred_modes", "default_mode"), + [ + pytest.param(("BAD",), "IOC", id="bad-preferred"), + pytest.param(("IOC",), "BAD", id="bad-default"), + ], + ) + def test_resolve_broker_filling_mode_rejects_invalid_mode_names( + self, + preferred_modes: tuple[str, ...], + default_mode: str, + ) -> None: + """Test invalid preferred/default filling mode names raise ValueError.""" + client = _mock_trade_client() + + with pytest.raises(ValueError, match="Unsupported order_filling mode"): + resolve_broker_filling_mode( + client, + symbol="EURUSD", + preferred_modes=cast("Any", preferred_modes), + default_mode=cast("Any", default_mode), + ) + + def test_resolve_broker_filling_mode_keeps_preferred_when_metadata_missing( + self, + caplog: pytest.LogCaptureFixture, + ) -> None: + """Missing metadata should fail open to the caller-preferred mode.""" + client = _mock_trade_client() + client.symbol_info_as_dict.return_value = {"filling_mode": None} + + with caplog.at_level(logging.DEBUG): + result = resolve_broker_filling_mode( + client, + symbol="EURUSD", + preferred_modes=("FOK", "IOC"), + ) + + assert result == "FOK" + assert "keeping preferred mode" in caplog.text + + def test_resolve_broker_filling_mode_supports_return_without_bitmask(self) -> None: + """RETURN should be allowed when execution mode is non-market.""" + client = _mock_trade_client() + client.symbol_info_as_dict.return_value = { + "filling_mode": None, + "trade_exemode": client.mt5.SYMBOL_TRADE_EXECUTION_REQUEST, + } + + result = resolve_broker_filling_mode( + client, + symbol="EURUSD", + preferred_modes=("RETURN", "FOK"), + ) + + assert result == "RETURN" + + def test_resolve_broker_filling_mode_keeps_preferred_when_metadata_unparseable( + self, + caplog: pytest.LogCaptureFixture, + ) -> None: + """Unparseable metadata should still fail open to the preferred mode.""" + client = _mock_trade_client() + client.symbol_info_as_dict.return_value = { + "filling_mode": 0, + "trade_exemode": client.mt5.SYMBOL_TRADE_EXECUTION_MARKET, + } + + with caplog.at_level(logging.DEBUG): + result = resolve_broker_filling_mode( + client, + symbol="EURUSD", + preferred_modes=("FOK", "IOC"), + ) + + assert result == "FOK" + assert "unparseable" in caplog.text + @pytest.mark.parametrize( ("filter_kwargs", "expected_order_side", "expected_position"), [ @@ -2306,6 +2509,50 @@ class TestVolumeAndExecution: assert client.order_send.call_args.args[0]["position"] == 9 + def test_close_open_positions_forwards_optional_request_fields(self) -> None: + """Test close helper forwards deviation/comment/magic into dry-run requests.""" + client = _mock_trade_client() + client.positions_get_as_df.return_value = pd.DataFrame( + [{"ticket": 9, "symbol": "EURUSD", "type": 0, "volume": 0.1, "magic": 42}], + ) + client.symbol_info_tick_as_dict.return_value = {"ask": 1.2, "bid": 1.1} + + result = close_open_positions( + client, + tickets=[9], + deviation=8, + comment="close-me", + magic=42, + dry_run=True, + ) + + request = _request_from_result(result[0]) + assert request["deviation"] == 8 + assert request["comment"] == "close-me" + assert request["magic"] == 42 + + def test_close_open_positions_magic_filter_is_fail_closed_without_column( + self, + ) -> None: + """Test magic-scoped close operations skip rows without magic metadata.""" + client = _mock_trade_client() + client.positions_get_as_df.return_value = pd.DataFrame( + [{"ticket": 9, "symbol": "EURUSD", "type": 0, "volume": 0.1}], + ) + + result = close_open_positions(client, magic=42, dry_run=True) + + assert result == [] + client.order_send.assert_not_called() + + def test_filter_positions_magic_is_fail_closed_without_magic_column(self) -> None: + """Test direct magic filtering fails closed when the DataFrame lacks magic.""" + positions = pd.DataFrame([{"ticket": 1, "symbol": "EURUSD"}]) + + result = _filter_positions(positions, magic=42) + + assert result.empty + def test_calculate_trailing_stop_updates_no_positions(self) -> None: """Test empty position sets produce no trailing updates.""" client = _mock_trade_client() diff --git a/uv.lock b/uv.lock index 4479b47..236bb1b 100644 --- a/uv.lock +++ b/uv.lock @@ -370,7 +370,7 @@ name = "metatrader5" version = "5.0.5735" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "numpy", marker = "sys_platform == 'win32'" }, + { name = "numpy" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/07/0b/3dd5143f14319dca393a3bcf11ddf24f36f16dfb8c6d1bf2b193ce0c5733/metatrader5-5.0.5735-cp311-cp311-win_amd64.whl", hash = "sha256:0d05b69cef5eb3f43ea47cb6d36dac0e7e15f82a4fb77974439c565daa54d1c9", size = 48091, upload-time = "2026-04-04T16:44:08.677Z" }, @@ -543,7 +543,7 @@ requires-dist = [ { name = "opentelemetry-exporter-otlp-proto-http", marker = "extra == 'otel'" }, { name = "opentelemetry-sdk", marker = "extra == 'otel'" }, { name = "pandas", specifier = ">=2.2.2" }, - { name = "pdmt5", specifier = ">=1.0.4" }, + { name = "pdmt5", specifier = ">=1.1.0" }, { name = "pyarrow", marker = "extra == 'parquet'", specifier = ">=19.0.0" }, { name = "pydantic", specifier = ">=2.13.4" }, { name = "typer", specifier = ">=0.15.0" }, @@ -800,16 +800,16 @@ wheels = [ [[package]] name = "pdmt5" -version = "1.0.4" +version = "1.1.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "metatrader5", marker = "sys_platform == 'win32'" }, { name = "pandas" }, { name = "pydantic" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/8c/96/2822d5d9a99c425b3472e803742ad87c972cbe01def7fadeb995a48bace4/pdmt5-1.0.4.tar.gz", hash = "sha256:2ced70cd1b6bb3ae99ac5f6a72420ba0da5ddd35857a61f13625556658d8d1b1", size = 22161, upload-time = "2026-07-03T14:41:30.575Z" } +sdist = { url = "https://files.pythonhosted.org/packages/35/7a/7caafb93b74ccaa60a1c844948698bed714e04aa3be292a05fd71864faa0/pdmt5-1.1.0.tar.gz", hash = "sha256:fcdab1924e001204ae776dd2afc1f150fe01e6944cba99dae72af23bc99137c6", size = 22160, upload-time = "2026-07-04T04:41:51.476Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/66/4e/b99e7824020c9108ea450304e36687324870abb87494434c8dbdf8ffe413/pdmt5-1.0.4-py3-none-any.whl", hash = "sha256:f5063a09e312d578874237f990b43484ffed892972e7771fa3154d642a0c1c83", size = 20485, upload-time = "2026-07-03T14:41:29.364Z" }, + { url = "https://files.pythonhosted.org/packages/6f/80/1062ea4e4c82de81d91ba8bb43b1a6d4903771010d87eca15d5d18d827df/pdmt5-1.1.0-py3-none-any.whl", hash = "sha256:603273065814824680d6ec8847139b3029c7dd04da7c25868ad7f4ede55a9acf", size = 20489, upload-time = "2026-07-04T04:41:49.999Z" }, ] [[package]]