Fix mt5cli issues #92-#95 and #97 (#102)

* feat: add MT5 order metadata, coverage report, and env-backed CLI config

* fix: align review-driven trading and history contracts

* Bump pdmt5 to 1.1.0

* test: stabilize history gaps CLI assertion

* fix: address review follow-ups for gaps and filling mode
This commit is contained in:
Daichi Narushima
2026-07-04 14:19:56 +09:00
committed by GitHub
parent c2cf0656dd
commit 74e3754a80
15 changed files with 1372 additions and 80 deletions
+48 -33
View File
@@ -144,9 +144,10 @@ mt5cli -o ticks.json ticks-from --symbol EURUSD \
# Export symbols to SQLite3 with custom table name
mt5cli -o data.db --table symbols symbols --group "*USD*"
# Export with connection credentials
mt5cli --login 12345 --password mypass --server MyBroker-Demo \
-o positions.csv positions
# Export with connection credentials from env or placeholders
MT5_LOGIN=12345 MT5_PASSWORD=secret MT5_SERVER=MyBroker-Demo \
mt5cli -o positions.csv positions
MT5_PATH="/path/to/terminal64.exe" mt5cli -o positions.csv positions
```
Run as a Python module:
@@ -157,40 +158,54 @@ python -m mt5cli -o account.csv account-info
## Commands
| Command | Description |
| ---------------------- | ------------------------------------------------------------------------------------------------------------------------------------------- |
| `rates-from` | Export rates from a start date |
| `rates-from-pos` | Export rates from a start position |
| `latest-rates` | Export latest rates from a start position |
| `rates-range` | Export rates for a date range |
| `ticks-from` | Export ticks from a start date |
| `ticks-range` | Export ticks for a date range |
| `ticks-recent` | Export ticks from a recent trailing window |
| `account-info` | Export account information |
| `terminal-info` | Export terminal information |
| `version` | Export MetaTrader 5 version information |
| `last-error` | Export the last error information |
| `symbols` | Export symbol list |
| `symbol-info` | Export symbol details |
| `symbol-info-tick` | Export the last tick for a symbol |
| `minimum-margins` | Export minimum-volume buy and sell margin requirements |
| `market-book` | Export market depth (order book) |
| `orders` | Export active orders |
| `positions` | Export open positions |
| `history-orders` | Export historical orders |
| `history-deals` | Export historical deals |
| `recent-history-deals` | Export historical deals from a recent trailing window |
| `mt5-summary` | Export terminal/account status summary |
| `order-check` | Check funds sufficiency for a trade request |
| `order-send` | Send a raw trade request to the trade server (`--yes` required; expert path) |
| `close-positions` | Close open positions by `--symbol` or `--ticket` (`--yes` required for live; `--dry-run` available) |
| `collect-history` | Collect rates, history-orders, and history-deals for one or more symbols into a single SQLite database (ticks opt-in via `--dataset ticks`) |
| `grafana-schema` | Create or refresh Grafana-ready views and indexes in an existing SQLite database (idempotent, no MT5 connection) |
| `snapshot` | Snapshot current account, position, order, and terminal state into SQLite for live Grafana dashboards |
| Command | Description |
| ---------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------- |
| `rates-from` | Export rates from a start date |
| `rates-from-pos` | Export rates from a start position |
| `latest-rates` | Export latest rates from a start position |
| `rates-range` | Export rates for a date range |
| `ticks-from` | Export ticks from a start date |
| `ticks-range` | Export ticks for a date range |
| `ticks-recent` | Export ticks from a recent trailing window |
| `account-info` | Export account information |
| `terminal-info` | Export terminal information |
| `version` | Export MetaTrader 5 version information |
| `last-error` | Export the last error information |
| `symbols` | Export symbol list |
| `symbol-info` | Export symbol details |
| `symbol-info-tick` | Export the last tick for a symbol |
| `minimum-margins` | Export minimum-volume buy and sell margin requirements |
| `market-book` | Export market depth (order book) |
| `orders` | Export active orders |
| `positions` | Export open positions |
| `history-orders` | Export historical orders |
| `history-deals` | Export historical deals |
| `recent-history-deals` | Export historical deals from a recent trailing window |
| `mt5-summary` | Export terminal/account status summary |
| `order-check` | Check funds sufficiency for a trade request |
| `order-send` | Send a raw trade request to the trade server (`--yes` required; expert path) |
| `close-positions` | Close open positions by `--symbol` or `--ticket` (`--yes` required for live; `--dry-run` available; optional `--deviation` / `--comment` / `--magic`) |
| `collect-history` | Collect rates, history-orders, and history-deals for one or more symbols into a single SQLite database (ticks opt-in via `--dataset ticks`) |
| `history-gaps` | Export a SQLite-only one-row-per-gap report from managed rate compatibility views without connecting to MT5 |
| `grafana-schema` | Create or refresh Grafana-ready views and indexes in an existing SQLite database (idempotent, no MT5 connection) |
| `snapshot` | Snapshot current account, position, order, and terminal state into SQLite for live Grafana dashboards |
Use `order-check` to validate a request payload before running `order-send --yes`.
`close-positions` is the safer high-level alternative that builds correct close
requests automatically. At least one `--symbol` or `--ticket` must be provided.
CLI connection flags fall back to `MT5_LOGIN`, `MT5_PASSWORD`, `MT5_SERVER`,
and `MT5_PATH` when unset, and explicit CLI values still win.
### `history-gaps`
Inspect collected SQLite rate views offline and export one row per detected gap.
For managed compatibility views, the command infers bar spacing from the view
name. Use `--granularity-seconds` for custom tables or views.
```bash
mt5cli -o gaps.json history-gaps --sqlite3 history.db
mt5cli -o eurusd.csv history-gaps --sqlite3 history.db --table rate_EURUSD__M1_1
```
### `collect-history`
+13 -1
View File
@@ -70,6 +70,7 @@ timestamp normalization in downstream apps.
| Symbol | Role |
| ----------------------------------------------------------------- | -------------------------------------------------------------------------------------------- |
| `collect_history` | One-shot date-range export into SQLite |
| `report_rate_gaps` | SQLite-only one-row-per-gap report for a rate table or compatibility view |
| `update_history`, `update_history_with_config` | Incremental append from `MAX(time)` cursors |
| `ThrottledHistoryUpdater` | Minimum interval between successful incremental updates; optional `update_backend` injection |
| `RateTarget`, `build_rate_targets` | Neutral `(symbol, timeframe)` series descriptors |
@@ -98,6 +99,7 @@ strategy entries, exits, Kelly sizing, or signal logic.
| `calculate_symbol_group_margin_ratio` | Estimated symbol-group margin/equity with optional exposure |
| `determine_order_limits` | SL/TP price levels from ratios |
| `calculate_trailing_stop_updates` | Per-ticket generic trailing stop-loss update plan |
| `resolve_broker_filling_mode` | Broker-supported filling-mode selection helper |
| `ensure_symbol_selected` | Select/verify Market Watch visibility |
| `fetch_recent_history_deals_for_trading_client` | Recent deal history from a connected trading client |
| `place_market_order`, `close_open_positions`, `update_sltp_for_open_positions`, `update_trailing_stop_loss_for_open_positions` | Order execution helpers (`dry_run` supported) |
@@ -211,6 +213,9 @@ The Typer application in `mt5cli.cli` exposes file-export commands documented in
- Require `-o/--output` and write CSV, JSON, Parquet, or SQLite.
- Accept global MT5 connection options (`--login`, `--password`, `--server`,
`--path`, `--timeout`).
- Resolve unset CLI connection options from `MT5_LOGIN`, `MT5_PASSWORD`,
`MT5_SERVER`, and `MT5_PATH`, and expand `${ENV_VAR}` placeholders in CLI
string fields before building the MT5 config.
- Delegate to the same Python APIs described here; they are not duplicated
business logic.
@@ -228,7 +233,14 @@ constructed request payload. `close-positions` is the safer high-level helper
that closes open positions by `--symbol` or `--ticket` using
`close_open_positions()`. Both `order-send --yes` and `close-positions --yes`
are live execution paths. `close-positions --dry-run` previews close orders
without placing them and does not require `--yes`.
without placing them and does not require `--yes`. `close-positions` also
accepts optional `--deviation`, `--comment`, and `--magic`; `--magic` scopes
the selected open positions fail-closed when position magic metadata is absent.
`history-gaps` reads an existing SQLite history database and exports one row
per detected gap from managed rate compatibility views. It never initializes
MT5. Pass `--granularity-seconds` for custom tables or views whose bar spacing
cannot be inferred from the name.
## Internal helpers (not stable)
+10 -7
View File
@@ -109,8 +109,10 @@ mt5cli -o ticks.json ticks-from --symbol EURUSD \
# Export symbols to SQLite3 with custom table name
mt5cli -o data.db --table symbols symbols --group "*USD*"
# Export with connection credentials
mt5cli --login 12345 --password mypass --server MyBroker-Demo \
# Export with connection credentials from env or placeholders
MT5_LOGIN=12345 MT5_PASSWORD=secret MT5_SERVER=MyBroker-Demo \
mt5cli -o positions.csv positions
mt5cli --login '${MT5_LOGIN}' --password '${MT5_PASSWORD}' --server '${MT5_SERVER}' \
-o positions.csv positions
```
@@ -164,10 +166,10 @@ mt5cli --login 12345 --password mypass --server MyBroker-Demo \
These commands send requests to the live trade server and can place or close
real trades. Both require `--yes` for live execution.
| Command | Description |
| ----------------- | ---------------------------------------------------------------------------------------------------- |
| `order-send` | Send a **raw** trade request directly to MT5 (`--yes` required; expert path — no extra validation) |
| `close-positions` | Close open positions by `--symbol` or `--ticket` (`--yes` required for live; `--dry-run` to preview) |
| Command | Description |
| ----------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------ |
| `order-send` | Send a **raw** trade request directly to MT5 (`--yes` required; expert path — no extra validation) |
| `close-positions` | Close open positions by `--symbol` or `--ticket` (`--yes` required for live; `--dry-run` to preview; optional `--deviation` / `--comment` / `--magic`) |
Use `order-check` (Trading State) to validate funds before running `order-send --yes`.
`close-positions` is the safer high-level alternative that builds correct close
@@ -179,6 +181,7 @@ applications should prefer dedicated closing helpers or their own risk controls.
| Command | Description |
| ----------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| `collect-history` | Collect rates, history-orders, and history-deals (ticks opt-in via `--dataset ticks`) for one or more symbols into a single SQLite database (optional cash-event/position views) |
| `history-gaps` | Export a SQLite-only one-row-per-gap report from managed rate compatibility views without connecting to MT5 |
```bash
mt5cli -o history.db collect-history \
@@ -213,7 +216,7 @@ See the [History schema diagram](api/history.md#entity-relationship-diagram) for
| `-f, --format` | Output format (auto-detected from extension if omitted) |
| `--table` | Table name for SQLite3 output (default: "data") |
| `--login` | Trading account login |
| `--password` | Trading account password |
| `--password` | Trading account password (`MT5_PASSWORD`) |
| `--server` | Trading server name |
| `--path` | Path to MetaTrader5 terminal EXE file |
| `--timeout` | Connection timeout in milliseconds |
+4
View File
@@ -22,6 +22,7 @@ from .history import (
drop_forming_rate_bar,
load_rate_series_by_granularity,
load_rate_series_from_sqlite,
report_rate_gaps,
)
from .sdk import (
AccountSpec,
@@ -76,6 +77,7 @@ from .trading import (
mt5_trading_session,
normalize_order_volume,
place_market_order,
resolve_broker_filling_mode,
update_sltp_for_open_positions,
update_trailing_stop_loss_for_open_positions,
)
@@ -140,8 +142,10 @@ __all__ = [
"mt5_trading_session",
"normalize_order_volume",
"place_market_order",
"report_rate_gaps",
"resolve_account_spec",
"resolve_account_specs",
"resolve_broker_filling_mode",
"update_history",
"update_history_with_config",
"update_observability",
+166 -13
View File
@@ -4,6 +4,9 @@ from __future__ import annotations
import json
import logging
import os
import re
import sqlite3
from dataclasses import dataclass
from datetime import datetime # noqa: TC003
from pathlib import Path # noqa: TC003
@@ -11,10 +14,10 @@ from typing import TYPE_CHECKING, Annotated, Any, cast
import pandas as pd
import typer
from pdmt5 import Mt5Config
from . import sdk
from .client import MT5Client
from .history import report_rate_gaps, resolve_granularity_name
from .trading import OrderExecutionResult, close_open_positions, create_trading_client
from .utils import (
DATETIME_TYPE,
@@ -32,6 +35,8 @@ from .utils import (
if TYPE_CHECKING:
from collections.abc import Callable
from pdmt5 import Mt5Config
logger = logging.getLogger(__name__)
# ---------------------------------------------------------------------------
@@ -66,12 +71,55 @@ app = typer.Typer(
_REQUEST_OPTION_HELP = (
"Order request as a JSON object string, or '@path' to load JSON from a file."
)
_CLI_ENV_DEFAULTS: dict[str, str] = {
"path": "MT5_PATH",
"login": "MT5_LOGIN",
"password": "MT5_PASSWORD",
"server": "MT5_SERVER",
}
_RATE_VIEW_NAME_RE = re.compile(
r"^rate_(?P<symbol>.+)__(?:(?P<granularity>[A-Z0-9]+)_)?(?P<timeframe>\d+)$",
)
def _get_export_context(ctx: typer.Context) -> _ExportContext:
return cast("_ExportContext", ctx.obj)
def _resolve_cli_option(value: str | None, env_name: str) -> str | None:
return value if value is not None else os.environ.get(env_name)
def _timeframe_interval_seconds(timeframe: int) -> int | None:
granularity = resolve_granularity_name(timeframe)
units = {
"M": 60,
"H": 3600,
"D": 86400,
"W": 604800,
}
for prefix, seconds in units.items():
suffix = granularity.removeprefix(prefix)
if granularity.startswith(prefix) and suffix.isdigit():
return int(suffix) * seconds
return None
def _infer_gap_table_granularity_seconds(table: str) -> int | None:
if (match := _RATE_VIEW_NAME_RE.fullmatch(table)) is None:
return None
return _timeframe_interval_seconds(int(match.group("timeframe")))
def _default_gap_tables(conn: sqlite3.Connection) -> list[str]:
rows = conn.execute(
"SELECT name FROM sqlite_master"
" WHERE type IN ('table', 'view') AND name GLOB 'rate_*__*'"
" ORDER BY name",
).fetchall()
return [str(row[0]) for row in rows]
def _execute_export(
ctx: typer.Context,
fetch_fn: Callable[[], pd.DataFrame],
@@ -132,20 +180,36 @@ def _callback( # pyright: ignore[reportUnusedFunction]
typer.Option(help="Table name for SQLite3 output."),
] = "data",
login: Annotated[
int | None,
typer.Option(help="Trading account login."),
str | None,
typer.Option(
help="Trading account login.",
envvar=_CLI_ENV_DEFAULTS["login"],
show_envvar=True,
),
] = None,
password: Annotated[
str | None,
typer.Option(help="Trading account password."),
typer.Option(
help="Trading account password.",
envvar=_CLI_ENV_DEFAULTS["password"],
show_envvar=True,
),
] = None,
server: Annotated[
str | None,
typer.Option(help="Trading server name."),
typer.Option(
help="Trading server name.",
envvar=_CLI_ENV_DEFAULTS["server"],
show_envvar=True,
),
] = None,
path: Annotated[
str | None,
typer.Option(help="Path to MetaTrader5 terminal EXE file."),
typer.Option(
help="Path to MetaTrader5 terminal EXE file.",
envvar=_CLI_ENV_DEFAULTS["path"],
show_envvar=True,
),
] = None,
timeout: Annotated[
int | None,
@@ -169,17 +233,22 @@ def _callback( # pyright: ignore[reportUnusedFunction]
)
except ValueError as exc:
raise typer.BadParameter(str(exc)) from exc
try:
config = sdk.build_config(
path=_resolve_cli_option(path, _CLI_ENV_DEFAULTS["path"]),
login=_resolve_cli_option(login, _CLI_ENV_DEFAULTS["login"]),
password=_resolve_cli_option(password, _CLI_ENV_DEFAULTS["password"]),
server=_resolve_cli_option(server, _CLI_ENV_DEFAULTS["server"]),
timeout=timeout,
allow_whole_dollar_env=True,
)
except ValueError as exc:
raise typer.BadParameter(str(exc)) from exc
ctx.obj = _ExportContext(
output=output,
output_format=output_format,
table=table,
config=Mt5Config(
path=path,
login=login,
password=password,
server=server,
timeout=timeout,
),
config=config,
)
@@ -658,6 +727,20 @@ def close_positions(
help="Position ticket to close (repeat for multiple tickets).",
),
] = None,
deviation: Annotated[
int | None,
typer.Option(help="Optional slippage/deviation for each close request."),
] = None,
comment: Annotated[
str | None,
typer.Option(help="Optional comment attached to each close request."),
] = None,
magic: Annotated[
int | None,
typer.Option(
help="Optional magic tag for close requests and position filtering.",
),
] = None,
dry_run: Annotated[
bool,
typer.Option("--dry-run", help="Preview close orders without executing them."),
@@ -694,6 +777,9 @@ def close_positions(
client,
symbols=list(symbol) if symbol else None,
tickets=list(ticket) if ticket else None,
deviation=deviation,
comment=comment,
magic=magic,
dry_run=dry_run,
)
finally:
@@ -702,6 +788,73 @@ def close_positions(
_execute_export(ctx, lambda: df)
@app.command("history-gaps", rich_help_panel="Collection")
def history_gaps(
ctx: typer.Context,
sqlite3_path: Annotated[
Path,
typer.Option(
"--sqlite3",
help="Source SQLite history database to analyze.",
),
],
table: Annotated[
list[str] | None,
typer.Option(
"--table",
help="Rate table or compatibility view to inspect (repeat for multiple).",
),
] = None,
granularity_seconds: Annotated[
int | None,
typer.Option(help="Explicit bar interval in seconds for custom tables/views."),
] = None,
min_gap_intervals: Annotated[
int,
typer.Option(help="Minimum missing-bar count required to emit a gap row."),
] = 1,
) -> None:
"""Export SQLite rate gaps without connecting to MT5.
Raises:
typer.BadParameter: If no compatible rate view is available and no
explicit table is provided, or if granularity inference fails.
"""
with sqlite3.connect(sqlite3_path) as conn:
tables = list(table) if table else _default_gap_tables(conn)
if not tables:
msg = (
"No managed rate compatibility views found; pass --table for a rate "
"table or view."
)
raise typer.BadParameter(msg, param_hint="--table")
frames: list[pd.DataFrame] = []
for table_name in tables:
interval_seconds = (
granularity_seconds or _infer_gap_table_granularity_seconds(table_name)
)
if interval_seconds is None:
msg = (
f"Could not infer granularity for {table_name!r}; pass "
"--granularity-seconds."
)
raise typer.BadParameter(msg, param_hint="--granularity-seconds")
frames.append(
report_rate_gaps(
conn,
table_name,
granularity_seconds=interval_seconds,
min_gap_intervals=min_gap_intervals,
)
)
df = (
pd.concat(frames, ignore_index=True)
if frames
else pd.DataFrame(columns=["table"])
)
_execute_export(ctx, lambda: df)
@app.command(rich_help_panel="Collection")
def collect_history(
ctx: typer.Context,
+2
View File
@@ -59,6 +59,8 @@ STABLE_SDK_EXPORTS: frozenset[str] = frozenset({
"mt5_trading_session",
"normalize_order_volume",
"place_market_order",
"report_rate_gaps",
"resolve_broker_filling_mode",
"resolve_account_spec",
"resolve_account_specs",
"update_history",
+166 -2
View File
@@ -3,11 +3,12 @@
from __future__ import annotations
import logging
import re
import sqlite3
from dataclasses import dataclass
from datetime import UTC, datetime
from datetime import UTC, datetime, timedelta
from pathlib import Path
from typing import TYPE_CHECKING, Literal, cast, overload
from typing import TYPE_CHECKING, Literal, SupportsInt, cast, overload
import pandas as pd
from pdmt5 import get_timeframe_name as _get_timeframe_name
@@ -63,6 +64,20 @@ _POSITIONS_VIEW_REQUIRED_COLUMNS: frozenset[str] = frozenset({
"price",
"profit",
})
_RATE_GAP_COLUMNS: tuple[str, ...] = (
"table",
"symbol",
"timeframe",
"granularity",
"granularity_seconds",
"gap_start",
"gap_end",
"missing_intervals",
)
_RATE_VIEW_NAME_RE = re.compile(
r"^rate_(?P<symbol>.+)__(?:(?P<granularity>[A-Z0-9]+)_)?(?P<timeframe>\d+)$",
)
_MIN_TIMESTAMPS_FOR_GAPS = 2
def quote_sqlite_identifier(identifier: str) -> str:
@@ -231,6 +246,155 @@ def _open_existing_sqlite_database(
return conn, True
def _empty_rate_gap_report() -> pd.DataFrame:
return pd.DataFrame(columns=_RATE_GAP_COLUMNS)
def _coerce_optional_int(value: object) -> int | None:
if value is None or isinstance(value, bool):
return None
if isinstance(value, str):
text = value.strip()
if text.lstrip("+-").isdigit():
return int(text)
return None
if not hasattr(value, "__int__"):
return None
try:
return int(cast("SupportsInt", value))
except (TypeError, ValueError):
return None
def _rate_gap_metadata(
table: str,
frame: pd.DataFrame,
*,
granularity_seconds: int,
) -> dict[str, object]:
symbol: str | None = None
timeframe: int | None = None
granularity: str | None = None
if "symbol" in frame.columns:
symbols = {str(value) for value in frame["symbol"].dropna().unique()}
if len(symbols) == 1:
symbol = next(iter(symbols))
if "timeframe" in frame.columns:
timeframes = {
coerced
for value in frame["timeframe"].dropna().unique()
if (coerced := _coerce_optional_int(value)) is not None
}
if len(timeframes) == 1:
timeframe = next(iter(timeframes))
if timeframe is None and (match := _RATE_VIEW_NAME_RE.fullmatch(table)) is not None:
symbol = symbol or match.group("symbol")
timeframe = int(match.group("timeframe"))
granularity = match.group("granularity") or resolve_granularity_name(timeframe)
if timeframe is not None and granularity is None:
granularity = resolve_granularity_name(timeframe)
return {
"table": table,
"symbol": symbol,
"timeframe": timeframe,
"granularity": granularity,
"granularity_seconds": granularity_seconds,
}
def _iter_rate_gap_groups(frame: pd.DataFrame) -> list[pd.DataFrame]:
series_columns = [
column for column in ("symbol", "timeframe") if column in frame.columns
]
if not series_columns:
return [frame]
return [
group for _, group in frame.groupby(series_columns, dropna=False, sort=False)
]
def report_rate_gaps(
conn: sqlite3.Connection,
table: str,
*,
granularity_seconds: int,
min_gap_intervals: int = 1,
) -> pd.DataFrame:
"""Return one row per detected gap from a SQLite rate table or view.
Raises:
ValueError: If the table name, schema, timestamps, or gap parameters
are invalid.
"""
table_name = _validate_rate_load_request(table, count=None)
if granularity_seconds <= 0:
msg = "granularity_seconds must be positive."
raise ValueError(msg)
if min_gap_intervals <= 0:
msg = "min_gap_intervals must be positive."
raise ValueError(msg)
columns = get_table_columns(conn, table_name)
_ensure_rate_columns(columns, table_name)
quoted_table = quote_sqlite_identifier(table_name)
frame = cast(
"pd.DataFrame",
pd.read_sql_query( # type: ignore[reportUnknownMemberType]
f"SELECT * FROM {quoted_table} ORDER BY time ASC", # noqa: S608
conn,
),
)
if frame.empty:
return _empty_rate_gap_report()
parsed_times = frame["time"].map(parse_sqlite_timestamp)
if parsed_times.isna().any():
msg = f"SQLite table or view {table_name!r} contains unparsable time values."
raise ValueError(msg)
series_frame = frame.copy()
series_frame["time"] = parsed_times
rows: list[dict[str, object]] = []
for group in _iter_rate_gap_groups(series_frame):
unique_times = group["time"].drop_duplicates().sort_values(ignore_index=True)
if len(unique_times) < _MIN_TIMESTAMPS_FOR_GAPS:
continue
metadata = _rate_gap_metadata(
table_name,
group,
granularity_seconds=granularity_seconds,
)
deltas = unique_times.diff().dropna()
for index, delta in enumerate(deltas, start=1):
delta_seconds = int(delta.total_seconds())
missing_intervals = max(
((delta_seconds + (granularity_seconds - 1)) // granularity_seconds)
- 1,
0,
)
if missing_intervals < min_gap_intervals:
continue
previous_time = unique_times.iloc[index - 1]
next_time = unique_times.iloc[index]
rows.append({
**metadata,
"gap_start": (
previous_time.to_pydatetime()
+ timedelta(seconds=granularity_seconds)
),
"gap_end": (
next_time.to_pydatetime() - timedelta(seconds=granularity_seconds)
),
"missing_intervals": missing_intervals,
})
return pd.DataFrame(rows, columns=_RATE_GAP_COLUMNS)
def _validate_rate_load_request(table: str, count: int | None) -> str:
table_name = _require_non_empty_identifier(table, "table or view")
if count is not None and count <= 0:
+2
View File
@@ -33,6 +33,7 @@ from .history import (
create_history_indexes,
create_positions_reconstructed_view,
drop_forming_rate_bar,
report_rate_gaps,
resolve_granularity_name,
resolve_history_datasets,
resolve_history_tick_flags,
@@ -150,6 +151,7 @@ __all__ = [
"positions",
"recent_history_deals",
"recent_ticks",
"report_rate_gaps",
"resolve_account_spec",
"resolve_account_specs",
"substitute_env_placeholders",
+119 -3
View File
@@ -199,6 +199,7 @@ _SYMBOL_SNAPSHOT_FIELDS = (
"trade_tick_value",
"trade_stops_level",
"filling_mode",
"trade_exemode",
)
_TICK_SNAPSHOT_FIELDS = ("symbol", "time", "bid", "ask", "last", "volume")
POSITION_COLUMNS = (
@@ -214,6 +215,7 @@ POSITION_COLUMNS = (
"profit",
"swap",
"comment",
"magic",
)
__all__ = [
@@ -255,6 +257,7 @@ __all__ = [
"mt5_trading_session",
"normalize_order_volume",
"place_market_order",
"resolve_broker_filling_mode",
"update_sltp_for_open_positions",
"update_trailing_stop_loss_for_open_positions",
]
@@ -658,19 +661,24 @@ def create_trading_client(
def detect_position_side(
client: _Mt5ClientProtocol,
symbol: str,
*,
magic: int | None = None,
) -> PositionSide | None:
"""Detect the net open position side for a symbol.
Args:
client: Connected MT5 client instance.
symbol: Symbol to inspect.
magic: Optional magic number filter applied fail-closed.
Returns:
``"long"`` when there are buy positions and no sell positions,
``"short"`` when there are sell positions and no buy positions, or
``None`` when no positions or mixed exposure exists.
"""
positions = get_positions_frame(client, symbol=symbol)
positions = _filter_positions(
get_positions_frame(client, symbol=symbol), magic=magic
)
if positions.empty:
return None
@@ -734,6 +742,85 @@ def get_positions_frame(
return frame
def _supported_filling_modes(
client: _Mt5ClientProtocol,
*,
symbol: str,
preferred_default: OrderFillingMode,
) -> set[str] | None:
snapshot = get_symbol_snapshot(client, symbol)
filling_mode = _optional_int(snapshot.get("filling_mode"))
trade_exemode = _optional_int(snapshot.get("trade_exemode"))
if filling_mode is None and trade_exemode is None:
_logger.debug(
"Filling-mode metadata unavailable for %s; keeping preferred mode %s.",
symbol,
preferred_default,
)
return None
supported: set[str] = set()
if filling_mode is not None:
fok_flag = getattr(client.mt5, "SYMBOL_FILLING_FOK", None)
ioc_flag = getattr(client.mt5, "SYMBOL_FILLING_IOC", None)
if isinstance(fok_flag, int) and filling_mode & fok_flag:
supported.add("FOK")
if isinstance(ioc_flag, int) and filling_mode & ioc_flag:
supported.add("IOC")
market_execution = getattr(client.mt5, "SYMBOL_TRADE_EXECUTION_MARKET", None)
if trade_exemode is not None and not (
isinstance(market_execution, int) and trade_exemode == market_execution
):
supported.add("RETURN")
if supported:
return supported
_logger.debug(
"Filling-mode metadata was unparseable for %s; keeping preferred mode %s.",
symbol,
preferred_default,
)
return None
def resolve_broker_filling_mode(
client: _Mt5ClientProtocol,
*,
symbol: str,
preferred_modes: Sequence[OrderFillingMode] = ("IOC", "FOK", "RETURN"),
default_mode: OrderFillingMode = "IOC",
) -> OrderFillingMode:
"""Return the first broker-supported filling mode from a preferred order.
Raises:
ValueError: If any preferred or default mode name is unsupported.
"""
preferred = [mode.upper() for mode in preferred_modes]
for mode in [*preferred, default_mode]:
if mode not in _ORDER_FILLING_MODES:
msg = f"Unsupported order_filling mode: {mode!r}."
raise ValueError(msg)
preferred_default = cast(
"OrderFillingMode", preferred[0] if preferred else default_mode
)
supported = _supported_filling_modes(
client,
symbol=symbol,
preferred_default=preferred_default,
)
if supported is None:
return preferred_default
for mode in preferred:
if mode in supported:
return cast("OrderFillingMode", mode)
fallback_mode = next(
mode for mode in (default_mode, "IOC", "FOK", "RETURN") if mode in supported
)
return cast("OrderFillingMode", fallback_mode)
def _order_side_from_position_type(
client: _Mt5ClientProtocol,
position_type: object,
@@ -1334,7 +1421,7 @@ def determine_order_limits(
}
def place_market_order(
def place_market_order( # noqa: C901, PLR0913
client: _Mt5ClientProtocol,
*,
symbol: str,
@@ -1345,6 +1432,9 @@ def place_market_order(
sl: float | None = None,
tp: float | None = None,
position: int | None = None,
deviation: int | None = None,
comment: str | None = None,
magic: int | None = None,
dry_run: bool = False,
) -> OrderExecutionResult:
"""Place one normalized market order or return a dry-run result.
@@ -1398,6 +1488,12 @@ def place_market_order(
request["tp"] = tp
if position is not None:
request["position"] = position
if deviation is not None:
request["deviation"] = deviation
if comment is not None:
request["comment"] = comment
if magic is not None:
request["magic"] = magic
if dry_run:
return {
"status": "dry_run",
@@ -1432,6 +1528,7 @@ def _filter_positions(
*,
symbols: str | list[str] | None = None,
tickets: list[int] | None = None,
magic: int | None = None,
) -> pd.DataFrame:
frame = positions
if symbols is not None:
@@ -1439,6 +1536,10 @@ def _filter_positions(
frame = frame.loc[frame["symbol"].isin(symbol_set)]
if tickets is not None:
frame = frame.loc[frame["ticket"].isin(tickets)]
if magic is not None:
if "magic" not in frame.columns:
return frame.iloc[0:0].copy()
frame = frame.loc[frame["magic"] == magic]
return frame
@@ -1447,6 +1548,9 @@ def close_open_positions(
*,
symbols: str | list[str] | None = None,
tickets: list[int] | None = None,
deviation: int | None = None,
comment: str | None = None,
magic: int | None = None,
dry_run: bool = False,
) -> list[OrderExecutionResult]:
"""Close matching open positions.
@@ -1458,6 +1562,7 @@ def close_open_positions(
get_positions_frame(client),
symbols=symbols,
tickets=tickets,
magic=magic,
)
results: list[OrderExecutionResult] = []
for row in positions.to_dict("records"):
@@ -1469,6 +1574,9 @@ def close_open_positions(
volume=float(row["volume"]),
order_side=side,
position=int(row["ticket"]),
deviation=deviation,
comment=comment,
magic=magic,
dry_run=dry_run,
)
results.append(result)
@@ -1523,6 +1631,7 @@ def calculate_trailing_stop_updates(
*,
symbol: str,
trailing_stop_ratio: float,
magic: int | None = None,
) -> dict[int, float]:
"""Return per-ticket trailing stop-loss updates for open symbol positions.
@@ -1533,7 +1642,9 @@ def calculate_trailing_stop_updates(
missing side-specific tick price are skipped.
"""
_require_protective_ratio(trailing_stop_ratio, "trailing_stop_ratio")
positions = get_positions_frame(client, symbol=symbol)
positions = _filter_positions(
get_positions_frame(client, symbol=symbol), magic=magic
)
if positions.empty:
return {}
tick = get_tick_snapshot(client, symbol)
@@ -1568,6 +1679,7 @@ def update_trailing_stop_loss_for_open_positions(
*,
symbol: str,
trailing_stop_ratio: float,
magic: int | None = None,
dry_run: bool = False,
) -> list[OrderExecutionResult]:
"""Update open positions whose trailing stop loss should move favorably.
@@ -1579,6 +1691,7 @@ def update_trailing_stop_loss_for_open_positions(
client,
symbol=symbol,
trailing_stop_ratio=trailing_stop_ratio,
magic=magic,
)
results: list[OrderExecutionResult] = []
for ticket, stop_loss in updates.items():
@@ -1588,6 +1701,7 @@ def update_trailing_stop_loss_for_open_positions(
symbol=symbol,
tickets=[ticket],
stop_loss=stop_loss,
magic=magic,
dry_run=dry_run,
),
)
@@ -1599,6 +1713,7 @@ def update_sltp_for_open_positions(
*,
symbol: str | None = None,
tickets: list[int] | None = None,
magic: int | None = None,
stop_loss: float | None = None,
take_profit: float | None = None,
dry_run: bool = False,
@@ -1612,6 +1727,7 @@ def update_sltp_for_open_positions(
get_positions_frame(client),
symbols=symbol,
tickets=tickets,
magic=magic,
)
results: list[OrderExecutionResult] = []
for row in positions.to_dict("records"):
+1 -1
View File
@@ -9,7 +9,7 @@ license-files = ["LICENSE"]
readme = "README.md"
requires-python = ">= 3.11, < 3.14"
dependencies = [
"pdmt5>=1.0.4",
"pdmt5 >= 1.1.0",
"pandas >= 2.2.2",
"pydantic >= 2.13.4",
"click >= 8.1.0",
+6 -5
View File
@@ -32,10 +32,10 @@ Global options MUST precede the subcommand.
| `-o, --output PATH` | Output file path (required). |
| `-f, --format FORMAT` | `csv`, `json`, `parquet`, or `sqlite3` (auto from extension). |
| `--table NAME` | Table name for SQLite3 output (default: `data`). |
| `--login INT` | MT5 trading account login. |
| `--password TEXT` | MT5 trading account password. |
| `--server TEXT` | MT5 trading server name. |
| `--path TEXT` | Path to MetaTrader 5 terminal EXE. |
| `--login INT` | MT5 trading account login (`MT5_LOGIN`). |
| `--password TEXT` | MT5 trading account password (`MT5_PASSWORD`). |
| `--server TEXT` | MT5 trading server name (`MT5_SERVER`). |
| `--path TEXT` | Path to MetaTrader 5 terminal EXE (`MT5_PATH`). |
| `--timeout INT` | Connection timeout in milliseconds. |
| `--log-level LEVEL` | `DEBUG`, `INFO`, `WARNING` (default), `ERROR`. |
@@ -106,7 +106,8 @@ mt5cli -o history.db collect-history \
local MT5 terminal is already logged in.
- Avoid passing `--password` on the command line in shared or logged
environments — it is visible in `ps`, shell history, and CI logs. Prefer
logging in through the MT5 terminal first, then omit credentials here.
`MT5_PASSWORD`/`MT5_LOGIN`/`MT5_SERVER`/`MT5_PATH` environment variables or a
pre-authenticated local terminal session.
- Reach for `--log-level DEBUG` when a command fails silently — MT5
connection errors surface there.
- If the user asks to run from source in this repo, prefix with `uv run`
+289 -10
View File
@@ -21,7 +21,9 @@ if TYPE_CHECKING:
from mt5cli.cli import (
_execute_export, # type: ignore[reportPrivateUsage]
_ExportContext, # type: ignore[reportPrivateUsage]
_infer_gap_table_granularity_seconds, # type: ignore[reportPrivateUsage]
_sdk_client, # type: ignore[reportPrivateUsage]
_timeframe_interval_seconds, # type: ignore[reportPrivateUsage]
app,
main,
)
@@ -631,8 +633,8 @@ class TestClosePositions:
"""Patch create_trading_client and return a mock trading client."""
client = _build_mock_trading_client()
client.positions_get_as_df.return_value = pd.DataFrame([
{"ticket": 1, "symbol": "JP225", "type": 0, "volume": 1.0},
{"ticket": 2, "symbol": "EURUSD", "type": 1, "volume": 0.5},
{"ticket": 1, "symbol": "JP225", "type": 0, "volume": 1.0, "magic": 7},
{"ticket": 2, "symbol": "EURUSD", "type": 1, "volume": 0.5, "magic": 9},
])
client.symbol_info_tick_as_dict.return_value = {"ask": 1.2, "bid": 1.1}
mocker.patch("mt5cli.cli.create_trading_client", return_value=client)
@@ -762,6 +764,40 @@ class TestClosePositions:
assert data[0]["dry_run"] is True
assert data[0]["order_side"] == "SELL"
def test_close_positions_dry_run_forwards_request_fields(
self,
tmp_path: Path,
trading_client: MagicMock,
) -> None:
"""Dry-run close export preserves deviation/comment/magic passthrough."""
output = tmp_path / "close.json"
result = runner.invoke(
app,
[
"-o",
str(output),
"close-positions",
"--symbol",
"JP225",
"--deviation",
"7",
"--comment",
"close-me",
"--magic",
"7",
"--dry-run",
],
)
assert result.exit_code == 0, result.output
trading_client.order_send.assert_not_called()
data = json.loads(output.read_text())
assert len(data) == 1
assert data[0]["symbol"] == "JP225"
request = json.loads(data[0]["request"])
assert request["deviation"] == 7
assert request["comment"] == "close-me"
assert request["magic"] == 7
def test_order_send_unchanged(
self,
tmp_path: Path,
@@ -847,11 +883,10 @@ class TestCallback:
"""Test that connection arguments reach Mt5Config."""
mock_client = MagicMock()
mock_client.account_info_as_df.return_value = pd.DataFrame({"a": [1]})
mocker.patch(
mt5_client = mocker.patch(
"mt5cli.sdk.Mt5DataClient",
return_value=mock_client,
)
mock_config = mocker.patch("mt5cli.cli.Mt5Config")
output = tmp_path / "out.csv"
result = runner.invoke(
app,
@@ -868,13 +903,132 @@ class TestCallback:
],
)
assert result.exit_code == 0, result.output
mock_config.assert_called_once_with(
path=None,
login=123,
password="pw",
server="srv",
timeout=None,
config = mt5_client.call_args.kwargs["config"]
assert config.path is None
assert config.login == 123
assert config.password is not None
assert config.password.get_secret_value() == "pw"
assert config.server == "srv"
assert config.timeout is None
@pytest.mark.parametrize(
("env", "extra_args", "expected"),
[
pytest.param(
{
"MT5_LOGIN": "456",
"MT5_PASSWORD": "env-pass",
"MT5_SERVER": "Env-Server",
},
[],
{"login": 456, "password": "env-pass", "server": "Env-Server"},
id="env-defaults",
),
pytest.param(
{
"MT5_LOGIN": "456",
"MT5_PASSWORD": "env-pass",
"MT5_SERVER": "Env-Server",
},
["--login", "123", "--password", "cli-pass", "--server", "Cli-Server"],
{"login": 123, "password": "cli-pass", "server": "Cli-Server"},
id="cli-overrides-env",
),
],
)
def test_connection_args_resolve_env_and_precedence(
self,
tmp_path: Path,
mocker: MockerFixture,
monkeypatch: pytest.MonkeyPatch,
env: dict[str, str],
extra_args: list[str],
expected: dict[str, object],
) -> None:
"""CLI args fall back to env vars and preserve explicit precedence."""
for name, value in env.items():
monkeypatch.setenv(name, value)
mock_client = MagicMock()
mock_client.account_info_as_df.return_value = pd.DataFrame({"a": [1]})
mt5_client = mocker.patch(
"mt5cli.sdk.Mt5DataClient",
return_value=mock_client,
)
output = tmp_path / "out.csv"
result = runner.invoke(app, [*extra_args, "-o", str(output), "account-info"])
assert result.exit_code == 0, result.output
config = mt5_client.call_args.kwargs["config"]
assert config.login == expected["login"]
assert config.password is not None
assert config.password.get_secret_value() == expected["password"]
assert config.server == expected["server"]
def test_help_documents_mt5_env_vars(self) -> None:
"""Top-level help output should expose the supported MT5 env vars."""
result = runner.invoke(app, ["--help"])
assert result.exit_code == 0, result.output
normalized = normalize_cli_output(result.output)
for env_name in ("MT5_LOGIN", "MT5_PASSWORD", "MT5_SERVER", "MT5_PATH"):
assert env_name in normalized
def test_gap_granularity_helpers_cover_unknown_cases(self) -> None:
"""Gap-table granularity helpers should fail cleanly for unknown inputs."""
assert _timeframe_interval_seconds(49153) is None
assert _infer_gap_table_granularity_seconds("custom_rates") is None
@pytest.mark.parametrize(
("args", "env", "exit_code", "match"),
[
pytest.param(
["--login", "${CLI_MT5_LOGIN}", "--server", "${CLI_MT5_SERVER}"],
{"CLI_MT5_LOGIN": "789", "CLI_MT5_SERVER": "Placeholder-Server"},
0,
None,
id="placeholder-expansion",
),
pytest.param(
["--password", "${CLI_MT5_MISSING}"],
{},
2,
"Environment variable 'CLI_MT5_MISSING' is not set.",
id="missing-placeholder",
),
],
)
def test_cli_placeholder_resolution(
self,
tmp_path: Path,
mocker: MockerFixture,
monkeypatch: pytest.MonkeyPatch,
args: list[str],
env: dict[str, str],
exit_code: int,
match: str | None,
) -> None:
"""CLI config fields support SDK-style ${ENV_VAR} placeholders."""
for name, value in env.items():
monkeypatch.setenv(name, value)
mock_client = MagicMock()
mock_client.account_info_as_df.return_value = pd.DataFrame({"a": [1]})
mt5_client = mocker.patch(
"mt5cli.sdk.Mt5DataClient",
return_value=mock_client,
)
output = tmp_path / "out.csv"
result = runner.invoke(app, [*args, "-o", str(output), "account-info"])
assert result.exit_code == exit_code, result.output
if exit_code == 0:
config = mt5_client.call_args.kwargs["config"]
assert config.login == 789
assert config.server == "Placeholder-Server"
else:
assert match is not None
assert match in normalize_cli_output(result.output)
def test_explicit_format(
self,
@@ -1526,6 +1680,131 @@ class TestCollectHistory:
)
class TestHistoryGapsCommand:
"""Tests for the history-gaps CLI command."""
@pytest.mark.parametrize(
("extra_args", "expected_tables", "expected_rows"),
[
pytest.param([], {"rate_EURUSD__M1_1", "rate_GBPUSD__M1_1"}, 2, id="all"),
pytest.param(
["--table", "rate_EURUSD__M1_1"],
{"rate_EURUSD__M1_1"},
1,
id="explicit-table",
),
],
)
def test_history_gaps_exports_sqlite_report_without_mt5(
self,
tmp_path: Path,
mock_client: MagicMock,
extra_args: list[str],
expected_tables: set[str],
expected_rows: int,
) -> None:
"""history-gaps reads SQLite only and exports one row per gap."""
database = tmp_path / "history.db"
output = tmp_path / "gaps.json"
with sqlite3.connect(database) as conn:
conn.execute(
"CREATE TABLE rates("
"symbol TEXT, timeframe INTEGER, time TEXT, close REAL"
")",
)
conn.executemany(
"INSERT INTO rates(symbol, timeframe, time, close) VALUES (?, ?, ?, ?)",
[
("EURUSD", 1, "2024-01-01T00:00:00+00:00", 1.0),
("EURUSD", 1, "2024-01-01T00:02:00+00:00", 1.1),
("GBPUSD", 1, "2024-01-01T00:00:00+00:00", 1.2),
("GBPUSD", 1, "2024-01-01T00:02:00+00:00", 1.3),
],
)
conn.execute(
'CREATE VIEW "rate_EURUSD__M1_1" AS '
"SELECT time, close FROM rates "
"WHERE symbol = 'EURUSD' AND timeframe = 1",
)
conn.execute(
'CREATE VIEW "rate_GBPUSD__M1_1" AS '
"SELECT time, close FROM rates "
"WHERE symbol = 'GBPUSD' AND timeframe = 1",
)
result = runner.invoke(
app,
[
"-o",
str(output),
"history-gaps",
"--sqlite3",
str(database),
*extra_args,
],
)
assert result.exit_code == 0, result.output
data = json.loads(output.read_text())
assert len(data) == expected_rows
assert {row["table"] for row in data} == expected_tables
mock_client.initialize_and_login_mt5.assert_not_called()
def test_history_gaps_requires_compatible_default_views(
self,
tmp_path: Path,
) -> None:
"""Without --table, history-gaps should reject DBs with no managed views."""
database = tmp_path / "empty.db"
output = tmp_path / "gaps.json"
with sqlite3.connect(database):
pass
result = runner.invoke(
app,
["-o", str(output), "history-gaps", "--sqlite3", str(database)],
)
assert result.exit_code != 0
assert "No managed rate compatibility views found" in result.output
def test_history_gaps_requires_granularity_for_custom_tables(
self,
tmp_path: Path,
) -> None:
"""Custom tables need an explicit granularity when no view naming exists."""
database = tmp_path / "custom.db"
output = tmp_path / "gaps.json"
with sqlite3.connect(database) as conn:
conn.execute("CREATE TABLE custom_rates(time TEXT, close REAL)")
conn.executemany(
"INSERT INTO custom_rates(time, close) VALUES (?, ?)",
[
("2024-01-01T00:00:00+00:00", 1.0),
("2024-01-01T00:02:00+00:00", 1.1),
],
)
result = runner.invoke(
app,
[
"-o",
str(output),
"history-gaps",
"--sqlite3",
str(database),
"--table",
"custom_rates",
],
)
assert result.exit_code != 0
output = normalize_cli_output(result.output)
assert "Could not infer granularity" in output
assert "'custom_rates'" in output
assert re.search(r"--granularity-\s*seconds", output) is not None
class TestGrafanaSchemaCommand:
"""Tests for the grafana-schema CLI command."""
+294
View File
@@ -48,6 +48,7 @@ from mt5cli.history import (
parse_sqlite_timestamp,
quote_sqlite_identifier,
record_written_columns,
report_rate_gaps,
resolve_granularity_name,
resolve_history_datasets,
resolve_history_tick_flags,
@@ -3075,6 +3076,299 @@ class TestRateSourceHelpers:
assert set(result) == {("EURUSD", "M1"), ("EURUSD", "H1")}
def test_report_rate_gaps_reports_one_row_per_gap(self, tmp_path: Path) -> None:
"""Gap reports emit one row for each detected missing interval run."""
db_path = tmp_path / "gaps.db"
with sqlite3.connect(db_path) as conn:
conn.execute(
"CREATE TABLE rates("
"symbol TEXT, timeframe INTEGER, time TEXT, close REAL"
")",
)
conn.executemany(
"INSERT INTO rates(symbol, timeframe, time, close) VALUES (?, ?, ?, ?)",
[
("EURUSD", 1, "2024-01-01T00:00:00+00:00", 1.0),
("EURUSD", 1, "2024-01-01T00:01:00+00:00", 1.1),
("EURUSD", 1, "2024-01-01T00:03:00+00:00", 1.2),
],
)
conn.execute(
'CREATE VIEW "rate_EURUSD__M1_1" AS '
"SELECT time, close FROM rates "
"WHERE symbol = 'EURUSD' AND timeframe = 1",
)
result = report_rate_gaps(
conn,
"rate_EURUSD__M1_1",
granularity_seconds=60,
)
records = cast("list[dict[str, object]]", result.to_dict("records"))
assert records == [
{
"table": "rate_EURUSD__M1_1",
"symbol": "EURUSD",
"timeframe": 1,
"granularity": "M1",
"granularity_seconds": 60,
"gap_start": datetime(2024, 1, 1, 0, 2, tzinfo=UTC),
"gap_end": datetime(2024, 1, 1, 0, 2, tzinfo=UTC),
"missing_intervals": 1,
},
]
def test_report_rate_gaps_parses_numeric_sqlite_times_as_epoch_seconds(
self,
tmp_path: Path,
) -> None:
"""Numeric SQLite timestamps must be interpreted as Unix seconds."""
db_path = tmp_path / "numeric-gaps.db"
with sqlite3.connect(db_path) as conn:
conn.execute("CREATE TABLE custom_rates(time INTEGER, close REAL)")
conn.executemany(
"INSERT INTO custom_rates(time, close) VALUES (?, ?)",
[
(1704067200, 1.0),
(1704067320, 1.1),
],
)
result = report_rate_gaps(
conn,
"custom_rates",
granularity_seconds=60,
)
assert len(result) == 1
assert result.iloc[0]["missing_intervals"] == 1
def test_report_rate_gaps_empty_schema_for_zero_gap_tables(
self,
tmp_path: Path,
) -> None:
"""Tables without gaps return the stable empty result schema."""
db_path = tmp_path / "no-gaps.db"
with sqlite3.connect(db_path) as conn:
conn.execute("CREATE TABLE custom_rates(time TEXT, close REAL)")
conn.executemany(
"INSERT INTO custom_rates(time, close) VALUES (?, ?)",
[
("2024-01-01T00:00:00+00:00", 1.0),
("2024-01-01T00:01:00+00:00", 1.1),
],
)
result = report_rate_gaps(
conn,
"custom_rates",
granularity_seconds=60,
)
assert list(result.columns) == [
"table",
"symbol",
"timeframe",
"granularity",
"granularity_seconds",
"gap_start",
"gap_end",
"missing_intervals",
]
assert result.empty
def test_report_rate_gaps_filters_by_min_gap_intervals(
self,
tmp_path: Path,
) -> None:
"""Small gaps are filtered out when min_gap_intervals is raised."""
db_path = tmp_path / "filtered-gaps.db"
with sqlite3.connect(db_path) as conn:
conn.execute("CREATE TABLE custom_rates(time TEXT, close REAL)")
conn.executemany(
"INSERT INTO custom_rates(time, close) VALUES (?, ?)",
[
("2024-01-01T00:00:00+00:00", 1.0),
("2024-01-01T00:02:00+00:00", 1.1),
],
)
result = report_rate_gaps(
conn,
"custom_rates",
granularity_seconds=60,
min_gap_intervals=2,
)
assert result.empty
def test_report_rate_gaps_computes_gaps_per_series_key(
self,
tmp_path: Path,
) -> None:
"""Managed rates tables must detect gaps within each symbol/timeframe series."""
db_path = tmp_path / "series-gaps.db"
with sqlite3.connect(db_path) as conn:
conn.execute(
"CREATE TABLE rates("
"symbol TEXT, timeframe INTEGER, time TEXT, close REAL"
")",
)
conn.executemany(
"INSERT INTO rates(symbol, timeframe, time, close) VALUES (?, ?, ?, ?)",
[
("EURUSD", 1, "2024-01-01T00:00:00+00:00", 1.0),
("GBPUSD", 1, "2024-01-01T00:01:00+00:00", 1.1),
("EURUSD", 1, "2024-01-01T00:02:00+00:00", 1.2),
],
)
result = report_rate_gaps(
conn,
"rates",
granularity_seconds=60,
)
records = cast("list[dict[str, object]]", result.to_dict("records"))
assert records == [
{
"table": "rates",
"symbol": "EURUSD",
"timeframe": 1,
"granularity": "M1",
"granularity_seconds": 60,
"gap_start": datetime(2024, 1, 1, 0, 1, tzinfo=UTC),
"gap_end": datetime(2024, 1, 1, 0, 1, tzinfo=UTC),
"missing_intervals": 1,
},
]
def test_rate_gap_private_helpers_and_validation(self) -> None:
"""Private helpers should preserve schema and reject invalid inputs."""
empty = history._empty_rate_gap_report() # type: ignore[attr-defined]
assert list(empty.columns) == [
"table",
"symbol",
"timeframe",
"granularity",
"granularity_seconds",
"gap_start",
"gap_end",
"missing_intervals",
]
class _BadInt:
def __int__(self) -> int:
msg = "bad-int"
raise ValueError(msg)
assert history._coerce_optional_int(None) is None # type: ignore[attr-defined]
false_value: object = False
assert history._coerce_optional_int(false_value) is None # type: ignore[attr-defined]
assert history._coerce_optional_int(" +7 ") == 7 # type: ignore[attr-defined]
assert history._coerce_optional_int("bad") is None # type: ignore[attr-defined]
assert history._coerce_optional_int(object()) is None # type: ignore[attr-defined]
assert history._coerce_optional_int(_BadInt()) is None # type: ignore[attr-defined]
metadata = history._rate_gap_metadata( # type: ignore[attr-defined]
"custom_rates",
pd.DataFrame({
"symbol": ["EURUSD", "GBPUSD"],
"timeframe": [1, 1],
}),
granularity_seconds=60,
)
assert metadata["symbol"] is None
assert metadata["timeframe"] == 1
assert metadata["granularity"] == "M1"
fallback_metadata = history._rate_gap_metadata( # type: ignore[attr-defined]
"rate_USDJPY__M1_1",
pd.DataFrame({"time": []}),
granularity_seconds=60,
)
assert fallback_metadata["symbol"] == "USDJPY"
assert fallback_metadata["timeframe"] == 1
assert fallback_metadata["granularity"] == "M1"
unique_symbol_metadata = history._rate_gap_metadata( # type: ignore[attr-defined]
"custom_rates",
pd.DataFrame({"symbol": ["EURUSD"], "timeframe": [1]}),
granularity_seconds=60,
)
assert unique_symbol_metadata["symbol"] == "EURUSD"
multi_timeframe_metadata = history._rate_gap_metadata( # type: ignore[attr-defined]
"custom_rates",
pd.DataFrame({"timeframe": [1, 5]}),
granularity_seconds=60,
)
assert multi_timeframe_metadata["timeframe"] is None
assert multi_timeframe_metadata["granularity"] is None
@pytest.mark.parametrize(
("granularity_seconds", "min_gap_intervals", "match"),
[
pytest.param(
0, 1, "granularity_seconds must be positive", id="bad-seconds"
),
pytest.param(60, 0, "min_gap_intervals must be positive", id="bad-min-gap"),
],
)
def test_report_rate_gaps_rejects_invalid_parameters(
self,
tmp_path: Path,
granularity_seconds: int,
min_gap_intervals: int,
match: str,
) -> None:
"""Gap reports reject non-positive granularity and min-gap values."""
db_path = tmp_path / "invalid-gaps.db"
with sqlite3.connect(db_path) as conn:
conn.execute("CREATE TABLE custom_rates(time TEXT, close REAL)")
conn.execute(
"INSERT INTO custom_rates(time, close) VALUES (?, ?)",
("2024-01-01T00:00:00+00:00", 1.0),
)
with pytest.raises(ValueError, match=match):
report_rate_gaps(
conn,
"custom_rates",
granularity_seconds=granularity_seconds,
min_gap_intervals=min_gap_intervals,
)
def test_report_rate_gaps_rejects_unparseable_timestamps(
self,
tmp_path: Path,
) -> None:
"""Unparseable table times should fail clearly."""
db_path = tmp_path / "bad-times.db"
with sqlite3.connect(db_path) as conn:
conn.execute("CREATE TABLE custom_rates(time TEXT, close REAL)")
conn.execute(
"INSERT INTO custom_rates(time, close) VALUES (?, ?)",
("bad-time", 1.0),
)
with pytest.raises(ValueError, match="contains unparsable time values"):
report_rate_gaps(
conn,
"custom_rates",
granularity_seconds=60,
)
def test_report_rate_gaps_empty_and_single_row_sources_return_empty(
self,
tmp_path: Path,
) -> None:
"""Empty or single-row sources cannot produce gap rows."""
db_path = tmp_path / "too-short.db"
with sqlite3.connect(db_path) as conn:
conn.execute("CREATE TABLE custom_rates(time TEXT, close REAL)")
assert report_rate_gaps(conn, "custom_rates", granularity_seconds=60).empty
conn.execute(
"INSERT INTO custom_rates(time, close) VALUES (?, ?)",
("2024-01-01T00:00:00+00:00", 1.0),
)
assert report_rate_gaps(conn, "custom_rates", granularity_seconds=60).empty
def test_load_rate_series_by_granularity_explicit_tables(
self,
tmp_path: Path,
+247
View File
@@ -23,6 +23,7 @@ from mt5cli.trading import (
OrderLimits,
OrderSide,
ProjectionMode,
_filter_positions, # type: ignore[reportPrivateUsage]
_Mt5ClientProtocol, # type: ignore[reportPrivateUsage]
calculate_account_projected_margin_ratio,
calculate_margin_and_volume,
@@ -52,6 +53,7 @@ from mt5cli.trading import (
mt5_trading_session,
normalize_order_volume,
place_market_order,
resolve_broker_filling_mode,
update_sltp_for_open_positions,
update_trailing_stop_loss_for_open_positions,
)
@@ -66,7 +68,11 @@ def _mock_trade_client() -> MagicMock:
client.mt5.TRADE_ACTION_DEAL = 20
client.mt5.TRADE_ACTION_SLTP = 21
client.mt5.ORDER_FILLING_IOC = 30
client.mt5.SYMBOL_FILLING_FOK = 1
client.mt5.SYMBOL_FILLING_IOC = 2
client.mt5.ORDER_TIME_GTC = 40
client.mt5.SYMBOL_TRADE_EXECUTION_MARKET = 3
client.mt5.SYMBOL_TRADE_EXECUTION_REQUEST = 4
client.mt5.TRADE_RETCODE_PLACED = 10008
client.mt5.TRADE_RETCODE_DONE = 10009
client.mt5.TRADE_RETCODE_DONE_PARTIAL = 10010
@@ -115,6 +121,34 @@ class TestDetectPositionSide:
assert detect_position_side(client, "EURUSD") == expected
def test_detect_position_side_filters_by_magic(self) -> None:
"""Magic-scoped side detection ignores foreign positions."""
client = MagicMock()
client.mt5.POSITION_TYPE_BUY = 0
client.mt5.POSITION_TYPE_SELL = 1
client.positions_get_as_df.return_value = pd.DataFrame(
[
{"type": 0, "volume": 0.3, "magic": 7},
{"type": 1, "volume": 0.2, "magic": 9},
],
)
assert detect_position_side(client, "EURUSD", magic=7) == "long"
assert detect_position_side(client, "EURUSD", magic=9) == "short"
def test_detect_position_side_magic_is_fail_closed_without_magic_column(
self,
) -> None:
"""Magic-scoped side detection returns None without magic metadata."""
client = MagicMock()
client.mt5.POSITION_TYPE_BUY = 0
client.mt5.POSITION_TYPE_SELL = 1
client.positions_get_as_df.return_value = pd.DataFrame(
[{"type": 0, "volume": 0.3}],
)
assert detect_position_side(client, "EURUSD", magic=7) is None
class TestCalculateMarginAndVolume:
"""Tests for calculate_margin_and_volume."""
@@ -2102,6 +2136,28 @@ class TestVolumeAndExecution:
_assert_close(_request_from_result(result)["sl"], 1.0)
_assert_close(_request_from_result(result)["tp"], 1.4)
def test_place_market_order_supports_optional_deviation_comment_and_magic(
self,
) -> None:
"""Test optional request metadata is preserved for market orders."""
client = _mock_trade_client()
client.symbol_info_tick_as_dict.return_value = {"ask": 1.2, "bid": 1.1}
result = place_market_order(
client,
symbol="EURUSD",
volume=0.1,
order_side="BUY",
deviation=7,
comment="close-me",
magic=42,
dry_run=True,
)
assert _request_from_result(result)["deviation"] == 7
assert _request_from_result(result)["comment"] == "close-me"
assert _request_from_result(result)["magic"] == 42
@pytest.mark.parametrize(
("mode_kwarg", "match"),
[
@@ -2252,6 +2308,153 @@ class TestVolumeAndExecution:
assert result["retcode"] == expected_retcode
assert result["status"] == "failed"
@pytest.mark.parametrize(
("symbol_info", "preferred_modes", "default_mode", "expected"),
[
(
{"filling_mode": 2, "trade_exemode": 3},
("IOC", "FOK"),
"IOC",
"IOC",
),
(
{"filling_mode": 1, "trade_exemode": 3},
("IOC", "FOK"),
"IOC",
"FOK",
),
(
{"filling_mode": 0, "trade_exemode": 4},
("RETURN", "IOC"),
"IOC",
"RETURN",
),
(
{"filling_mode": None, "trade_exemode": None},
("RETURN", "FOK"),
"IOC",
"RETURN",
),
(
{"filling_mode": 2, "trade_exemode": 3},
("FOK",),
"IOC",
"IOC",
),
(
{"filling_mode": 2, "trade_exemode": 3},
("FOK",),
"RETURN",
"IOC",
),
],
ids=[
"ioc",
"fok-fallback",
"return",
"default-fallback",
"supported-default-fallback",
"ignore-unsupported-default",
],
)
def test_resolve_broker_filling_mode(
self,
symbol_info: dict[str, object],
preferred_modes: tuple[str, ...],
default_mode: str,
expected: str,
) -> None:
"""Test filling-mode resolution prefers supported modes then falls back."""
client = _mock_trade_client()
client.symbol_info_as_dict.return_value = symbol_info
result = resolve_broker_filling_mode(
client,
symbol="EURUSD",
preferred_modes=cast("Any", preferred_modes),
default_mode=cast("Any", default_mode),
)
assert result == expected
@pytest.mark.parametrize(
("preferred_modes", "default_mode"),
[
pytest.param(("BAD",), "IOC", id="bad-preferred"),
pytest.param(("IOC",), "BAD", id="bad-default"),
],
)
def test_resolve_broker_filling_mode_rejects_invalid_mode_names(
self,
preferred_modes: tuple[str, ...],
default_mode: str,
) -> None:
"""Test invalid preferred/default filling mode names raise ValueError."""
client = _mock_trade_client()
with pytest.raises(ValueError, match="Unsupported order_filling mode"):
resolve_broker_filling_mode(
client,
symbol="EURUSD",
preferred_modes=cast("Any", preferred_modes),
default_mode=cast("Any", default_mode),
)
def test_resolve_broker_filling_mode_keeps_preferred_when_metadata_missing(
self,
caplog: pytest.LogCaptureFixture,
) -> None:
"""Missing metadata should fail open to the caller-preferred mode."""
client = _mock_trade_client()
client.symbol_info_as_dict.return_value = {"filling_mode": None}
with caplog.at_level(logging.DEBUG):
result = resolve_broker_filling_mode(
client,
symbol="EURUSD",
preferred_modes=("FOK", "IOC"),
)
assert result == "FOK"
assert "keeping preferred mode" in caplog.text
def test_resolve_broker_filling_mode_supports_return_without_bitmask(self) -> None:
"""RETURN should be allowed when execution mode is non-market."""
client = _mock_trade_client()
client.symbol_info_as_dict.return_value = {
"filling_mode": None,
"trade_exemode": client.mt5.SYMBOL_TRADE_EXECUTION_REQUEST,
}
result = resolve_broker_filling_mode(
client,
symbol="EURUSD",
preferred_modes=("RETURN", "FOK"),
)
assert result == "RETURN"
def test_resolve_broker_filling_mode_keeps_preferred_when_metadata_unparseable(
self,
caplog: pytest.LogCaptureFixture,
) -> None:
"""Unparseable metadata should still fail open to the preferred mode."""
client = _mock_trade_client()
client.symbol_info_as_dict.return_value = {
"filling_mode": 0,
"trade_exemode": client.mt5.SYMBOL_TRADE_EXECUTION_MARKET,
}
with caplog.at_level(logging.DEBUG):
result = resolve_broker_filling_mode(
client,
symbol="EURUSD",
preferred_modes=("FOK", "IOC"),
)
assert result == "FOK"
assert "unparseable" in caplog.text
@pytest.mark.parametrize(
("filter_kwargs", "expected_order_side", "expected_position"),
[
@@ -2306,6 +2509,50 @@ class TestVolumeAndExecution:
assert client.order_send.call_args.args[0]["position"] == 9
def test_close_open_positions_forwards_optional_request_fields(self) -> None:
"""Test close helper forwards deviation/comment/magic into dry-run requests."""
client = _mock_trade_client()
client.positions_get_as_df.return_value = pd.DataFrame(
[{"ticket": 9, "symbol": "EURUSD", "type": 0, "volume": 0.1, "magic": 42}],
)
client.symbol_info_tick_as_dict.return_value = {"ask": 1.2, "bid": 1.1}
result = close_open_positions(
client,
tickets=[9],
deviation=8,
comment="close-me",
magic=42,
dry_run=True,
)
request = _request_from_result(result[0])
assert request["deviation"] == 8
assert request["comment"] == "close-me"
assert request["magic"] == 42
def test_close_open_positions_magic_filter_is_fail_closed_without_column(
self,
) -> None:
"""Test magic-scoped close operations skip rows without magic metadata."""
client = _mock_trade_client()
client.positions_get_as_df.return_value = pd.DataFrame(
[{"ticket": 9, "symbol": "EURUSD", "type": 0, "volume": 0.1}],
)
result = close_open_positions(client, magic=42, dry_run=True)
assert result == []
client.order_send.assert_not_called()
def test_filter_positions_magic_is_fail_closed_without_magic_column(self) -> None:
"""Test direct magic filtering fails closed when the DataFrame lacks magic."""
positions = pd.DataFrame([{"ticket": 1, "symbol": "EURUSD"}])
result = _filter_positions(positions, magic=42)
assert result.empty
def test_calculate_trailing_stop_updates_no_positions(self) -> None:
"""Test empty position sets produce no trailing updates."""
client = _mock_trade_client()
Generated
+5 -5
View File
@@ -370,7 +370,7 @@ name = "metatrader5"
version = "5.0.5735"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "numpy", marker = "sys_platform == 'win32'" },
{ name = "numpy" },
]
wheels = [
{ url = "https://files.pythonhosted.org/packages/07/0b/3dd5143f14319dca393a3bcf11ddf24f36f16dfb8c6d1bf2b193ce0c5733/metatrader5-5.0.5735-cp311-cp311-win_amd64.whl", hash = "sha256:0d05b69cef5eb3f43ea47cb6d36dac0e7e15f82a4fb77974439c565daa54d1c9", size = 48091, upload-time = "2026-04-04T16:44:08.677Z" },
@@ -543,7 +543,7 @@ requires-dist = [
{ name = "opentelemetry-exporter-otlp-proto-http", marker = "extra == 'otel'" },
{ name = "opentelemetry-sdk", marker = "extra == 'otel'" },
{ name = "pandas", specifier = ">=2.2.2" },
{ name = "pdmt5", specifier = ">=1.0.4" },
{ name = "pdmt5", specifier = ">=1.1.0" },
{ name = "pyarrow", marker = "extra == 'parquet'", specifier = ">=19.0.0" },
{ name = "pydantic", specifier = ">=2.13.4" },
{ name = "typer", specifier = ">=0.15.0" },
@@ -800,16 +800,16 @@ wheels = [
[[package]]
name = "pdmt5"
version = "1.0.4"
version = "1.1.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "metatrader5", marker = "sys_platform == 'win32'" },
{ name = "pandas" },
{ name = "pydantic" },
]
sdist = { url = "https://files.pythonhosted.org/packages/8c/96/2822d5d9a99c425b3472e803742ad87c972cbe01def7fadeb995a48bace4/pdmt5-1.0.4.tar.gz", hash = "sha256:2ced70cd1b6bb3ae99ac5f6a72420ba0da5ddd35857a61f13625556658d8d1b1", size = 22161, upload-time = "2026-07-03T14:41:30.575Z" }
sdist = { url = "https://files.pythonhosted.org/packages/35/7a/7caafb93b74ccaa60a1c844948698bed714e04aa3be292a05fd71864faa0/pdmt5-1.1.0.tar.gz", hash = "sha256:fcdab1924e001204ae776dd2afc1f150fe01e6944cba99dae72af23bc99137c6", size = 22160, upload-time = "2026-07-04T04:41:51.476Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/66/4e/b99e7824020c9108ea450304e36687324870abb87494434c8dbdf8ffe413/pdmt5-1.0.4-py3-none-any.whl", hash = "sha256:f5063a09e312d578874237f990b43484ffed892972e7771fa3154d642a0c1c83", size = 20485, upload-time = "2026-07-03T14:41:29.364Z" },
{ url = "https://files.pythonhosted.org/packages/6f/80/1062ea4e4c82de81d91ba8bb43b1a6d4903771010d87eca15d5d18d827df/pdmt5-1.1.0-py3-none-any.whl", hash = "sha256:603273065814824680d6ec8847139b3029c7dd04da7c25868ad7f4ede55a9acf", size = 20489, upload-time = "2026-07-04T04:41:49.999Z" },
]
[[package]]