diff --git a/README.md b/README.md index 0c34e1b..4dbeb0e 100644 --- a/README.md +++ b/README.md @@ -184,7 +184,7 @@ python -m mt5cli -o account.csv account-info | `order-check` | Check funds sufficiency for a trade request | | `order-send` | Send a raw trade request to the trade server (`--yes` required; expert path) | | `close-positions` | Close open positions by `--symbol` or `--ticket` (`--yes` required for live; `--dry-run` available) | -| `collect-history` | Bundle rates, ticks, history-orders, and history-deals for one or more symbols into a single SQLite database | +| `collect-history` | Collect rates, history-orders, and history-deals for one or more symbols into a single SQLite database (ticks opt-in via `--dataset ticks`) | Use `order-check` to validate a request payload before running `order-send --yes`. `close-positions` is the safer high-level alternative that builds correct close @@ -192,7 +192,7 @@ requests automatically. At least one `--symbol` or `--ticket` must be provided. ### `collect-history` -Collect several historical datasets per symbol into one SQLite database in a single MT5 session. Pick datasets with repeatable `--dataset` (default: all four), choose conflict behavior with `--if-exists append|replace|fail` (default: `fail`), and optionally derive `cash_events` / `positions_reconstructed` views from `history_deals` via `--with-views`. +Collect several historical datasets per symbol into one SQLite database in a single MT5 session. Pick datasets with repeatable `--dataset` (default: `rates`, `history-orders`, `history-deals`; add `--dataset ticks` when tick-level history is required — tick data can grow the SQLite database quickly), choose conflict behavior with `--if-exists append|replace|fail` (default: `fail`), and optionally derive `cash_events` / `positions_reconstructed` views from `history_deals` via `--with-views`. ```bash mt5cli -o history.db collect-history \ diff --git a/docs/index.md b/docs/index.md index 1313189..dee25d7 100644 --- a/docs/index.md +++ b/docs/index.md @@ -149,15 +149,15 @@ mt5cli --login 12345 --password mypass --server MyBroker-Demo \ ### Trading State -| Command | Description | -| ---------------------- | --------------------------------------------------------------------- | -| `orders` | Export active orders | -| `positions` | Export open positions | -| `history-orders` | Export historical orders | -| `history-deals` | Export historical deals | -| `recent-history-deals` | Export historical deals from a trailing window | -| `mt5-summary` | Export terminal/account status summary | -| `order-check` | Check funds sufficiency for a trade request (read-only, no `--yes`) | +| Command | Description | +| ---------------------- | ------------------------------------------------------------------- | +| `orders` | Export active orders | +| `positions` | Export open positions | +| `history-orders` | Export historical orders | +| `history-deals` | Export historical deals | +| `recent-history-deals` | Export historical deals from a trailing window | +| `mt5-summary` | Export terminal/account status summary | +| `order-check` | Check funds sufficiency for a trade request (read-only, no `--yes`) | ### Execution (live / mutating) @@ -176,9 +176,9 @@ applications should prefer dedicated closing helpers or their own risk controls. ### Bulk Collection -| Command | Description | -| ----------------- | -------------------------------------------------------------------------------------------------------------------------------------------------- | -| `collect-history` | Collect rates, ticks, history-orders, and history-deals for one or more symbols into a single SQLite database (optional cash-event/position views) | +| Command | Description | +| ----------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `collect-history` | Collect rates, history-orders, and history-deals (ticks opt-in via `--dataset ticks`) for one or more symbols into a single SQLite database (optional cash-event/position views) | ```bash mt5cli -o history.db collect-history \ @@ -190,16 +190,16 @@ mt5cli -o history.db collect-history \ `collect-history` options: -| Option | Default | Description | -| -------------- | ---------- | --------------------------------------------------------------------------------------------- | -| `--symbol/-s` | _required_ | Symbol to collect (repeat for multiple). | -| `--date-from` | _required_ | Start date in ISO 8601. | -| `--date-to` | _required_ | End date in ISO 8601. | -| `--dataset` | all four | Repeatable: `rates`, `ticks`, `history-orders`, `history-deals`. | -| `--timeframe` | `M1` | Rates timeframe; recorded in a `timeframe` column on the `rates` table. | -| `--flags` | `ALL` | Tick copy flags forwarded to `copy_ticks_range`. | -| `--if-exists` | `fail` | `append`, `replace`, or `fail` when a target table already exists. | -| `--with-views` | off | Add `cash_events` and `positions_reconstructed` views (requires the `history-deals` dataset). | +| Option | Default | Description | +| -------------- | ------------------------------------ | -------------------------------------------------------------------------------------------------------------------------- | +| `--symbol/-s` | _required_ | Symbol to collect (repeat for multiple). | +| `--date-from` | _required_ | Start date in ISO 8601. | +| `--date-to` | _required_ | End date in ISO 8601. | +| `--dataset` | rates, history-orders, history-deals | Repeatable: `rates`, `ticks`, `history-orders`, `history-deals`. Ticks are opt-in: pass `--dataset ticks` to include them. | +| `--timeframe` | `M1` | Rates timeframe; recorded in a `timeframe` column on the `rates` table. | +| `--flags` | `ALL` | Tick copy flags forwarded to `copy_ticks_range`. | +| `--if-exists` | `fail` | `append`, `replace`, or `fail` when a target table already exists. | +| `--with-views` | off | Add `cash_events` and `positions_reconstructed` views (requires the `history-deals` dataset). | History orders and deals are fetched per symbol and concatenated, so the symbol filter is applied consistently across all datasets. The `cash_events` view is derived from symbol-filtered `history_deals`, so account-level cash events with empty or non-matching symbols may be excluded. The `positions_reconstructed` view excludes positions with no closing deal, uses volume-weighted open/close prices, and reports reversal deals (`DEAL_ENTRY_INOUT`) via `volume_reversal` / `reversal_count`. diff --git a/mt5cli/cli.py b/mt5cli/cli.py index 6d39818..7883876 100644 --- a/mt5cli/cli.py +++ b/mt5cli/cli.py @@ -727,7 +727,8 @@ def collect_history( "--dataset", help=( "Dataset to include (repeat for multiple)." - " Defaults to all: rates, ticks, history-orders, history-deals." + " Defaults to rates, history-orders, history-deals." + " Ticks are opt-in: pass --dataset ticks to include them." ), ), ] = None, @@ -765,10 +766,12 @@ def collect_history( ) -> None: """Collect historical datasets into a single SQLite database. - Tables written depend on ``--dataset``: ``rates``, ``ticks``, - ``history_orders``, ``history_deals``. History datasets are fetched per - symbol and concatenated. Rates rows carry the requested ``timeframe`` so - appended runs at different timeframes remain distinguishable. + Tables written depend on ``--dataset``: ``rates``, ``history_orders``, + ``history_deals`` by default. ``ticks`` are opt-in: pass + ``--dataset ticks`` to include them (tick data grows the database quickly). + History datasets are fetched per symbol and concatenated. Rates rows carry + the requested ``timeframe`` so appended runs at different timeframes remain + distinguishable. With ``--with-views`` (requires the ``history-deals`` dataset), optional views ``cash_events`` and ``positions_reconstructed`` are derived from @@ -784,7 +787,7 @@ def collect_history( " Use a .db/.sqlite/.sqlite3 extension or --format sqlite3." ) raise typer.BadParameter(msg) - datasets = set(dataset) if dataset else set(Dataset) + datasets = set(dataset) if dataset is not None else None sdk.collect_history( output=export_ctx.output, symbols=symbol, diff --git a/mt5cli/history.py b/mt5cli/history.py index 7eacaef..566815f 100644 --- a/mt5cli/history.py +++ b/mt5cli/history.py @@ -30,6 +30,11 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) DEFAULT_HISTORY_TIMEFRAMES: tuple[str, ...] = TIMEFRAME_NAMES +DEFAULT_HISTORY_DATASETS: frozenset[Dataset] = frozenset({ + Dataset.rates, + Dataset.history_orders, + Dataset.history_deals, +}) _HISTORY_DEDUP_KEYS: dict[Dataset, tuple[tuple[str, ...], ...]] = { Dataset.rates: DEDUP_KEYS[DataKind.rates], @@ -62,11 +67,12 @@ def resolve_history_datasets(datasets: set[Dataset] | None) -> set[Dataset]: """Resolve configured history datasets. Returns: - All supported datasets when ``datasets`` is None, otherwise the - configured selection (which may be empty). + ``DEFAULT_HISTORY_DATASETS`` (rates, history-orders, history-deals) + when ``datasets`` is None, otherwise the configured selection (which + may be empty or explicitly include ``Dataset.ticks``). """ if datasets is None: - return set(Dataset) + return set(DEFAULT_HISTORY_DATASETS) return set(datasets) diff --git a/mt5cli/sdk.py b/mt5cli/sdk.py index 4669279..d3646be 100644 --- a/mt5cli/sdk.py +++ b/mt5cli/sdk.py @@ -981,7 +981,8 @@ def update_history( # noqa: PLR0913 client: Connected MT5 data client. output: SQLite database path. symbols: Symbols to update. - datasets: Datasets to include (defaults to all). + datasets: Datasets to include (defaults to rates, history-orders, + history-deals; pass ``{Dataset.ticks}`` to opt in to ticks). timeframes: Rate timeframes to update (defaults to all fixed MT5 timeframes when None). flags: Tick copy flags as integer or name (e.g. ``ALL``). @@ -1110,7 +1111,8 @@ class ThrottledHistoryUpdater: Args: output: SQLite database path. - datasets: Datasets to include (defaults to all). + datasets: Datasets to include (defaults to rates, history-orders, + history-deals; pass ``{Dataset.ticks}`` to opt in to ticks). timeframes: Rate timeframes to update (defaults to all fixed MT5 timeframes). flags: Tick copy flags as integer or name (e.g. ``ALL``). @@ -1242,7 +1244,8 @@ def collect_history( symbols: Symbols to collect. date_from: Start date. date_to: End date. - datasets: Datasets to include (defaults to all). + datasets: Datasets to include (defaults to rates, history-orders, + history-deals; pass ``{Dataset.ticks}`` to opt in to ticks). timeframe: Rates timeframe as integer or name (e.g. ``M1``). flags: Tick copy flags as integer or name (e.g. ``ALL``). if_exists: Behavior when a target table already exists. @@ -1251,7 +1254,7 @@ def collect_history( """ start = _require_datetime(date_from) end = _require_datetime(date_to) - selected = datasets if datasets is not None else set(Dataset) + selected = resolve_history_datasets(datasets) tf = _coerce_timeframe(timeframe) tick_flags = _coerce_tick_flags(flags) mt5_config = config or build_config() diff --git a/tests/test_cli.py b/tests/test_cli.py index 4ab8663..768539c 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1298,12 +1298,12 @@ class TestCollectHistory: """Create a mocked Mt5DataClient with history-style DataFrames.""" return _build_history_client(mocker) - def test_collect_history_writes_all_tables( + def test_collect_history_writes_default_tables( self, tmp_path: Path, history_client: MagicMock, ) -> None: - """Test that collect-history writes rates, ticks, and history tables.""" + """Test that collect-history default excludes ticks.""" output = tmp_path / "history.db" result = runner.invoke( app, @@ -1323,8 +1323,42 @@ class TestCollectHistory: ) assert result.exit_code == 0, result.output assert history_client.copy_rates_range_as_df.call_count == 2 - assert history_client.copy_ticks_range_as_df.call_count == 2 - history_client.copy_ticks_range_as_df.assert_any_call( + assert history_client.copy_ticks_range_as_df.call_count == 0 + with sqlite3.connect(output) as conn: + tables = { + row[0] + for row in conn.execute( + "SELECT name FROM sqlite_master WHERE type='table'", + ).fetchall() + } + assert {"rates", "history_orders", "history_deals"} <= tables + assert "ticks" not in tables + + def test_collect_history_explicit_ticks_dataset( + self, + tmp_path: Path, + history_client: MagicMock, + ) -> None: + """Test that --dataset ticks writes the ticks table with the correct flags.""" + output = tmp_path / "history.db" + result = runner.invoke( + app, + [ + "-o", + str(output), + "collect-history", + "--symbol", + "EURUSD", + "--date-from", + "2024-01-01", + "--date-to", + "2024-02-01", + "--dataset", + "ticks", + ], + ) + assert result.exit_code == 0, result.output + history_client.copy_ticks_range_as_df.assert_called_once_with( symbol="EURUSD", date_from=datetime(2024, 1, 1, tzinfo=UTC), date_to=datetime(2024, 2, 1, tzinfo=UTC), @@ -1337,7 +1371,8 @@ class TestCollectHistory: "SELECT name FROM sqlite_master WHERE type='table'", ).fetchall() } - assert {"rates", "ticks", "history_orders", "history_deals"} <= tables + assert "ticks" in tables + assert "rates" not in tables def test_collect_history_history_fetched_per_symbol( self, @@ -1520,7 +1555,7 @@ class TestCollectHistory: tmp_path: Path, history_client: MagicMock, ) -> None: - """Test that --flags defaults to ALL for ticks.""" + """Test that --flags defaults to ALL when --dataset ticks is explicit.""" output = tmp_path / "history.db" result = runner.invoke( app, @@ -1534,6 +1569,8 @@ class TestCollectHistory: "2024-01-01", "--date-to", "2024-02-01", + "--dataset", + "ticks", ], ) assert result.exit_code == 0, result.output diff --git a/tests/test_history.py b/tests/test_history.py index 10ba1fb..8bc24b3 100644 --- a/tests/test_history.py +++ b/tests/test_history.py @@ -19,6 +19,7 @@ from pdmt5 import TIMEFRAME_MAP from mt5cli import history from mt5cli.history import ( + DEFAULT_HISTORY_DATASETS, DEFAULT_HISTORY_TIMEFRAMES, DedupScope, RateTarget, @@ -547,10 +548,23 @@ class TestResolveHistorySettings: """Tests for history dataset and timeframe resolution.""" def test_resolve_history_datasets_defaults_and_empty(self) -> None: - """Test dataset resolution distinguishes None from empty selection.""" - assert resolve_history_datasets(None) == set(Dataset) + """Test dataset resolution excludes ticks by default.""" + resolved = resolve_history_datasets(None) + assert resolved == set(DEFAULT_HISTORY_DATASETS) + assert Dataset.ticks not in resolved + assert { + Dataset.rates, + Dataset.history_orders, + Dataset.history_deals, + } == resolved assert resolve_history_datasets(set()) == set() + def test_resolve_history_datasets_explicit_ticks(self) -> None: + """Test that explicit ticks selection is honored.""" + assert resolve_history_datasets({Dataset.ticks}) == {Dataset.ticks} + all_ds = resolve_history_datasets(set(Dataset)) + assert Dataset.ticks in all_ds + def test_resolve_history_timeframes_defaults(self) -> None: """Test default timeframes include all fixed MT5 values.""" resolved = resolve_history_timeframes(None) diff --git a/tests/test_sdk.py b/tests/test_sdk.py index bfc8fc8..1401e1c 100644 --- a/tests/test_sdk.py +++ b/tests/test_sdk.py @@ -618,12 +618,12 @@ class TestCollectHistory: """Create a mocked Mt5DataClient with history-style DataFrames.""" return _build_history_client(mocker) - def test_collect_history_writes_all_tables( + def test_collect_history_writes_default_tables( self, tmp_path: Path, history_client: MagicMock, ) -> None: - """Test that collect_history writes rates, ticks, and history tables.""" + """Test that collect_history default excludes ticks.""" output = tmp_path / "history.db" collect_history( output, @@ -632,7 +632,7 @@ class TestCollectHistory: "2024-02-01", ) assert history_client.copy_rates_range_as_df.call_count == 2 - assert history_client.copy_ticks_range_as_df.call_count == 2 + assert history_client.copy_ticks_range_as_df.call_count == 0 with sqlite3.connect(output) as conn: tables = { row[0] @@ -640,7 +640,34 @@ class TestCollectHistory: "SELECT name FROM sqlite_master WHERE type='table'", ).fetchall() } - assert {"rates", "ticks", "history_orders", "history_deals"} <= tables + assert {"rates", "history_orders", "history_deals"} <= tables + assert "ticks" not in tables + + def test_collect_history_explicit_ticks_dataset( + self, + tmp_path: Path, + history_client: MagicMock, + ) -> None: + """Test that explicit datasets={Dataset.ticks} writes the ticks table.""" + output = tmp_path / "history.db" + collect_history( + output, + ["EURUSD", "GBPUSD"], + "2024-01-01", + "2024-02-01", + datasets={Dataset.ticks}, + ) + assert history_client.copy_ticks_range_as_df.call_count == 2 + assert history_client.copy_rates_range_as_df.call_count == 0 + with sqlite3.connect(output) as conn: + tables = { + row[0] + for row in conn.execute( + "SELECT name FROM sqlite_master WHERE type='table'", + ).fetchall() + } + assert "ticks" in tables + assert "rates" not in tables def test_collect_history_with_views( self, @@ -1066,6 +1093,40 @@ class TestUpdateHistory: after = datetime.now(UTC) assert before <= captured["end"] <= after + def test_update_history_default_datasets_exclude_ticks( + self, + connected_client: MagicMock, + mocker: MockerFixture, + tmp_path: Path, + ) -> None: + """Test update_history with datasets=None does not collect ticks.""" + datasets_written: list[set[Dataset]] = [] + + def capture( + *args: object, + **_kwargs: object, + ) -> tuple[set[Dataset], dict[Dataset, set[str]]]: + datasets_written.append(args[3]) # type: ignore[arg-type] + return set(), {} + + mocker.patch("mt5cli.sdk.write_incremental_datasets", side_effect=capture) + update_history( + client=connected_client, + output=tmp_path / "default-datasets.db", + symbols=["EURUSD"], + datasets=None, + timeframes=["M1"], + lookback_hours=1, + date_to=datetime(2024, 1, 1, tzinfo=UTC), + ) + assert len(datasets_written) == 1 + assert Dataset.ticks not in datasets_written[0] + assert { + Dataset.rates, + Dataset.history_orders, + Dataset.history_deals, + } == datasets_written[0] + class TestRecentTicks: """Tests for recent_ticks helper."""