[codex] Guard dedup scopes by written columns (#23)

* Guard dedup scopes by written columns

* Address dedup scope review feedback

* Remove legacy dedup scope support

* Remove stale legacy descriptions

* chore: bump version from 0.5.1 to 0.5.2

README.md

@@ -13,6 +13,7 @@ Built on top of [pdmt5](https://github.com/dceoy/pdmt5), a pandas-based data han
 - **Comprehensive data access**: Rates, ticks, account info, symbols, orders, positions, and trading history
 - **Flexible timeframes**: Named timeframes (M1, H1, D1, etc.) and numeric values
 - **Connection management**: Optional credentials, server, and timeout configuration
+- **SQLite rate loading**: Load mt5cli-managed rate tables/views for offline workflows
 
 ## Installation
 
@@ -50,28 +51,33 @@ python -m mt5cli -o account.csv account-info
 
 ## Commands
 
-| Command            | Description                                                                                                  |
+| Command                | Description                                                                                                  |
-| ------------------ | ------------------------------------------------------------------------------------------------------------ |
+| ---------------------- | ------------------------------------------------------------------------------------------------------------ |
-| `rates-from`       | Export rates from a start date                                                                               |
+| `rates-from`           | Export rates from a start date                                                                               |
-| `rates-from-pos`   | Export rates from a start position                                                                           |
+| `rates-from-pos`       | Export rates from a start position                                                                           |
-| `rates-range`      | Export rates for a date range                                                                                |
+| `latest-rates`         | Export latest rates from a start position                                                                    |
-| `ticks-from`       | Export ticks from a start date                                                                               |
+| `rates-range`          | Export rates for a date range                                                                                |
-| `ticks-range`      | Export ticks for a date range                                                                                |
+| `ticks-from`           | Export ticks from a start date                                                                               |
-| `account-info`     | Export account information                                                                                   |
+| `ticks-range`          | Export ticks for a date range                                                                                |
-| `terminal-info`    | Export terminal information                                                                                  |
+| `ticks-recent`         | Export ticks from a recent trailing window                                                                   |
-| `version`          | Export MetaTrader 5 version information                                                                      |
+| `account-info`         | Export account information                                                                                   |
-| `last-error`       | Export the last error information                                                                            |
+| `terminal-info`        | Export terminal information                                                                                  |
-| `symbols`          | Export symbol list                                                                                           |
+| `version`              | Export MetaTrader 5 version information                                                                      |
-| `symbol-info`      | Export symbol details                                                                                        |
+| `last-error`           | Export the last error information                                                                            |
-| `symbol-info-tick` | Export the last tick for a symbol                                                                            |
+| `symbols`              | Export symbol list                                                                                           |
-| `market-book`      | Export market depth (order book)                                                                             |
+| `symbol-info`          | Export symbol details                                                                                        |
-| `orders`           | Export active orders                                                                                         |
+| `symbol-info-tick`     | Export the last tick for a symbol                                                                            |
-| `positions`        | Export open positions                                                                                        |
+| `minimum-margins`      | Export minimum-volume buy and sell margin requirements                                                       |
-| `history-orders`   | Export historical orders                                                                                     |
+| `market-book`          | Export market depth (order book)                                                                             |
-| `history-deals`    | Export historical deals                                                                                      |
+| `orders`               | Export active orders                                                                                         |
-| `order-check`      | Check funds sufficiency for a trade request                                                                  |
+| `positions`            | Export open positions                                                                                        |
-| `order-send`       | Send a trade request to the trade server (`--yes` required)                                                  |
+| `history-orders`       | Export historical orders                                                                                     |
-| `collect-history`  | Bundle rates, ticks, history-orders, and history-deals for one or more symbols into a single SQLite database |
+| `history-deals`        | Export historical deals                                                                                      |
+| `recent-history-deals` | Export historical deals from a recent trailing window                                                        |
+| `mt5-summary`          | Export terminal/account status summary                                                                       |
+| `order-check`          | Check funds sufficiency for a trade request                                                                  |
+| `order-send`           | Send a trade request to the trade server (`--yes` required)                                                  |
+| `collect-history`      | Bundle rates, ticks, history-orders, and history-deals for one or more symbols into a single SQLite database |
 
 Use `order-check` to validate a request payload before running `order-send --yes`.
 
@@ -87,7 +93,53 @@ mt5cli -o history.db collect-history \
   --timeframe M1 --flags ALL --if-exists append --with-views
 ```
 
-History orders and deals are fetched per symbol and concatenated, so the symbol filter is applied consistently across all datasets. The `cash_events` view is derived from symbol-filtered `history_deals`, so account-level cash events with empty or non-matching symbols may be excluded. The `rates` table records the requested `timeframe` so appended runs at different timeframes remain distinguishable. The `positions_reconstructed` view aggregates trade deals by `position_id`, excludes positions without closing deals, and uses volume-weighted open/close prices; reversal deals (`DEAL_ENTRY_INOUT`) are reported via `volume_reversal` / `reversal_count` columns and do not contribute to the weighted prices.
+History orders and deals are fetched per symbol and concatenated, so the symbol filter is applied consistently across all datasets. The `cash_events` view is derived from symbol-filtered `history_deals`, so account-level cash events with empty or non-matching symbols may be excluded. The `rates` table records the requested `timeframe` so appended runs at different timeframes remain distinguishable. The `positions_reconstructed` view aggregates trade deals by `position_id`, excludes positions without closing-side entries, and uses volume-weighted open/close prices; reversal deals (`DEAL_ENTRY_INOUT`) are reported via `volume_reversal` / `reversal_count` columns.
+
+### Incremental history SDK
+
+For automated pipelines, use the importable incremental API instead of re-fetching fixed date ranges:
+
+```python
+from pdmt5 import Mt5Config, Mt5DataClient
+from mt5cli import Dataset, update_history, update_history_with_config
+
+# Reuse an already-connected pdmt5 client (does not open/close MT5)
+client = Mt5DataClient(config=Mt5Config(login=12345))
+client.initialize_and_login_mt5()
+try:
+    update_history(
+        client=client,
+        output="history.db",
+        symbols=["EURUSD", "GBPUSD"],
+        datasets={Dataset.rates, Dataset.history_deals},
+        timeframes=["M1", "H1"],  # default: all fixed MT5 timeframes
+        lookback_hours=24,
+        create_rate_views=True,
+        with_views=True,
+        include_account_events=True,
+    )
+finally:
+    client.shutdown()
+
+# Standalone wrapper that opens and closes MT5 for you
+update_history_with_config(
+    output="history.db",
+    symbols=["EURUSD"],
+    config=Mt5Config(login=12345),
+)
+```
+
+- **`collect-history`**: explicit date-range export into SQLite.
+- **`update_history`**: incremental append based on existing SQLite `MAX(time)` per symbol (and timeframe for rates); account-level deals use a separate cursor when `include_account_events=True`.
+- **`rates` table**: normalized storage with `symbol` and `timeframe` columns.
+- **Rate compatibility views**: mt5cli manages all `rate_*` views. Naming is `rate_<symbol>__<timeframe>` when a symbol has one timeframe, otherwise `rate_<symbol>__<granularity>_<timeframe>` (for example `rate_EURUSD__M1_1`). Stale `rate_*` views are dropped and recreated when rates change for offline tools such as mteor optimize.
+- **Rate view resolution**: use `resolve_rate_view_name()` / `resolve_rate_view_names()` to map symbols and granularities to existing SQLite compatibility views without creating databases. Both accept `None` (or a missing path) and return deterministic default names unless `require_existing=True`.
+- **Rate view loading**: use `load_rate_data()` / `load_rate_data_from_connection()` to load a SQLite rate table or view into a `DatetimeIndex` DataFrame.
+- **Multi-series rate loading**: use `build_rate_targets()` to build neutral `RateTarget(symbol, timeframe)` pairs, `resolve_rate_tables()` to map them to table/view names (pass `require_existing=True` for strict resolution), and `load_rate_series_from_sqlite()` to load them into a mapping keyed by `(symbol, integer timeframe)`. The loader requires existing managed views unless `explicit_tables` is supplied, and rejects duplicate `(symbol, timeframe)` targets.
+- **Multi-account latest rates**: use `collect_latest_rates_for_accounts()` with `AccountSpec` to read the latest bars for several account groups, merged into a `(symbol, integer timeframe)` mapping.
+- **MT5 session helper**: use the `mt5_session()` context manager to attach to (or, when `Mt5Config.path` is set, launch) an MT5 terminal, log in, and yield a connected `Mt5CliClient` that shuts down on exit.
+- **SQLite export helpers**: use `export_dataframe_to_sqlite()` for append mode, optional index export, and post-write deduplication by key columns.
+- **Recent ticks and margins**: `recent_ticks()` and `minimum_margins()` SDK helpers (and matching CLI commands) cover common downstream read-only queries.
 
 ## Requirements
 

docs/api/history.md

@@ -0,0 +1,217 @@
+# History Collection (SQLite)
+
+::: mt5cli.history
+
+## `collect-history` schema
+
+The `collect-history` command (and the matching `collect_history` SDK function) writes
+selected MT5 datasets into one SQLite database. Each dataset becomes a table; column
+names and types mirror the pdmt5 DataFrame schema for that export, with two additions:
+
+- `symbol` is prepended on every table.
+- `timeframe` is prepended on `rates` so appended runs at different bar sizes stay
+  distinguishable.
+
+SQLite does not declare foreign keys. Rows are linked logically by `symbol`, time
+windows, and (for deals) `position_id` / `order`. Duplicate rows are removed on
+append using dataset-specific keys (for example `ticket` on history tables, or
+`(symbol, timeframe, time)` on rates).
+
+Optional views are created when `--with-views` is set and the `history-deals` dataset
+was written.
+
+### Entity-relationship diagram
+
+Sample layout for a full collection with `--with-views`:
+
+```mermaid
+erDiagram
+    rates {
+        TEXT symbol "dedup key"
+        INTEGER timeframe "dedup key"
+        TEXT time "dedup key"
+        REAL open
+        REAL high
+        REAL low
+        REAL close
+        INTEGER tick_volume
+        INTEGER spread
+        INTEGER real_volume
+    }
+
+    ticks {
+        TEXT symbol "dedup key"
+        TEXT time "dedup key"
+        INTEGER time_msc "dedup key (preferred)"
+        REAL bid
+        REAL ask
+        REAL last
+        INTEGER volume
+        INTEGER flags
+        REAL volume_real
+    }
+
+    history_orders {
+        INTEGER ticket "dedup key"
+        TEXT symbol
+        TEXT time
+        INTEGER type
+        INTEGER state
+        REAL volume_initial
+        REAL price_open
+        REAL price_current
+        INTEGER magic
+    }
+
+    history_deals {
+        INTEGER ticket "dedup key"
+        INTEGER order
+        INTEGER position_id "groups position view"
+        TEXT symbol
+        TEXT time
+        INTEGER type "0/1 trade, else cash event"
+        INTEGER entry "0 IN, 1 OUT, 2 INOUT, 3 OUT_BY"
+        REAL volume
+        REAL price
+        REAL profit
+        REAL commission
+        REAL swap
+        REAL fee
+    }
+
+    cash_events {
+        INTEGER ticket
+        TEXT symbol
+        TEXT time
+        INTEGER type
+        REAL profit
+    }
+
+    positions_reconstructed {
+        INTEGER position_id
+        TEXT symbol
+        TEXT open_time
+        TEXT close_time
+        INTEGER direction
+        REAL volume_open
+        REAL volume_close
+        REAL volume_reversal
+        REAL open_price
+        REAL close_price
+        REAL total_profit
+        INTEGER reversal_count
+        INTEGER deals_count
+    }
+
+    rates ||--o{ history_deals : "symbol (logical)"
+    ticks ||--o{ history_deals : "symbol (logical)"
+    history_orders ||--o{ history_deals : "order ~ ticket (logical)"
+    history_deals ||--|| cash_events : "VIEW: type NOT IN (0,1)"
+    history_deals ||--o{ positions_reconstructed : "VIEW: GROUP BY position_id"
+```
+
+### Tables and views
+
+| Object                    | Kind  | Source               | Notes                                                                                       |
+| ------------------------- | ----- | -------------------- | ------------------------------------------------------------------------------------------- |
+| `rates`                   | table | `copy_rates_range`   | Indexed on `(symbol, timeframe, time)` when columns exist.                                  |
+| `ticks`                   | table | `copy_ticks_range`   | Indexed on `(symbol, time)` when columns exist.                                             |
+| `history_orders`          | table | `history_orders_get` | Fetched per `--symbol`, then concatenated.                                                  |
+| `history_deals`           | table | `history_deals_get`  | Fetched per `--symbol`, then concatenated. Indexed on `(position_id, symbol)` when present. |
+| `cash_events`             | view  | `history_deals`      | Non-trade deal types (deposits, balance ops, etc.). Requires `type` column.                 |
+| `positions_reconstructed` | view  | `history_deals`      | One row per closed `position_id`; volume-weighted prices and reversal stats.                |
+
+Column sets can vary with terminal and pdmt5 version. Views are skipped with a warning
+when required columns are missing.
+
+### Incremental collection
+
+The `update_history` SDK path uses the same base tables and optional
+`cash_events` / `positions_reconstructed` views. It additionally maintains
+`rate_<symbol>__<timeframe>` compatibility views when `create_rate_views=True`.
+
+### Rate view resolution
+
+Downstream tools can resolve mt5cli-managed compatibility view names from an
+existing SQLite history database without creating files or guessing naming
+schemes:
+
+```python
+from pathlib import Path
+
+from mt5cli.history import resolve_rate_view_name, resolve_rate_view_names
+
+# Single symbol and granularity
+view = resolve_rate_view_name(Path("history.db"), "EURUSD", "M1")
+
+# Batch resolution in row-major order
+views = resolve_rate_view_names(
+    Path("history.db"),
+    ["EURUSD", "GBPUSD"],
+    ["M1", "H1"],
+)
+```
+
+Resolution rules:
+
+- Returns `rate_<symbol>__<timeframe>` when a symbol stores one timeframe.
+- Returns `rate_<symbol>__<granularity>_<timeframe>` when multiple timeframes
+  are stored for the same symbol.
+- When multiple naming candidates apply, prefers an existing managed
+  `rate_*__*` view from the candidate list.
+- Falls back to single-timeframe naming when the database path is missing or
+  `rates` metadata is unavailable.
+- Pass `require_existing=True` to raise `ValueError` instead of returning a
+  best-guess name when the database or view is missing.
+- Accepts either a SQLite path or an open `sqlite3.Connection`.
+
+### Rate data loading
+
+Use `load_rate_data()` to load a table or view from a SQLite path, or
+`load_rate_data_from_connection()` when you already have a connection:
+
+```python
+from pathlib import Path
+
+from mt5cli import load_rate_data
+from mt5cli.history import resolve_rate_view_name
+
+view = resolve_rate_view_name(Path("history.db"), "EURUSD", "M1", require_existing=True)
+rates = load_rate_data(Path("history.db"), view, count=1000)
+```
+
+The loader accepts close-based OHLC rate data or tick-like bid/ask data. It
+validates that `time` exists, parses timestamps with pandas, and returns a
+DataFrame indexed by ascending `DatetimeIndex` named `time`.
+
+### Multi-series rate loading
+
+For loading many rate series at once, build neutral `RateTarget` pairs and load
+them from SQLite in one call. View names are resolved via the same
+compatibility-view rules, or you can pass `explicit_tables` to bypass resolution:
+
+```python
+from pathlib import Path
+
+from mt5cli import build_rate_targets, load_rate_series_from_sqlite
+
+targets = build_rate_targets(["EURUSD", "GBPUSD"], ["M1", "H1"])
+series = load_rate_series_from_sqlite(Path("history.db"), targets, count=1000)
+frame = series["EURUSD", 1]  # keyed by (symbol, integer timeframe)
+```
+
+- `build_rate_targets()` returns `RateTarget(symbol, timeframe)` pairs in
+  row-major order, normalizing timeframe names such as `"M1"` to their integer
+  values; set `allow_missing_symbol=True` to address series solely by
+  `explicit_tables` (targets carry `symbol=None`).
+- `resolve_rate_tables()` maps targets to table or view names and validates that
+  any `explicit_tables` count matches the target count. Pass
+  `require_existing=True` to raise `ValueError` instead of returning a
+  best-guess name when the database or managed view is missing. When
+  `explicit_tables` is provided, names are returned as-is and
+  `require_existing` is ignored.
+- `load_rate_series_from_sqlite()` returns a mapping keyed by
+  `(symbol, integer timeframe)`. Unless `explicit_tables` is supplied, it
+  requires existing managed `rate_*` compatibility views and raises
+  `ValueError` when they are missing. Duplicate `(symbol, timeframe)` targets
+  are rejected.

docs/api/index.md

@@ -18,6 +18,10 @@ Utility module providing constants, enums, Click parameter types, and helper fun
 
 Programmatic SDK for read-only MetaTrader 5 data collection. Returns pandas DataFrames and provides `collect_history` for SQLite bulk collection.
 
+### [History Collection (SQLite)](history.md)
+
+SQLite storage helpers for the `collect-history` command schema, incremental updates, deduplication, indexes, and optional views.
+
 ## Architecture Overview
 
 The package follows a simple architecture built on top of pdmt5:
@@ -61,12 +65,18 @@ from datetime import UTC, datetime
 from pathlib import Path
 
 from mt5cli import (
+    Dataset,
+    IfExists,
     Mt5CliClient,
     collect_history,
     copy_rates_range,
     detect_format,
     export_dataframe,
+    export_dataframe_to_sqlite,
+    minimum_margins,
+    recent_ticks,
 )
+from mt5cli.history import resolve_rate_view_name
 
 # Fetch rates programmatically
 rates = copy_rates_range(
@@ -82,6 +92,20 @@ fmt = detect_format(Path("output.parquet"))  # Returns "parquet"
 # Export a DataFrame
 export_dataframe(rates, Path("output.csv"), "csv")
 
+# Append to SQLite with deduplication
+export_dataframe_to_sqlite(
+    rates,
+    Path("history.db"),
+    "rates",
+    if_exists=IfExists.APPEND,
+    deduplicate_on=("symbol", "timeframe", "time"),
+)
+
+# Resolve rate compatibility views and fetch recent ticks
+view = resolve_rate_view_name(Path("history.db"), "EURUSD", "M1")
+ticks = recent_ticks("EURUSD", seconds=300)
+margins = minimum_margins("EURUSD")
+
 # Collect history into SQLite
 collect_history(
     Path("history.db"),

docs/index.md

@@ -13,6 +13,7 @@ mt5cli is a CLI application that exports MetaTrader 5 trading data to multiple f
 - **Comprehensive data access**: Rates, ticks, account info, symbols, orders, positions, and trading history
 - **Flexible timeframes**: Named timeframes (M1, H1, D1, etc.) and numeric values
 - **Connection management**: Optional credentials, server, and timeout configuration
+- **SQLite rate loading**: Load mt5cli-managed rate tables/views for offline workflows
 
 ## Installation
 
@@ -22,13 +23,23 @@ pip install mt5cli
 
 ## Programmatic usage / SDK usage
 
-mt5cli can be used as a small Python SDK for read-only MetaTrader 5 data collection. SDK functions return pandas DataFrames without writing files. Use `export_dataframe` when you need to persist results.
+mt5cli can be used as a small Python SDK for read-only MetaTrader 5 data collection. SDK functions return pandas DataFrames without writing files. Use `export_dataframe` or `export_dataframe_to_sqlite` when you need to persist results.
 
 ```python
 from datetime import UTC, datetime
 from pathlib import Path
 
-from mt5cli import Mt5CliClient, collect_history, copy_rates_range, export_dataframe
+from mt5cli import (
+    Mt5CliClient,
+    collect_history,
+    copy_rates_range,
+    export_dataframe,
+    export_dataframe_to_sqlite,
+    load_rate_data,
+    minimum_margins,
+    recent_ticks,
+)
+from mt5cli.history import resolve_rate_view_name
 
 # One-off fetch with module-level helpers
 rates = copy_rates_range(
@@ -39,10 +50,21 @@ rates = copy_rates_range(
 )
 export_dataframe(rates, Path("rates.csv"), "csv")
 
+# Resolve SQLite rate compatibility views for downstream tools
+view = resolve_rate_view_name(Path("history.db"), "EURUSD", "M1", require_existing=True)
+offline_rates = load_rate_data(Path("history.db"), view, count=1000)
+
+# Recent tick window and minimum margin summary
+ticks = recent_ticks("EURUSD", seconds=300)
+margins = minimum_margins("EURUSD")
+
 # Reuse one MT5 connection for multiple calls
 with Mt5CliClient(login=12345, password="secret", server="Broker-Demo") as client:
     account = client.account_info()
     positions = client.positions()
+    latest = client.latest_rates("EURUSD", "M1", count=100)
+    summary = client.mt5_summary()
+    summary_table = client.mt5_summary_as_df()
 
 # Bulk SQLite collection (same behavior as the collect-history CLI command)
 collect_history(
@@ -58,6 +80,8 @@ collect_history(
 
 Timeframes, tick flags, and ISO 8601 date strings are accepted wherever noted in the SDK API.
 
+`Mt5CliClient.mt5_summary()` returns the SDK structured form as plain nested Python values. Use `Mt5CliClient.mt5_summary_as_df()` when you need a one-row DataFrame for export. The `mt5-summary` CLI command uses this tabular form, so nested terminal/account fields are JSON-encoded strings that are safe for CSV, JSON, Parquet, and SQLite output.
+
 ## Quick Start
 
 ```bash
@@ -88,14 +112,16 @@ mt5cli --login 12345 --password mypass --server MyBroker-Demo \
 | ---------------- | ---------------------------------- |
 | `rates-from`     | Export rates from a start date     |
 | `rates-from-pos` | Export rates from a start position |
+| `latest-rates`   | Export latest rates                |
 | `rates-range`    | Export rates for a date range      |
 
 ### Ticks
 
-| Command       | Description                    |
+| Command        | Description                         |
-| ------------- | ------------------------------ |
+| -------------- | ----------------------------------- |
-| `ticks-from`  | Export ticks from a start date |
+| `ticks-from`   | Export ticks from a start date      |
-| `ticks-range` | Export ticks for a date range  |
+| `ticks-range`  | Export ticks for a date range       |
+| `ticks-recent` | Export ticks from a trailing window |
 
 ### Information
 
@@ -108,18 +134,21 @@ mt5cli --login 12345 --password mypass --server MyBroker-Demo \
 | `symbols`          | Export symbol list                      |
 | `symbol-info`      | Export symbol details                   |
 | `symbol-info-tick` | Export the last tick for a symbol       |
+| `minimum-margins`  | Export minimum-volume margin summary    |
 | `market-book`      | Export market depth (order book)        |
 
 ### Trading
 
-| Command          | Description                                                 |
+| Command                | Description                                                 |
-| ---------------- | ----------------------------------------------------------- |
+| ---------------------- | ----------------------------------------------------------- |
-| `orders`         | Export active orders                                        |
+| `orders`               | Export active orders                                        |
-| `positions`      | Export open positions                                       |
+| `positions`            | Export open positions                                       |
-| `history-orders` | Export historical orders                                    |
+| `history-orders`       | Export historical orders                                    |
-| `history-deals`  | Export historical deals                                     |
+| `history-deals`        | Export historical deals                                     |
-| `order-check`    | Check funds sufficiency for a trade request                 |
+| `recent-history-deals` | Export historical deals from a trailing window              |
-| `order-send`     | Send a trade request to the trade server (`--yes` required) |
+| `mt5-summary`          | Export terminal/account status summary                      |
+| `order-check`          | Check funds sufficiency for a trade request                 |
+| `order-send`           | Send a trade request to the trade server (`--yes` required) |
 
 Use `order-check` to validate a request payload before running `order-send --yes`.
 
@@ -152,6 +181,8 @@ mt5cli -o history.db collect-history \
 
 History orders and deals are fetched per symbol and concatenated, so the symbol filter is applied consistently across all datasets. The `cash_events` view is derived from symbol-filtered `history_deals`, so account-level cash events with empty or non-matching symbols may be excluded. The `positions_reconstructed` view excludes positions with no closing deal, uses volume-weighted open/close prices, and reports reversal deals (`DEAL_ENTRY_INOUT`) via `volume_reversal` / `reversal_count`.
 
+See the [History schema diagram](api/history.md#entity-relationship-diagram) for a sample ER layout of the resulting database.
+
 ## Global Options
 
 | Option         | Description                                             |

mkdocs.yml

@@ -24,6 +24,7 @@ theme:
   features:
     - content.code.annotate
     - content.code.copy
+    - content.code.mermaid
     - navigation.indexes
     - navigation.sections
     - navigation.tabs
@@ -57,6 +58,7 @@ nav:
       - Overview: api/index.md
       - CLI: api/cli.md
       - SDK: api/sdk.md
+      - History Collection (SQLite): api/history.md
       - Utils: api/utils.md
 
 markdown_extensions:

mt5cli/__init__.py

@@ -2,11 +2,28 @@
 
 from importlib.metadata import version
 
+from .history import (
+    RateTarget,
+    build_rate_targets,
+    build_rate_view_name,
+    load_rate_data,
+    load_rate_data_from_connection,
+    load_rate_series_from_sqlite,
+    resolve_history_datasets,
+    resolve_history_tick_flags,
+    resolve_history_timeframes,
+    resolve_rate_tables,
+    resolve_rate_view_name,
+    resolve_rate_view_names,
+)
 from .sdk import (
+    AccountSpec,
     Mt5CliClient,
     account_info,
     build_config,
     collect_history,
+    collect_latest_rates,
+    collect_latest_rates_for_accounts,
     copy_rates_from,
     copy_rates_from_pos,
     copy_rates_range,
@@ -15,26 +32,56 @@ from .sdk import (
     history_deals,
     history_orders,
     last_error,
+    latest_rates,
     market_book,
+    minimum_margins,
+    mt5_session,
+    mt5_summary,
+    mt5_summary_as_df,
     orders,
     positions,
+    recent_history_deals,
+    recent_ticks,
     symbol_info,
     symbol_info_tick,
     symbols,
     terminal_info,
+    update_history,
+    update_history_with_config,
 )
 from .sdk import (
     version as mt5_version,
 )
-from .utils import detect_format, export_dataframe
+from .utils import (
+    TICK_FLAG_MAP,
+    TIMEFRAME_MAP,
+    Dataset,
+    IfExists,
+    detect_format,
+    export_dataframe,
+    export_dataframe_to_sqlite,
+    parse_datetime,
+    parse_tick_flags,
+    parse_timeframe,
+)
 
 __version__ = version(__package__) if __package__ else None
 
 __all__ = [
+    "TICK_FLAG_MAP",
+    "TIMEFRAME_MAP",
+    "AccountSpec",
+    "Dataset",
+    "IfExists",
     "Mt5CliClient",
+    "RateTarget",
     "account_info",
     "build_config",
+    "build_rate_targets",
+    "build_rate_view_name",
     "collect_history",
+    "collect_latest_rates",
+    "collect_latest_rates_for_accounts",
     "copy_rates_from",
     "copy_rates_from_pos",
     "copy_rates_range",
@@ -42,15 +89,37 @@ __all__ = [
     "copy_ticks_range",
     "detect_format",
     "export_dataframe",
+    "export_dataframe_to_sqlite",
     "history_deals",
     "history_orders",
     "last_error",
+    "latest_rates",
+    "load_rate_data",
+    "load_rate_data_from_connection",
+    "load_rate_series_from_sqlite",
     "market_book",
+    "minimum_margins",
+    "mt5_session",
+    "mt5_summary",
+    "mt5_summary_as_df",
     "mt5_version",
     "orders",
+    "parse_datetime",
+    "parse_tick_flags",
+    "parse_timeframe",
     "positions",
+    "recent_history_deals",
+    "recent_ticks",
+    "resolve_history_datasets",
+    "resolve_history_tick_flags",
+    "resolve_history_timeframes",
+    "resolve_rate_tables",
+    "resolve_rate_view_name",
+    "resolve_rate_view_names",
     "symbol_info",
     "symbol_info_tick",
     "symbols",
     "terminal_info",
+    "update_history",
+    "update_history_with_config",
 ]

mt5cli/cli.py

@@ -222,6 +222,31 @@ def rates_from_pos(
     )
 
 
+@app.command()
+def latest_rates(
+    ctx: typer.Context,
+    symbol: Annotated[str, typer.Option(help="Symbol name.")],
+    timeframe: Annotated[
+        int,
+        typer.Option(
+            click_type=TIMEFRAME_TYPE,
+            help="Timeframe.",
+        ),
+    ],
+    count: Annotated[int, typer.Option(help="Number of records.")],
+    start_pos: Annotated[
+        int,
+        typer.Option(help="Start position (0 = current bar)."),
+    ] = 0,
+) -> None:
+    """Export latest rates from a start position."""
+    client = _sdk_client(ctx)
+    _execute_export(
+        ctx,
+        lambda: client.latest_rates(symbol, timeframe, count, start_pos=start_pos),
+    )
+
+
 @app.command()
 def rates_range(
     ctx: typer.Context,
@@ -300,6 +325,44 @@ def ticks_range(
     )
 
 
+@app.command()
+def ticks_recent(
+    ctx: typer.Context,
+    symbol: Annotated[str, typer.Option(help="Symbol name.")],
+    seconds: Annotated[
+        float,
+        typer.Option(help="Lookback window in seconds."),
+    ],
+    date_to: Annotated[
+        datetime | None,
+        typer.Option(click_type=DATETIME_TYPE, help="Window end date."),
+    ] = None,
+    count: Annotated[
+        int,
+        typer.Option(help="Maximum number of ticks to return."),
+    ] = 10000,
+    flags: Annotated[
+        int,
+        typer.Option(
+            click_type=TICK_FLAGS_TYPE,
+            help="Tick flags (ALL, INFO, TRADE, or integer).",
+        ),
+    ] = 1,
+) -> None:
+    """Export ticks from a recent time window."""
+    client = _sdk_client(ctx)
+    _execute_export(
+        ctx,
+        lambda: client.recent_ticks(
+            symbol,
+            seconds,
+            date_to=date_to,
+            count=count,
+            flags=flags,
+        ),
+    )
+
+
 @app.command()
 def account_info(ctx: typer.Context) -> None:
     """Export account information."""
@@ -335,6 +398,16 @@ def symbol_info(
     _execute_export(ctx, lambda: client.symbol_info(symbol))
 
 
+@app.command()
+def minimum_margins(
+    ctx: typer.Context,
+    symbol: Annotated[str, typer.Option(help="Symbol name.")],
+) -> None:
+    """Export minimum-volume buy and sell margin requirements."""
+    client = _sdk_client(ctx)
+    _execute_export(ctx, lambda: client.minimum_margins(symbol))
+
+
 @app.command()
 def orders(
     ctx: typer.Context,
@@ -427,6 +500,37 @@ def history_deals(
     )
 
 
+@app.command()
+def recent_history_deals(
+    ctx: typer.Context,
+    hours: Annotated[float, typer.Option(help="Lookback window in hours.")],
+    date_to: Annotated[
+        datetime | None,
+        typer.Option(click_type=DATETIME_TYPE, help="Window end date."),
+    ] = None,
+    group: Annotated[str | None, typer.Option(help="Group filter.")] = None,
+    symbol: Annotated[str | None, typer.Option(help="Symbol filter.")] = None,
+) -> None:
+    """Export historical deals from a recent trailing window."""
+    client = _sdk_client(ctx)
+    _execute_export(
+        ctx,
+        lambda: client.recent_history_deals(
+            hours,
+            date_to=date_to,
+            group=group,
+            symbol=symbol,
+        ),
+    )
+
+
+@app.command()
+def mt5_summary(ctx: typer.Context) -> None:
+    """Export a compact terminal/account status summary."""
+    client = _sdk_client(ctx)
+    _execute_export(ctx, client.mt5_summary_as_df)
+
+
 @app.command()
 def version(ctx: typer.Context) -> None:
     """Export MetaTrader5 version information."""

mt5cli/utils.py

@@ -2,16 +2,18 @@
 
 from __future__ import annotations
 
-import importlib
 import json
+import sqlite3
 from datetime import UTC, datetime
 from enum import StrEnum
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, TypeGuard, cast
+from typing import TYPE_CHECKING, Any, TypeGuard
 
 import click
 
 if TYPE_CHECKING:
+    from collections.abc import Sequence
+
     import pandas as pd
 
 # ---------------------------------------------------------------------------
@@ -260,6 +262,50 @@ def detect_format(
     raise ValueError(msg)
 
 
+def export_dataframe_to_sqlite(
+    df: pd.DataFrame,
+    output_path: Path,
+    table_name: str = "data",
+    *,
+    if_exists: IfExists = IfExists.APPEND,
+    index: bool = False,
+    index_label: str | None = None,
+    deduplicate_on: Sequence[str] | None = None,
+) -> None:
+    """Write a DataFrame to SQLite with configurable append and deduplication.
+
+    Args:
+        df: DataFrame to export.
+        output_path: SQLite database path.
+        table_name: Target table name.
+        if_exists: Conflict behavior when the table already exists.
+        index: Whether to write the DataFrame index as a column.
+        index_label: Column name for the index when ``index=True``.
+        deduplicate_on: Optional key columns to deduplicate after writing,
+            keeping the latest ``ROWID`` per key group. Deduplication scans the
+            full table, so repeated appends cost O(table size); index the key
+            columns when appending frequently.
+    """
+    with sqlite3.connect(output_path) as conn:
+        df.to_sql(  # type: ignore[reportUnknownMemberType]
+            table_name,
+            conn,
+            if_exists=if_exists.value,
+            index=index,
+            index_label=index_label,
+        )
+        if deduplicate_on:
+            from .history import drop_duplicates_in_table  # noqa: PLC0415
+
+            drop_duplicates_in_table(
+                conn.cursor(),
+                table_name,
+                list(deduplicate_on),
+                keep="last",
+            )
+            conn.commit()
+
+
 def export_dataframe(
     df: pd.DataFrame,
     output_path: Path,
@@ -289,14 +335,13 @@ def export_dataframe(
     elif output_format == "parquet":
         df.to_parquet(output_path, index=False)
     elif output_format == "sqlite3":
-        sqlite3 = cast("Any", importlib.import_module("sqlite3"))
+        export_dataframe_to_sqlite(
-        with sqlite3.connect(output_path) as conn:
+            df,
-            df.to_sql(  # type: ignore[reportUnknownMemberType]
+            output_path,
-                table_name,
+            table_name,
-                conn,
+            if_exists=IfExists.REPLACE,
-                if_exists="replace",
+            index=False,
-                index=False,
+        )
-            )
     else:
         msg = f"Unsupported output format: {output_format}"
         raise ValueError(msg)

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "mt5cli"
-version = "0.4.0"
+version = "0.5.2"
 description = "Command-line tool for MetaTrader 5"
 authors = [{name = "dceoy", email = "dceoy@users.noreply.github.com"}]
 maintainers = [{name = "dceoy", email = "dceoy@users.noreply.github.com"}]
@@ -124,6 +124,7 @@ ignore = [
 ]
 
 [tool.ruff.lint.per-file-ignores]
+"mt5cli/history.py" = ["TC003"]
 "tests/**/*.py" = [
   "DOC201",   # Missing return documentation
   "DOC501",   # Raised exception missing from docstring

tests/test_cli.py

@@ -6,7 +6,7 @@ import json
 import logging
 import re
 import sqlite3
-from datetime import UTC, datetime
+from datetime import UTC, datetime, timedelta
 from typing import TYPE_CHECKING
 from unittest.mock import MagicMock
 
@@ -93,6 +93,10 @@ def mock_client(mocker: MockerFixture) -> MagicMock:
     client.market_book_get_as_df.return_value = sample_df
     client.order_check_as_df.return_value = sample_df
     client.order_send_as_df.return_value = sample_df
+    client.version.return_value = (5, 0, 1)
+    client.terminal_info.return_value = {"connected": True, "paths": ["terminal.exe"]}
+    client.account_info.return_value = {"login": 123, "limits": {"modes": ["demo"]}}
+    client.symbols_total.return_value = 42
     mocker.patch("mt5cli.sdk.Mt5DataClient", return_value=client)
     return client
 
@@ -223,6 +227,37 @@ class TestCommands:
             count=50,
         )
 
+    def test_latest_rates(
+        self,
+        tmp_path: Path,
+        mock_client: MagicMock,
+    ) -> None:
+        """Test latest-rates command."""
+        output = tmp_path / "out.csv"
+        result = runner.invoke(
+            app,
+            [
+                "-o",
+                str(output),
+                "latest-rates",
+                "--symbol",
+                "GBPUSD",
+                "--timeframe",
+                "H1",
+                "--count",
+                "50",
+                "--start-pos",
+                "2",
+            ],
+        )
+        assert result.exit_code == 0, result.output
+        mock_client.copy_rates_from_pos_as_df.assert_called_once_with(
+            symbol="GBPUSD",
+            timeframe=16385,
+            start_pos=2,
+            count=50,
+        )
+
     def test_rates_range(
         self,
         tmp_path: Path,
@@ -316,6 +351,65 @@ class TestCommands:
             flags=2,
         )
 
+    def test_ticks_recent(
+        self,
+        tmp_path: Path,
+        mock_client: MagicMock,
+    ) -> None:
+        """Test ticks-recent command."""
+        output = tmp_path / "out.csv"
+        result = runner.invoke(
+            app,
+            [
+                "-o",
+                str(output),
+                "ticks-recent",
+                "--symbol",
+                "EURUSD",
+                "--seconds",
+                "120",
+                "--date-to",
+                "2024-01-02",
+                "--count",
+                "500",
+                "--flags",
+                "ALL",
+            ],
+        )
+        assert result.exit_code == 0, result.output
+        mock_client.copy_ticks_from_as_df.assert_called_once_with(
+            symbol="EURUSD",
+            date_from=datetime(2024, 1, 2, tzinfo=UTC) - timedelta(seconds=120),
+            count=500,
+            flags=1,
+        )
+        mock_client.copy_ticks_range_as_df.assert_not_called()
+
+    def test_minimum_margins(
+        self,
+        tmp_path: Path,
+        mock_client: MagicMock,
+    ) -> None:
+        """Test minimum-margins command."""
+        sym = MagicMock(volume_min=0.01)
+        account = MagicMock(currency="USD")
+        tick = MagicMock(ask=1.1010, bid=1.1000)
+        mock_client.symbol_info.return_value = sym
+        mock_client.account_info.return_value = account
+        mock_client.symbol_info_tick.return_value = tick
+        mock_client.order_calc_margin.side_effect = [12.5, 12.4]
+        mock_client.mt5.ORDER_TYPE_BUY = 0
+        mock_client.mt5.ORDER_TYPE_SELL = 1
+        output = tmp_path / "out.csv"
+        result = runner.invoke(
+            app,
+            ["-o", str(output), "minimum-margins", "--symbol", "EURUSD"],
+        )
+        assert result.exit_code == 0, result.output
+        mock_client.symbol_info.assert_called_once_with("EURUSD")
+        mock_client.order_calc_margin.assert_any_call(0, "EURUSD", 0.01, 1.1010)
+        mock_client.order_calc_margin.assert_any_call(1, "EURUSD", 0.01, 1.1000)
+
     def test_orders(
         self,
         tmp_path: Path,
@@ -392,6 +486,84 @@ class TestCommands:
         assert result.exit_code == 0, result.output
         mock_client.history_deals_get_as_df.assert_called_once()
 
+    def test_recent_history_deals(
+        self,
+        tmp_path: Path,
+        mock_client: MagicMock,
+    ) -> None:
+        """Test recent-history-deals command."""
+        output = tmp_path / "out.csv"
+        result = runner.invoke(
+            app,
+            [
+                "-o",
+                str(output),
+                "recent-history-deals",
+                "--hours",
+                "6",
+                "--date-to",
+                "2024-01-02",
+                "--symbol",
+                "EURUSD",
+            ],
+        )
+        assert result.exit_code == 0, result.output
+        mock_client.history_deals_get_as_df.assert_called_once_with(
+            date_from=datetime(2024, 1, 1, 18, tzinfo=UTC),
+            date_to=datetime(2024, 1, 2, tzinfo=UTC),
+            group=None,
+            symbol="EURUSD",
+            ticket=None,
+            position=None,
+        )
+
+    @pytest.mark.parametrize(
+        ("filename", "reader"),
+        [
+            ("summary.csv", "csv"),
+            ("summary.json", "json"),
+            ("summary.db", "sqlite3"),
+            ("summary.parquet", "parquet"),
+        ],
+    )
+    def test_mt5_summary_export_formats(
+        self,
+        tmp_path: Path,
+        mock_client: MagicMock,
+        filename: str,
+        reader: str,
+    ) -> None:
+        """Test mt5-summary writes export-safe files for supported formats."""
+        output = tmp_path / filename
+        result = runner.invoke(app, ["-o", str(output), "mt5-summary"])
+        assert result.exit_code == 0, result.output
+        assert output.exists()
+        mock_client.version.assert_called_once()
+        mock_client.terminal_info.assert_called_once()
+        mock_client.account_info.assert_called_once()
+        mock_client.symbols_total.assert_called_once()
+        if reader == "csv":
+            frame = pd.read_csv(output)
+        elif reader == "json":
+            with output.open() as f:
+                records = json.load(f)
+            frame = pd.DataFrame(records)
+        elif reader == "sqlite3":
+            with sqlite3.connect(output) as conn:
+                frame = pd.read_sql(  # type: ignore[reportUnknownMemberType]
+                    "SELECT * FROM data",
+                    conn,
+                )
+        else:
+            frame = pd.read_parquet(output)
+        assert len(frame) == 1
+        assert frame.iloc[0].to_dict() == {
+            "version": "[5,0,1]",
+            "terminal_info": '{"connected":true,"paths":["terminal.exe"]}',
+            "account_info": '{"limits":{"modes":["demo"]},"login":123}',
+            "symbols_total": 42,
+        }
+
     def test_version(
         self,
         tmp_path: Path,
@@ -1089,7 +1261,7 @@ class TestCollectHistory:
         assert all(row[0] not in {0, 1} for row in cash)
         # Position 100 (BUY 1@1.10 + BUY 3@1.20 then SELL 4@1.50) is closed.
         # Position 200 (BUY 2@2.00 then SELL 2@2.20) is closed.
-        # Position 300 (open-only) and 400 (reversal-only) are excluded.
+        # Position 400 (reversal-only with non-trade deal type) stays excluded.
         assert set(positions) == {100, 200, 500, 600}
         pos_100 = positions[100]
         tol = 1e-9
@@ -1106,10 +1278,10 @@ class TestCollectHistory:
         assert abs(pos_500[5] - 1.05) < tol
         pos_600 = positions[600]
         assert abs(pos_600[1] - 3.0) < tol
-        assert abs(pos_600[2] - 3.0) < tol
+        assert abs(pos_600[2] - 4.0) < tol  # reversal + close volumes
         assert abs(pos_600[3] - 1.0) < tol
         assert abs(pos_600[4] - 1.10) < tol
-        assert abs(pos_600[5] - 1.40) < tol
+        assert abs(pos_600[5] - 3.5475) < tol
         assert pos_600[6] == 1
 
     def test_collect_history_filters_history_symbols_exactly(

tests/test_utils.py

@@ -21,8 +21,10 @@ from mt5cli.utils import (
     TIMEFRAME_MAP,
     TIMEFRAME_TYPE,
     Dataset,
+    IfExists,
     detect_format,
     export_dataframe,
+    export_dataframe_to_sqlite,
     parse_datetime,
     parse_request,
     parse_tick_flags,
@@ -130,6 +132,112 @@ class TestExportDataframe:
             export_dataframe(sample_df, tmp_path / "out.txt", "xml")
 
 
+class TestExportDataframeToSqlite:
+    """Tests for export_dataframe_to_sqlite."""
+
+    def test_append_preserves_existing_rows(self, tmp_path: Path) -> None:
+        """Test append mode keeps prior rows in the SQLite table."""
+        output = tmp_path / "append.db"
+        first = pd.DataFrame({"id": [1], "value": ["a"]})
+        second = pd.DataFrame({"id": [2], "value": ["b"]})
+        export_dataframe_to_sqlite(first, output, "items", if_exists=IfExists.REPLACE)
+        export_dataframe_to_sqlite(second, output, "items", if_exists=IfExists.APPEND)
+        with sqlite3.connect(output) as conn:
+            result = pd.read_sql(  # type: ignore[reportUnknownMemberType]
+                "SELECT id, value FROM items ORDER BY id",
+                conn,
+            )
+        pd.testing.assert_frame_equal(
+            result,
+            pd.DataFrame({"id": [1, 2], "value": ["a", "b"]}),
+        )
+
+    def test_deduplicate_keeps_latest_row(self, tmp_path: Path) -> None:
+        """Test deduplication keeps the latest ROWID for key columns."""
+        output = tmp_path / "dedup.db"
+        first = pd.DataFrame({
+            "symbol": ["EURUSD", "EURUSD"],
+            "time": ["2024-01-01", "2024-01-01"],
+            "bid": [1.0, 1.1],
+        })
+        second = pd.DataFrame({
+            "symbol": ["EURUSD"],
+            "time": ["2024-01-01"],
+            "bid": [1.2],
+        })
+        export_dataframe_to_sqlite(
+            first,
+            output,
+            "ticks",
+            if_exists=IfExists.REPLACE,
+            deduplicate_on=("symbol", "time"),
+        )
+        export_dataframe_to_sqlite(
+            second,
+            output,
+            "ticks",
+            if_exists=IfExists.APPEND,
+            deduplicate_on=("symbol", "time"),
+        )
+        with sqlite3.connect(output) as conn:
+            result = pd.read_sql(  # type: ignore[reportUnknownMemberType]
+                "SELECT symbol, time, bid FROM ticks",
+                conn,
+            )
+        pd.testing.assert_frame_equal(
+            result.reset_index(drop=True),
+            pd.DataFrame({
+                "symbol": ["EURUSD"],
+                "time": ["2024-01-01"],
+                "bid": [1.2],
+            }),
+        )
+
+    def test_default_if_exists_appends_without_dropping_rows(
+        self,
+        tmp_path: Path,
+    ) -> None:
+        """Test the default append mode keeps prior rows."""
+        output = tmp_path / "default-append.db"
+        first = pd.DataFrame({"id": [1], "value": ["a"]})
+        second = pd.DataFrame({"id": [2], "value": ["b"]})
+        export_dataframe_to_sqlite(first, output, "items")
+        export_dataframe_to_sqlite(second, output, "items")
+        with sqlite3.connect(output) as conn:
+            result = pd.read_sql(  # type: ignore[reportUnknownMemberType]
+                "SELECT id, value FROM items ORDER BY id",
+                conn,
+            )
+        pd.testing.assert_frame_equal(
+            result,
+            pd.DataFrame({"id": [1, 2], "value": ["a", "b"]}),
+        )
+
+    def test_writes_index_with_label(self, tmp_path: Path) -> None:
+        """Test optional index export with a custom label."""
+        output = tmp_path / "index.db"
+        frame = pd.DataFrame(
+            {"value": [1.0]}, index=pd.Index(["EURUSD"], name="symbol")
+        )
+        export_dataframe_to_sqlite(
+            frame,
+            output,
+            "margins",
+            if_exists=IfExists.REPLACE,
+            index=True,
+            index_label="symbol",
+        )
+        with sqlite3.connect(output) as conn:
+            result = pd.read_sql(  # type: ignore[reportUnknownMemberType]
+                "SELECT symbol, value FROM margins",
+                conn,
+            )
+        pd.testing.assert_frame_equal(
+            result,
+            pd.DataFrame({"symbol": ["EURUSD"], "value": [1.0]}),
+        )
+
+
 # ---------------------------------------------------------------------------
 # Parse helpers
 # ---------------------------------------------------------------------------

uv.lock

@@ -487,7 +487,7 @@ wheels = [
 
 [[package]]
 name = "mt5cli"
-version = "0.4.0"
+version = "0.5.2"
 source = { editable = "." }
 dependencies = [
     { name = "click" },