# Storage - mt5cli API Documentation
Author: dceoy

# Storage

## mt5cli.storage

Generic storage helpers for MT5 market and account history.

### __all__ `module-attribute`

    __all__ = [
        "Dataset",
        "IfExists",
        "OutputFormat",
        "RateTarget",
        "build_rate_targets",
        "build_rate_view_name",
        "collect_history",
        "detect_format",
        "drop_forming_rate_bar",
        "export_dataframe",
        "export_dataframe_to_sqlite",
        "load_rate_data",
        "load_rate_data_from_connection",
        "load_rate_series_by_granularity",
        "load_rate_series_from_sqlite",
        "resolve_rate_tables",
        "resolve_rate_view_name",
        "resolve_rate_view_names",
        "update_history",
        "update_history_with_config",
    ]

### Dataset

Bases: `StrEnum`

Datasets supported by the `collect-history` command.

#### history_deals `class-attribute` `instance-attribute`

    history_deals = 'history-deals'

#### history_orders `class-attribute` `instance-attribute`

    history_orders = 'history-orders'

#### rates `class-attribute` `instance-attribute`

    rates = 'rates'

#### table_name `property`

    table_name: str

Return the SQLite table name for this dataset.

#### ticks `class-attribute` `instance-attribute`

    ticks = 'ticks'

### IfExists

Bases: `StrEnum`

SQLite table conflict behavior for the `collect-history` command.

#### APPEND `class-attribute` `instance-attribute`

    APPEND = 'append'

#### FAIL `class-attribute` `instance-attribute`

    FAIL = 'fail'

#### REPLACE `class-attribute` `instance-attribute`

    REPLACE = 'replace'

### OutputFormat

Bases: `StrEnum`

Supported output file formats.

#### csv `class-attribute` `instance-attribute`

    csv = 'csv'

#### json `class-attribute` `instance-attribute`

    json = 'json'

#### parquet `class-attribute` `instance-attribute`

    parquet = 'parquet'

#### sqlite3 `class-attribute` `instance-attribute`

    sqlite3 = 'sqlite3'

### RateTarget `dataclass`

    RateTarget(symbol: str | None, timeframe: int | str)

A single rate series identified by symbol and timeframe.

Attributes:

Name | Type | Description
--- | --- | ---
`symbol` | `str | None` | MT5 symbol name, or None when the rate series is addressed only by an explicit table (for example a custom SQLite view).
`timeframe` | `int | str` | MT5 timeframe as an integer or name (for example `M1`).

#### symbol `instance-attribute`

    symbol: str | None

#### timeframe `instance-attribute`

    timeframe: int | str

#### timeframe_int `property`

    timeframe_int: int

Return the timeframe as its integer MT5 value.

#### __post_init__

    __post_init__() -> None

Normalize accepted timeframe aliases to the stored integer value.

Source code in `mt5cli/history.py`

    524
    525
    526
    527

    def __post_init__(self) -> None:
        """Normalize accepted timeframe aliases to the stored integer value."""
        if not isinstance(self.timeframe, int):
            object.__setattr__(self, "timeframe", parse_timeframe(self.timeframe))

### build_rate_targets

    build_rate_targets(
        symbols: Sequence[str],
        timeframes: Sequence[int | str],
        *,
        allow_missing_symbol: bool = False,
    ) -> list[RateTarget]

Build rate targets for every symbol and timeframe combination.

Parameters:

Name | Type | Description | Default
--- | --- | --- | ---
`symbols` | `Sequence[str]` | MT5 symbol names. May be empty when `allow_missing_symbol`. | required
`timeframes` | `Sequence[int | str]` | MT5 timeframes as integers or names (for example `M1`). | required
`allow_missing_symbol` | `bool` | When True and `symbols` is empty, build targets with `symbol=None` for each timeframe instead of raising. | `False`

Returns:

Type | Description
--- | ---
`list[RateTarget]` | Targets in row-major order: every timeframe for the first symbol, then
`list[RateTarget]` | every timeframe for the next symbol, and so on.

Raises:

Type | Description
--- | ---
`ValueError` | If `timeframes` is empty, or `symbols` is empty and `allow_missing_symbol` is False.

Source code in `mt5cli/history.py`

    535
    536
    537
    538
    539
    540
    541
    542
    543
    544
    545
    546
    547
    548
    549
    550
    551
    552
    553
    554
    555
    556
    557
    558
    559
    560
    561
    562
    563
    564
    565
    566
    567
    568
    569

    def build_rate_targets(
        symbols: Sequence[str],
        timeframes: Sequence[int | str],
        *,
        allow_missing_symbol: bool = False,
    ) -> list[RateTarget]:
        """Build rate targets for every symbol and timeframe combination.

        Args:
            symbols: MT5 symbol names. May be empty when ``allow_missing_symbol``.
            timeframes: MT5 timeframes as integers or names (for example ``M1``).
            allow_missing_symbol: When True and ``symbols`` is empty, build targets
                with ``symbol=None`` for each timeframe instead of raising.

        Returns:
            Targets in row-major order: every timeframe for the first symbol, then
            every timeframe for the next symbol, and so on.

        Raises:
            ValueError: If ``timeframes`` is empty, or ``symbols`` is empty and
                ``allow_missing_symbol`` is False.
        """
        if not timeframes:
            msg = "At least one timeframe is required."
            raise ValueError(msg)
        if not symbols:
            if not allow_missing_symbol:
                msg = "At least one symbol is required."
                raise ValueError(msg)
            return [RateTarget(symbol=None, timeframe=tf) for tf in timeframes]
        return [
            RateTarget(symbol=symbol, timeframe=tf)
            for symbol in symbols
            for tf in timeframes
        ]

### build_rate_view_name

    build_rate_view_name(
        *,
        symbol: str,
        granularity: str,
        granularity_count: int,
        timeframe: int,
    ) -> str

Return a collision-free offline optimize view name.

View names always include the timeframe integer after a `__` separator so a symbol such as `EURUSD_M1` cannot collide with `EURUSD` at timeframe `M1`.

Source code in `mt5cli/history.py`

    127
    128
    129
    130
    131
    132
    133
    134
    135
    136
    137
    138
    139
    140
    141
    142

    def build_rate_view_name(
        *,
        symbol: str,
        granularity: str,
        granularity_count: int,
        timeframe: int,
    ) -> str:
        """Return a collision-free offline optimize view name.

        View names always include the timeframe integer after a ``__`` separator so
        a symbol such as ``EURUSD_M1`` cannot collide with ``EURUSD`` at timeframe
        ``M1``.
        """
        if granularity_count == 1:
            return f"rate_{symbol}__{timeframe}"
        return f"rate_{symbol}__{granularity}_{timeframe}"

### collect_history

    collect_history(
        output: Path,
        symbols: list[str],
        date_from: datetime | str,
        date_to: datetime | str,
        *,
        datasets: set[Dataset] | None = None,
        timeframe: int | str = 1,
        flags: int | str = "ALL",
        if_exists: IfExists = FAIL,
        with_views: bool = False,
        config: Mt5Config | None = None,
    ) -> None

Collect historical datasets into a single SQLite database.

Parameters:

Name | Type | Description | Default
--- | --- | --- | ---
`output` | `Path` | SQLite database path. | required
`symbols` | `list[str]` | Symbols to collect. | required
`date_from` | `datetime | str` | Start date. | required
`date_to` | `datetime | str` | End date. | required
`datasets` | `set[Dataset] | None` | Datasets to include (defaults to all). | `None`
`timeframe` | `int | str` | Rates timeframe as integer or name (e.g. `M1`). | `1`
`flags` | `int | str` | Tick copy flags as integer or name (e.g. `ALL`). | `'ALL'`
`if_exists` | `IfExists` | Behavior when a target table already exists. | `FAIL`
`with_views` | `bool` | Create `cash_events` and `positions_reconstructed` views. | `False`
`config` | `Mt5Config | None` | MT5 connection configuration. | `None`

Source code in `mt5cli/sdk.py`

    1171
    1172
    1173
    1174
    1175
    1176
    1177
    1178
    1179
    1180
    1181
    1182
    1183
    1184
    1185
    1186
    1187
    1188
    1189
    1190
    1191
    1192
    1193
    1194
    1195
    1196
    1197
    1198
    1199
    1200
    1201
    1202
    1203
    1204
    1205
    1206
    1207
    1208
    1209
    1210
    1211
    1212
    1213
    1214
    1215
    1216
    1217
    1218
    1219
    1220
    1221
    1222
    1223
    1224
    1225
    1226
    1227
    1228
    1229
    1230
    1231
    1232
    1233
    1234

    def collect_history(
        output: Path,
        symbols: list[str],
        date_from: datetime | str,
        date_to: datetime | str,
        *,
        datasets: set[Dataset] | None = None,
        timeframe: int | str = 1,
        flags: int | str = "ALL",
        if_exists: IfExists = IfExists.FAIL,
        with_views: bool = False,
        config: Mt5Config | None = None,
    ) -> None:
        """Collect historical datasets into a single SQLite database.

        Args:
            output: SQLite database path.
            symbols: Symbols to collect.
            date_from: Start date.
            date_to: End date.
            datasets: Datasets to include (defaults to all).
            timeframe: Rates timeframe as integer or name (e.g. ``M1``).
            flags: Tick copy flags as integer or name (e.g. ``ALL``).
            if_exists: Behavior when a target table already exists.
            with_views: Create ``cash_events`` and ``positions_reconstructed`` views.
            config: MT5 connection configuration.
        """
        start = _require_datetime(date_from)
        end = _require_datetime(date_to)
        selected = datasets if datasets is not None else set(Dataset)
        tf = _coerce_timeframe(timeframe)
        tick_flags = _coerce_tick_flags(flags)
        mt5_config = config or build_config()
        with connected_client(mt5_config) as client, sqlite3.connect(output) as conn:
            conn.execute("PRAGMA journal_mode=WAL")
            conn.execute("PRAGMA synchronous=NORMAL")
            written_tables, written_columns = write_collected_datasets(
                conn,
                client,
                symbols,
                selected,
                tf,
                tick_flags,
                start,
                end,
                if_exists,
            )
            create_history_indexes(conn, written_columns)
            if with_views and Dataset.history_deals in written_tables:
                create_cash_events_view(conn, written_columns[Dataset.history_deals])
                create_positions_reconstructed_view(
                    conn,
                    written_columns[Dataset.history_deals],
                )
            elif with_views:
                logger.warning(
                    "--with-views ignored: history_deals table was not written",
                )
        logger.info(
            "Collected %s for %d symbol(s) into %s",
            ", ".join(sorted(ds.value for ds in selected)),
            len(symbols),
            output,
        )

### detect_format

    detect_format(
        output_path: Path, explicit_format: str | None = None
    ) -> str

Detect the output format from a file extension or explicit format string.

Parameters:

Name | Type | Description | Default
--- | --- | --- | ---
`output_path` | `Path` | Path to the output file. | required
`explicit_format` | `str | None` | Explicitly specified format, if any. | `None`

Returns:

Type | Description
--- | ---
`str` | The detected format string.

Raises:

Type | Description
--- | ---
`ValueError` | If the format cannot be determined.

Source code in `mt5cli/utils.py`

    216
    217
    218
    219
    220
    221
    222
    223
    224
    225
    226
    227
    228
    229
    230
    231
    232
    233
    234
    235
    236
    237
    238
    239
    240
    241

    def detect_format(
        output_path: Path,
        explicit_format: str | None = None,
    ) -> str:
        """Detect the output format from a file extension or explicit format string.

        Args:
            output_path: Path to the output file.
            explicit_format: Explicitly specified format, if any.

        Returns:
            The detected format string.

        Raises:
            ValueError: If the format cannot be determined.
        """
        if explicit_format is not None:
            return explicit_format
        suffix = output_path.suffix.lower()
        if suffix in _FORMAT_EXTENSIONS:
            return _FORMAT_EXTENSIONS[suffix]
        msg = (
            f"Cannot detect format from extension '{suffix}'."
            " Use --format to specify the output format."
        )
        raise ValueError(msg)

### drop_forming_rate_bar

    drop_forming_rate_bar(df_rate: DataFrame) -> DataFrame

Return closed bars from chronologically ordered MT5 rate data.

MetaTrader 5 `copy_rates_from_pos(start_pos=0)` includes the still-forming current bar as the last row. Slice it off so downstream logic only sees completed bars. Empty frames and single-row frames return empty results.

Parameters:

Name | Type | Description | Default
--- | --- | --- | ---
`df_rate` | `DataFrame` | Rate data ordered oldest-to-newest with the forming bar last. | required

Returns:

Type | Description
--- | ---
`DataFrame` | A new DataFrame with all rows except the last. Index and columns are
`DataFrame` | preserved. The input frame is not modified.

Source code in `mt5cli/history.py`

    110
    111
    112
    113
    114
    115
    116
    117
    118
    119
    120
    121
    122
    123
    124

    def drop_forming_rate_bar(df_rate: pd.DataFrame) -> pd.DataFrame:
        """Return closed bars from chronologically ordered MT5 rate data.

        MetaTrader 5 ``copy_rates_from_pos(start_pos=0)`` includes the still-forming
        current bar as the last row. Slice it off so downstream logic only sees
        completed bars. Empty frames and single-row frames return empty results.

        Args:
            df_rate: Rate data ordered oldest-to-newest with the forming bar last.

        Returns:
            A new DataFrame with all rows except the last. Index and columns are
            preserved. The input frame is not modified.
        """
        return df_rate.iloc[:-1].copy()

### export_dataframe

    export_dataframe(
        df: DataFrame,
        output_path: Path,
        output_format: str,
        table_name: str = "data",
    ) -> None

Export a pandas DataFrame to the specified file format.

Parameters:

Name | Type | Description | Default
--- | --- | --- | ---
`df` | `DataFrame` | DataFrame to export. | required
`output_path` | `Path` | Path to the output file. | required
`output_format` | `str` | Output format (csv, json, parquet, or sqlite3). | required
`table_name` | `str` | Table name for SQLite3 output. | `'data'`

Raises:

Type | Description
--- | ---
`ValueError` | If the output format is not supported.

Source code in `mt5cli/utils.py`

    288
    289
    290
    291
    292
    293
    294
    295
    296
    297
    298
    299
    300
    301
    302
    303
    304
    305
    306
    307
    308
    309
    310
    311
    312
    313
    314
    315
    316
    317
    318
    319
    320
    321
    322
    323
    324
    325
    326

    def export_dataframe(
        df: pd.DataFrame,
        output_path: Path,
        output_format: str,
        table_name: str = "data",
    ) -> None:
        """Export a pandas DataFrame to the specified file format.

        Args:
            df: DataFrame to export.
            output_path: Path to the output file.
            output_format: Output format (csv, json, parquet, or sqlite3).
            table_name: Table name for SQLite3 output.

        Raises:
            ValueError: If the output format is not supported.
        """
        if output_format == "csv":
            df.to_csv(output_path, index=False)
        elif output_format == "json":
            df.to_json(
                output_path,
                orient="records",
                date_format="iso",
                indent=2,
            )
        elif output_format == "parquet":
            df.to_parquet(output_path, index=False)
        elif output_format == "sqlite3":
            export_dataframe_to_sqlite(
                df,
                output_path,
                table_name,
                if_exists=IfExists.REPLACE,
                index=False,
            )
        else:
            msg = f"Unsupported output format: {output_format}"
            raise ValueError(msg)

### export_dataframe_to_sqlite

    export_dataframe_to_sqlite(
        df: DataFrame,
        output_path: Path,
        table_name: str = "data",
        *,
        if_exists: IfExists = APPEND,
        index: bool = False,
        index_label: str | None = None,
        deduplicate_on: Sequence[str] | None = None,
    ) -> None

Write a DataFrame to SQLite with configurable append and deduplication.

Parameters:

Name | Type | Description | Default
--- | --- | --- | ---
`df` | `DataFrame` | DataFrame to export. | required
`output_path` | `Path` | SQLite database path. | required
`table_name` | `str` | Target table name. | `'data'`
`if_exists` | `IfExists` | Conflict behavior when the table already exists. | `APPEND`
`index` | `bool` | Whether to write the DataFrame index as a column. | `False`
`index_label` | `str | None` | Column name for the index when `index=True`. | `None`
`deduplicate_on` | `Sequence[str] | None` | Optional key columns to deduplicate after writing, keeping the latest `ROWID` per key group. Deduplication scans the full table, so repeated appends cost O(table size); index the key columns when appending frequently. | `None`

Source code in `mt5cli/utils.py`

    244
    245
    246
    247
    248
    249
    250
    251
    252
    253
    254
    255
    256
    257
    258
    259
    260
    261
    262
    263
    264
    265
    266
    267
    268
    269
    270
    271
    272
    273
    274
    275
    276
    277
    278
    279
    280
    281
    282
    283
    284
    285

    def export_dataframe_to_sqlite(
        df: pd.DataFrame,
        output_path: Path,
        table_name: str = "data",
        *,
        if_exists: IfExists = IfExists.APPEND,
        index: bool = False,
        index_label: str | None = None,
        deduplicate_on: Sequence[str] | None = None,
    ) -> None:
        """Write a DataFrame to SQLite with configurable append and deduplication.

        Args:
            df: DataFrame to export.
            output_path: SQLite database path.
            table_name: Target table name.
            if_exists: Conflict behavior when the table already exists.
            index: Whether to write the DataFrame index as a column.
            index_label: Column name for the index when ``index=True``.
            deduplicate_on: Optional key columns to deduplicate after writing,
                keeping the latest ``ROWID`` per key group. Deduplication scans the
                full table, so repeated appends cost O(table size); index the key
                columns when appending frequently.
        """
        with sqlite3.connect(output_path) as conn:
            df.to_sql(  # type: ignore[reportUnknownMemberType]
                table_name,
                conn,
                if_exists=if_exists.value,
                index=index,
                index_label=index_label,
            )
            if deduplicate_on:
                from .history import drop_duplicates_in_table  # noqa: PLC0415

                drop_duplicates_in_table(
                    conn.cursor(),
                    table_name,
                    list(deduplicate_on),
                    keep="last",
                )
                conn.commit()

### load_rate_data

    load_rate_data(
        conn_or_path: SqliteConnOrPath,
        table: str,
        count: int | None = None,
    ) -> DataFrame

Load rate-like data from a SQLite database path or connection.

Parameters:

Name | Type | Description | Default
--- | --- | --- | ---
`conn_or_path` | `SqliteConnOrPath` | SQLite database path or open connection. | required
`table` | `str` | Source table or view name. | required
`count` | `int | None` | Optional number of most recent rows to load. | `None`

Returns:

Type | Description
--- | ---
`DataFrame` | DataFrame indexed by ascending `time`.

Source code in `mt5cli/history.py`

    280
    281
    282
    283
    284
    285
    286
    287
    288
    289
    290
    291
    292
    293
    294
    295
    296
    297
    298
    299
    300
    301

    def load_rate_data(
        conn_or_path: SqliteConnOrPath,
        table: str,
        count: int | None = None,
    ) -> pd.DataFrame:
        """Load rate-like data from a SQLite database path or connection.

        Args:
            conn_or_path: SQLite database path or open connection.
            table: Source table or view name.
            count: Optional number of most recent rows to load.

        Returns:
            DataFrame indexed by ascending ``time``.

        """
        conn, should_close = _open_existing_sqlite_database(conn_or_path)
        try:
            return load_rate_data_from_connection(conn, table, count=count)
        finally:
            if should_close:
                conn.close()

### load_rate_data_from_connection

    load_rate_data_from_connection(
        connection: Connection,
        table: str,
        count: int | None = None,
    ) -> DataFrame

Load rate-like data from a SQLite table or view.

Parameters:

Name | Type | Description | Default
--- | --- | --- | ---
`connection` | `Connection` | Open SQLite connection. | required
`table` | `str` | Source table or view name. | required
`count` | `int | None` | Optional number of most recent rows to load. | `None`

Returns:

Type | Description
--- | ---
`DataFrame` | DataFrame indexed by ascending `time`.

Raises:

Type | Description
--- | ---
`ValueError` | If inputs, schema, timestamps are invalid, or the table or view contains no rows.

Source code in `mt5cli/history.py`

    234
    235
    236
    237
    238
    239
    240
    241
    242
    243
    244
    245
    246
    247
    248
    249
    250
    251
    252
    253
    254
    255
    256
    257
    258
    259
    260
    261
    262
    263
    264
    265
    266
    267
    268
    269
    270
    271
    272
    273
    274
    275
    276
    277

    def load_rate_data_from_connection(
        connection: sqlite3.Connection,
        table: str,
        count: int | None = None,
    ) -> pd.DataFrame:
        """Load rate-like data from a SQLite table or view.

        Args:
            connection: Open SQLite connection.
            table: Source table or view name.
            count: Optional number of most recent rows to load.

        Returns:
            DataFrame indexed by ascending ``time``.

        Raises:
            ValueError: If inputs, schema, timestamps are invalid, or the table
                or view contains no rows.
        """
        table_name = _validate_rate_load_request(table, count)
        columns = get_table_columns(connection, table_name)
        _ensure_rate_columns(columns, table_name)
        quoted_table = quote_sqlite_identifier(table_name)
        if count is None:
            frame = cast(
                "pd.DataFrame",
                pd.read_sql_query(  # type: ignore[reportUnknownMemberType]
                    f"SELECT * FROM {quoted_table} ORDER BY time ASC",  # noqa: S608
                    connection,
                ),
            )
        else:
            frame = cast(
                "pd.DataFrame",
                pd.read_sql_query(  # type: ignore[reportUnknownMemberType]
                    f"SELECT * FROM {quoted_table} ORDER BY time DESC LIMIT ?",  # noqa: S608
                    connection,
                    params=(count,),
                ),
            )
        if frame.empty:
            msg = f"SQLite table or view {table_name!r} contains no rows."
            raise ValueError(msg)
        return _parse_rate_time_index(frame, table_name)

### load_rate_series_by_granularity

    load_rate_series_by_granularity(
        conn_or_path: SqliteConnOrPath,
        symbols: Sequence[str],
        granularities: Sequence[int | str],
        count: int,
        *,
        explicit_tables: Sequence[str] | None = None,
        allow_missing_symbol: bool = False,
    ) -> dict[tuple[str | None, str], DataFrame]

Load rate series keyed by symbol and string granularity name.

Builds targets with :func:`build_rate_targets` and loads them with :func:`load_rate_series_from_sqlite`, then rekeys the result by granularity name (for example `M1`) instead of the integer timeframe to reduce downstream boilerplate.

Parameters:

Name | Type | Description | Default
--- | --- | --- | ---
`conn_or_path` | `SqliteConnOrPath` | SQLite database path or open connection. | required
`symbols` | `Sequence[str]` | MT5 symbol names. May be empty when `allow_missing_symbol`. | required
`granularities` | `Sequence[int | str]` | MT5 timeframes as integers or names (for example `M1`). | required
`count` | `int` | Number of most recent rows to load per series. | required
`explicit_tables` | `Sequence[str] | None` | Optional explicit table or view names matching the built targets in row-major order. Required when symbols are omitted. | `None`
`allow_missing_symbol` | `bool` | When True and `symbols` is empty, build targets with `symbol=None` for each granularity instead of raising. | `False`

Returns:

Type | Description
--- | ---
`dict[tuple[str | None, str], DataFrame]` | Mapping keyed by `(symbol | None, granularity_name)` to each rate
`dict[tuple[str | None, str], DataFrame]` | DataFrame. Propagates `ValueError` (via :func:`build_rate_targets` and
`dict[tuple[str | None, str], DataFrame]` | func:`load_rate_series_from_sqlite`) when inputs are empty or invalid,
`dict[tuple[str | None, str], DataFrame]` | table resolution fails, or duplicate targets are present.

Source code in `mt5cli/history.py`

    727
    728
    729
    730
    731
    732
    733
    734
    735
    736
    737
    738
    739
    740
    741
    742
    743
    744
    745
    746
    747
    748
    749
    750
    751
    752
    753
    754
    755
    756
    757
    758
    759
    760
    761
    762
    763
    764
    765
    766
    767
    768
    769
    770
    771
    772
    773

    def load_rate_series_by_granularity(
        conn_or_path: SqliteConnOrPath,
        symbols: Sequence[str],
        granularities: Sequence[int | str],
        count: int,
        *,
        explicit_tables: Sequence[str] | None = None,
        allow_missing_symbol: bool = False,
    ) -> dict[tuple[str | None, str], pd.DataFrame]:
        """Load rate series keyed by symbol and string granularity name.

        Builds targets with :func:`build_rate_targets` and loads them with
        :func:`load_rate_series_from_sqlite`, then rekeys the result by granularity
        name (for example ``M1``) instead of the integer timeframe to reduce
        downstream boilerplate.

        Args:
            conn_or_path: SQLite database path or open connection.
            symbols: MT5 symbol names. May be empty when ``allow_missing_symbol``.
            granularities: MT5 timeframes as integers or names (for example ``M1``).
            count: Number of most recent rows to load per series.
            explicit_tables: Optional explicit table or view names matching the
                built targets in row-major order. Required when symbols are omitted.
            allow_missing_symbol: When True and ``symbols`` is empty, build targets
                with ``symbol=None`` for each granularity instead of raising.

        Returns:
            Mapping keyed by ``(symbol | None, granularity_name)`` to each rate
            DataFrame. Propagates ``ValueError`` (via :func:`build_rate_targets` and
            :func:`load_rate_series_from_sqlite`) when inputs are empty or invalid,
            table resolution fails, or duplicate targets are present.
        """
        targets = build_rate_targets(
            symbols,
            granularities,
            allow_missing_symbol=allow_missing_symbol,
        )
        series = load_rate_series_from_sqlite(
            conn_or_path,
            targets,
            count,
            explicit_tables=explicit_tables,
        )
        return {
            (symbol, resolve_granularity_name(timeframe)): frame
            for (symbol, timeframe), frame in series.items()
        }

### load_rate_series_from_sqlite

    load_rate_series_from_sqlite(
        conn_or_path: SqliteConnOrPath,
        targets: Sequence[RateTarget],
        count: int,
        explicit_tables: Sequence[str] | None = None,
    ) -> dict[tuple[str | None, int], DataFrame]

Load multiple rate series from a SQLite database.

Parameters:

Name | Type | Description | Default
--- | --- | --- | ---
`conn_or_path` | `SqliteConnOrPath` | SQLite database path or open connection. | required
`targets` | `Sequence[RateTarget]` | Rate targets to load. Each `(symbol, timeframe_int)` pair must be unique. | required
`count` | `int` | Number of most recent rows to load per series. | required
`explicit_tables` | `Sequence[str] | None` | Optional explicit table or view names matching targets. When omitted, managed `rate_*` compatibility views must already exist in the database. | `None`

Returns:

Type | Description
--- | ---
`dict[tuple[str | None, int], DataFrame]` | Mapping keyed by `(symbol, timeframe_int)` to each rate DataFrame.

Raises:

Type | Description
--- | ---
`ValueError` | If `count` is not positive, targets are empty, duplicate `(symbol, timeframe_int)` pairs are present, or table resolution fails.

Source code in `mt5cli/history.py`

    656
    657
    658
    659
    660
    661
    662
    663
    664
    665
    666
    667
    668
    669
    670
    671
    672
    673
    674
    675
    676
    677
    678
    679
    680
    681
    682
    683
    684
    685
    686
    687
    688
    689
    690
    691
    692
    693
    694
    695
    696
    697
    698
    699
    700
    701
    702
    703
    704
    705
    706
    707
    708
    709
    710
    711
    712
    713
    714
    715
    716
    717
    718
    719
    720
    721
    722
    723
    724

    def load_rate_series_from_sqlite(
        conn_or_path: SqliteConnOrPath,
        targets: Sequence[RateTarget],
        count: int,
        explicit_tables: Sequence[str] | None = None,
    ) -> dict[tuple[str | None, int], pd.DataFrame]:
        """Load multiple rate series from a SQLite database.

        Args:
            conn_or_path: SQLite database path or open connection.
            targets: Rate targets to load. Each ``(symbol, timeframe_int)`` pair
                must be unique.
            count: Number of most recent rows to load per series.
            explicit_tables: Optional explicit table or view names matching targets.
                When omitted, managed ``rate_*`` compatibility views must already
                exist in the database.

        Returns:
            Mapping keyed by ``(symbol, timeframe_int)`` to each rate DataFrame.

        Raises:
            ValueError: If ``count`` is not positive, targets are empty, duplicate
                ``(symbol, timeframe_int)`` pairs are present, or table resolution
                fails.
        """
        if count <= 0:
            msg = "count must be positive."
            raise ValueError(msg)
        target_list = list(targets)
        if not target_list:
            msg = "At least one rate target is required."
            raise ValueError(msg)
        if explicit_tables is None and any(target.symbol is None for target in target_list):
            msg = (
                "Cannot resolve a rate table for a target without a symbol; "
                "provide explicit_tables."
            )
            raise ValueError(msg)
        seen_keys: set[tuple[str | None, int]] = set()
        for target in target_list:
            key = (target.symbol, target.timeframe_int)
            if key in seen_keys:
                symbol_repr = repr(target.symbol)
                msg = f"Duplicate rate target: ({symbol_repr}, {target.timeframe_int})"
                raise ValueError(msg)
            seen_keys.add(key)
        tables = (
            resolve_rate_tables(None, target_list, explicit_tables)
            if explicit_tables is not None
            else None
        )
        conn, should_close = _open_existing_sqlite_database(conn_or_path)
        try:
            resolved_tables = tables or resolve_rate_tables(
                conn,
                target_list,
                require_existing=True,
            )
            return {
                (target.symbol, target.timeframe_int): load_rate_data_from_connection(
                    conn,
                    table,
                    count=count,
                )
                for target, table in zip(target_list, resolved_tables, strict=True)
            }
        finally:
            if should_close:
                conn.close()

### resolve_rate_tables

    resolve_rate_tables(
        conn_or_path: SqliteConnOrPath | None,
        targets: Sequence[RateTarget],
        explicit_tables: Sequence[str] | None = None,
        *,
        require_existing: bool = False,
    ) -> list[str]

Resolve SQLite table or view names for rate targets.

Parameters:

Name | Type | Description | Default
--- | --- | --- | ---
`conn_or_path` | `SqliteConnOrPath | None` | SQLite database path or open connection. May be None when `explicit_tables` is provided, or when `require_existing` is False and deterministic default view names are sufficient. | required
`targets` | `Sequence[RateTarget]` | Rate targets to resolve. | required
`explicit_tables` | `Sequence[str] | None` | Optional explicit table or view names. When provided, they are used as-is and must match the number of targets. | `None`
`require_existing` | `bool` | When True, require the database and managed views to exist for each symbol target. Ignored when `explicit_tables` is provided. | `False`

Returns:

Type | Description
--- | ---
`list[str]` | Table or view names aligned with `targets`.

Raises:

Type | Description
--- | ---
`ValueError` | If `targets` is empty, `explicit_tables` length does not match the target count, a target without a symbol is resolved without an explicit table, or `require_existing` is True and the database or a managed view is missing.

Source code in `mt5cli/history.py`

    572
    573
    574
    575
    576
    577
    578
    579
    580
    581
    582
    583
    584
    585
    586
    587
    588
    589
    590
    591
    592
    593
    594
    595
    596
    597
    598
    599
    600
    601
    602
    603
    604
    605
    606
    607
    608
    609
    610
    611
    612
    613
    614
    615
    616
    617
    618
    619
    620
    621
    622
    623
    624
    625
    626
    627
    628
    629
    630
    631
    632
    633
    634
    635
    636
    637
    638
    639
    640
    641
    642
    643
    644
    645
    646
    647
    648
    649
    650
    651
    652
    653

    def resolve_rate_tables(
        conn_or_path: SqliteConnOrPath | None,
        targets: Sequence[RateTarget],
        explicit_tables: Sequence[str] | None = None,
        *,
        require_existing: bool = False,
    ) -> list[str]:
        """Resolve SQLite table or view names for rate targets.

        Args:
            conn_or_path: SQLite database path or open connection. May be None when
                ``explicit_tables`` is provided, or when ``require_existing`` is
                False and deterministic default view names are sufficient.
            targets: Rate targets to resolve.
            explicit_tables: Optional explicit table or view names. When provided,
                they are used as-is and must match the number of targets.
            require_existing: When True, require the database and managed views to
                exist for each symbol target. Ignored when ``explicit_tables`` is
                provided.

        Returns:
            Table or view names aligned with ``targets``.

        Raises:
            ValueError: If ``targets`` is empty, ``explicit_tables`` length does not
                match the target count, a target without a symbol is resolved
                without an explicit table, or ``require_existing`` is True and the
                database or a managed view is missing.
        """
        target_list = list(targets)
        if not target_list:
            msg = "At least one rate target is required."
            raise ValueError(msg)
        if explicit_tables is not None:
            tables = list(explicit_tables)
            if len(tables) != len(target_list):
                msg = (
                    f"Expected {len(target_list)} explicit table(s) "
                    f"to match the targets, got {len(tables)}."
                )
                raise ValueError(msg)
            return tables
        if any(target.symbol is None for target in target_list):
            msg = (
                "Cannot resolve a rate table for a target without a symbol; "
                "provide explicit_tables."
            )
            raise ValueError(msg)
        conn, should_close = _open_history_connection(conn_or_path)
        try:
            if conn is None:
                if require_existing:
                    path = (
                        conn_or_path
                        if isinstance(conn_or_path, (Path, str))
                        else "database"
                    )
                    msg = f"SQLite database not found: {path}"
                    raise ValueError(msg)
                timeframe_counts = None
                existing_views: set[str] = set()
            else:
                timeframe_counts = _load_rates_timeframe_counts(conn)
                existing_views = _load_existing_rate_views(conn)
            resolved: list[str] = []
            for target in target_list:
                symbol = cast("str", target.symbol)
                timeframe = target.timeframe_int
                resolved.append(
                    _resolve_rate_view_name_from_context(
                        symbol=symbol,
                        timeframe=timeframe,
                        granularity_name=resolve_granularity_name(timeframe),
                        timeframe_counts=timeframe_counts,
                        existing_views=existing_views,
                        require_existing=require_existing,
                    ),
                )
            return resolved
        finally:
            if should_close and conn is not None:
                conn.close()

### resolve_rate_view_name

    resolve_rate_view_name(
        conn_or_path: SqliteConnOrPath | None,
        symbol: str,
        granularity: str,
        *,
        require_existing: bool = False,
    ) -> str

Resolve the mt5cli-managed rate compatibility view name.

Parameters:

Name | Type | Description | Default
--- | --- | --- | ---
`conn_or_path` | `SqliteConnOrPath | None` | SQLite database path or open connection. When None or a non-existing path and `require_existing` is False, the deterministic default view name is returned without creating a database file. | required
`symbol` | `str` | Symbol stored in the normalized `rates` table. | required
`granularity` | `str` | Timeframe name (for example `M1`) or integer string. | required
`require_existing` | `bool` | When True, require the database and a managed view to exist. | `False`

Returns:

Type | Description
--- | ---
`str` | View name such as `rate_EURUSD__1` or `rate_EURUSD__M1_1`.

Raises:

Type | Description
--- | ---
`ValueError` | If `require_existing` is True and the database or view is missing.

Source code in `mt5cli/history.py`

    400
    401
    402
    403
    404
    405
    406
    407
    408
    409
    410
    411
    412
    413
    414
    415
    416
    417
    418
    419
    420
    421
    422
    423
    424
    425
    426
    427
    428
    429
    430
    431
    432
    433
    434
    435
    436
    437
    438
    439
    440
    441
    442
    443
    444
    445
    446
    447
    448
    449
    450
    451
    452

    def resolve_rate_view_name(
        conn_or_path: SqliteConnOrPath | None,
        symbol: str,
        granularity: str,
        *,
        require_existing: bool = False,
    ) -> str:
        """Resolve the mt5cli-managed rate compatibility view name.

        Args:
            conn_or_path: SQLite database path or open connection. When None or a
                non-existing path and ``require_existing`` is False, the deterministic
                default view name is returned without creating a database file.
            symbol: Symbol stored in the normalized ``rates`` table.
            granularity: Timeframe name (for example ``M1``) or integer string.
            require_existing: When True, require the database and a managed view to exist.

        Returns:
            View name such as ``rate_EURUSD__1`` or ``rate_EURUSD__M1_1``.

        Raises:
            ValueError: If ``require_existing`` is True and the database or view is missing.
        """
        timeframe = parse_timeframe(granularity)
        granularity_name = resolve_granularity_name(timeframe)
        conn, should_close = _open_history_connection(conn_or_path)
        try:
            if conn is None:
                if require_existing:
                    path = (
                        conn_or_path
                        if isinstance(conn_or_path, (Path, str))
                        else "database"
                    )
                    msg = f"SQLite database not found: {path}"
                    raise ValueError(msg)
                return build_rate_view_name(
                    symbol=symbol,
                    granularity=granularity_name,
                    granularity_count=1,
                    timeframe=timeframe,
                )
            return _resolve_rate_view_name_from_context(
                symbol=symbol,
                timeframe=timeframe,
                granularity_name=granularity_name,
                timeframe_counts=_load_rates_timeframe_counts(conn),
                existing_views=_load_existing_rate_views(conn),
                require_existing=require_existing,
            )
        finally:
            if should_close and conn is not None:
                conn.close()

### resolve_rate_view_names

    resolve_rate_view_names(
        conn_or_path: SqliteConnOrPath | None,
        symbols: Sequence[str],
        granularities: Sequence[str],
        *,
        require_existing: bool = False,
    ) -> list[str]

Resolve rate compatibility view names for symbol and granularity pairs.

Parameters:

Name | Type | Description | Default
--- | --- | --- | ---
`conn_or_path` | `SqliteConnOrPath | None` | SQLite database path or open connection. When None or a non-existing path and `require_existing` is False, deterministic default view names are returned without creating a database file. | required
`symbols` | `Sequence[str]` | Symbols stored in the normalized `rates` table. | required
`granularities` | `Sequence[str]` | Timeframe names (for example `M1`) or integer strings. | required
`require_existing` | `bool` | When True, require the database and managed views to exist. | `False`

Returns:

Type | Description
--- | ---
`list[str]` | View names in row-major order: every `granularity` for the first
`list[str]` | symbol, then every granularity for the next symbol, and so on.

Source code in `mt5cli/history.py`

    455
    456
    457
    458
    459
    460
    461
    462
    463
    464
    465
    466
    467
    468
    469
    470
    471
    472
    473
    474
    475
    476
    477
    478
    479
    480
    481
    482
    483
    484
    485
    486
    487
    488
    489
    490
    491
    492
    493
    494
    495
    496
    497
    498
    499
    500
    501
    502
    503
    504
    505
    506
    507
    508

    def resolve_rate_view_names(
        conn_or_path: SqliteConnOrPath | None,
        symbols: Sequence[str],
        granularities: Sequence[str],
        *,
        require_existing: bool = False,
    ) -> list[str]:
        """Resolve rate compatibility view names for symbol and granularity pairs.

        Args:
            conn_or_path: SQLite database path or open connection. When None or a
                non-existing path and ``require_existing`` is False, deterministic
                default view names are returned without creating a database file.
            symbols: Symbols stored in the normalized ``rates`` table.
            granularities: Timeframe names (for example ``M1``) or integer strings.
            require_existing: When True, require the database and managed views to exist.

        Returns:
            View names in row-major order: every ``granularity`` for the first
            symbol, then every granularity for the next symbol, and so on.
        """
        conn, should_close = _open_history_connection(conn_or_path)
        try:
            if conn is None:
                return [
                    resolve_rate_view_name(
                        conn_or_path,
                        symbol,
                        granularity,
                        require_existing=require_existing,
                    )
                    for symbol in symbols
                    for granularity in granularities
                ]
            timeframe_counts = _load_rates_timeframe_counts(conn)
            existing_views = _load_existing_rate_views(conn)
            resolved: list[str] = []
            for symbol in symbols:
                for granularity in granularities:
                    timeframe = parse_timeframe(granularity)
                    resolved.append(
                        _resolve_rate_view_name_from_context(
                            symbol=symbol,
                            timeframe=timeframe,
                            granularity_name=resolve_granularity_name(timeframe),
                            timeframe_counts=timeframe_counts,
                            existing_views=existing_views,
                            require_existing=require_existing,
                        ),
                    )
            return resolved
        finally:
            if should_close and conn is not None:
                conn.close()

### update_history

    update_history(
        *,
        client: Mt5DataClient,
        output: Path | str,
        symbols: Sequence[str],
        datasets: set[Dataset] | None = None,
        timeframes: Sequence[int | str] | None = None,
        flags: int | str = "ALL",
        lookback_hours: float = 24.0,
        date_to: datetime | str | None = None,
        deduplicate: bool = True,
        create_rate_views: bool = True,
        with_views: bool = False,
        include_account_events: bool = True,
    ) -> None

Incrementally append MT5 history into a SQLite database.

Uses an already-connected `Mt5DataClient` and does not create or close the MT5 connection. For first-time tables, data is fetched from `date_to - lookback_hours`. Subsequent runs resume from existing `MAX(time)` per symbol (and timeframe for rates); when `include_account_events=True`, account-level deals use a separate cursor over `type NOT IN (0, 1)` / empty-symbol rows.

Parameters:

Name | Type | Description | Default
--- | --- | --- | ---
`client` | `Mt5DataClient` | Connected MT5 data client. | required
`output` | `Path | str` | SQLite database path. | required
`symbols` | `Sequence[str]` | Symbols to update. | required
`datasets` | `set[Dataset] | None` | Datasets to include (defaults to all). | `None`
`timeframes` | `Sequence[int | str] | None` | Rate timeframes to update (defaults to all fixed MT5 timeframes when None). | `None`
`flags` | `int | str` | Tick copy flags as integer or name (e.g. `ALL`). | `'ALL'`
`lookback_hours` | `float` | First-run lookback when a table has no prior rows. | `24.0`
`date_to` | `datetime | str | None` | Optional update end datetime. Defaults to now (UTC). | `None`
`deduplicate` | `bool` | Remove duplicate rows after append, keeping latest ROWID. | `True`
`create_rate_views` | `bool` | Create `rate_<symbol>__<timeframe>` views. | `True`
`with_views` | `bool` | Create `cash_events` and `positions_reconstructed` views. | `False`
`include_account_events` | `bool` | Include account-level cash events in `history_deals` when True. | `True`

Source code in `mt5cli/sdk.py`

    918
    919
    920
    921
    922
    923
    924
    925
    926
    927
    928
    929
    930
    931
    932
    933
    934
    935
    936
    937
    938
    939
    940
    941
    942
    943
    944
    945
    946
    947
    948
    949
    950
    951
    952
    953
    954
    955
    956
    957
    958
    959
    960
    961
    962
    963
    964
    965
    966
    967
    968
    969
    970
    971
    972
    973
    974
    975
    976
    977
    978
    979
    980
    981
    982
    983
    984
    985
    986
    987
    988
    989
    990
    991

    def update_history(  # noqa: PLR0913
        *,
        client: Mt5DataClient,
        output: Path | str,
        symbols: Sequence[str],
        datasets: set[Dataset] | None = None,
        timeframes: Sequence[int | str] | None = None,
        flags: int | str = "ALL",
        lookback_hours: float = 24.0,
        date_to: datetime | str | None = None,
        deduplicate: bool = True,
        create_rate_views: bool = True,
        with_views: bool = False,
        include_account_events: bool = True,
    ) -> None:
        """Incrementally append MT5 history into a SQLite database.

        Uses an already-connected ``Mt5DataClient`` and does not create or close
        the MT5 connection. For first-time tables, data is fetched from
        ``date_to - lookback_hours``. Subsequent runs resume from existing
        ``MAX(time)`` per symbol (and timeframe for rates); when
        ``include_account_events=True``, account-level deals use a separate cursor
        over ``type NOT IN (0, 1)`` / empty-symbol rows.

        Args:
            client: Connected MT5 data client.
            output: SQLite database path.
            symbols: Symbols to update.
            datasets: Datasets to include (defaults to all).
            timeframes: Rate timeframes to update (defaults to all fixed MT5
                timeframes when None).
            flags: Tick copy flags as integer or name (e.g. ``ALL``).
            lookback_hours: First-run lookback when a table has no prior rows.
            date_to: Optional update end datetime. Defaults to now (UTC).
            deduplicate: Remove duplicate rows after append, keeping latest ROWID.
            create_rate_views: Create ``rate_<symbol>__<timeframe>`` views.
            with_views: Create ``cash_events`` and ``positions_reconstructed`` views.
            include_account_events: Include account-level cash events in
                ``history_deals`` when True.
        """
        request = _resolve_update_history_request(
            output=output,
            symbols=symbols,
            datasets=datasets,
            timeframes=timeframes,
            flags=flags,
            lookback_hours=lookback_hours,
            date_to=date_to,
        )
        if request is None:
            return
        logger.info(
            "Updating history in SQLite: symbols=%s, datasets=%s, path=%s",
            list(symbols),
            sorted(dataset.value for dataset in request.selected),
            request.output_path,
        )
        with sqlite3.connect(request.output_path) as conn:
            conn.execute("PRAGMA journal_mode=WAL")
            conn.execute("PRAGMA synchronous=NORMAL")
            write_incremental_datasets(
                conn,
                client,
                symbols,
                request.selected,
                request.resolved_timeframes,
                request.resolved_tick_flags,
                request.fallback_start,
                request.end,
                deduplicate=deduplicate,
                create_rate_views=create_rate_views,
                with_views=with_views,
                include_account_events=include_account_events,
            )

### update_history_with_config

    update_history_with_config(
        *,
        output: Path | str,
        symbols: Sequence[str],
        config: Mt5Config | None = None,
        datasets: set[Dataset] | None = None,
        timeframes: Sequence[int | str] | None = None,
        flags: int | str = "ALL",
        lookback_hours: float = 24.0,
        date_to: datetime | str | None = None,
        deduplicate: bool = True,
        create_rate_views: bool = True,
        with_views: bool = False,
        include_account_events: bool = True,
    ) -> None

Incrementally append MT5 history, opening and closing the MT5 connection.

Convenience wrapper around :func:`update_history` for standalone use.

Source code in `mt5cli/sdk.py`

     994
     995
     996
     997
     998
     999
    1000
    1001
    1002
    1003
    1004
    1005
    1006
    1007
    1008
    1009
    1010
    1011
    1012
    1013
    1014
    1015
    1016
    1017
    1018
    1019
    1020
    1021
    1022
    1023
    1024
    1025
    1026
    1027
    1028
    1029
    1030
    1031
    1032
    1033
    1034
    1035
    1036
    1037
    1038
    1039

    def update_history_with_config(  # noqa: PLR0913
        *,
        output: Path | str,
        symbols: Sequence[str],
        config: Mt5Config | None = None,
        datasets: set[Dataset] | None = None,
        timeframes: Sequence[int | str] | None = None,
        flags: int | str = "ALL",
        lookback_hours: float = 24.0,
        date_to: datetime | str | None = None,
        deduplicate: bool = True,
        create_rate_views: bool = True,
        with_views: bool = False,
        include_account_events: bool = True,
    ) -> None:
        """Incrementally append MT5 history, opening and closing the MT5 connection.

        Convenience wrapper around :func:`update_history` for standalone use.
        """
        request = _resolve_update_history_request(
            output=output,
            symbols=symbols,
            datasets=datasets,
            timeframes=timeframes,
            flags=flags,
            lookback_hours=lookback_hours,
            date_to=date_to,
        )
        if request is None:
            return
        mt5_config = config or build_config()
        with connected_client(mt5_config) as client:
            update_history(
                client=client,
                output=output,
                symbols=symbols,
                datasets=datasets,
                timeframes=timeframes,
                flags=flags,
                lookback_hours=lookback_hours,
                date_to=date_to,
                deduplicate=deduplicate,
                create_rate_views=create_rate_views,
                with_views=with_views,
                include_account_events=include_account_events,
            )

#### Search

From here you can search these documents. Enter your search terms below.

#### Keyboard Shortcuts

Keys | Action
--- | ---
`?` | Open this help
`n` | Next page
`p` | Previous page
`s` | Search