+ +

SQLite History Module

+ + +
+ + + +

+ mt5cli.sqlite_history + + +

+ +
+ +

SQLite helpers for incremental MT5 history collection.

+ + + + + + + + + + +
+ + + + + + + +
+ + + +

+ DEFAULT_HISTORY_TIMEFRAMES + + + + module-attribute + + +

+
DEFAULT_HISTORY_TIMEFRAMES: tuple[str, ...] = tuple(
+    TIMEFRAME_MAP
+)
+
+ +
+ +
+ +
+ +
+ + + +

+ DedupScope + + + + module-attribute + + +

+
DedupScope = tuple[str, tuple[object, ...]]
+
+ +
+ +
+ +
+ +
+ + + +

+ logger + + + + module-attribute + + +

+
logger = getLogger(__name__)
+
+ +
+ +
+ +
+ + + + +
+ + +

+ append_dataframe + + +

+
append_dataframe(
+    conn: Connection,
+    frame: DataFrame,
+    table_name: str,
+    if_exists: IfExists,
+) -> bool
+
+ +
+ +

Append a DataFrame to SQLite when it has a schema.

+ + +

Returns:

+ + + + + + + + + + + + + +
TypeDescription
+ bool + +
+

True if a table was written, False if the frame had no columns.

+
+
+ + +
+ Source code in mt5cli/sqlite_history.py + 
298
+299
+300
+301
+302
+303
+304
+305
+306
+307
+308
+309
+310
+311
+312
+313
+314
+315
+316
+317
+318
+319
def append_dataframe(
+    conn: sqlite3.Connection,
+    frame: pd.DataFrame,
+    table_name: str,
+    if_exists: IfExists,
+) -> bool:
+    """Append a DataFrame to SQLite when it has a schema.
+
+    Returns:
+        True if a table was written, False if the frame had no columns.
+    """
+    if len(frame.columns) == 0:
+        logger.warning("Skipping %s: dataset returned no columns", table_name)
+        return False
+    frame.to_sql(  # type: ignore[reportUnknownMemberType]
+        table_name,
+        conn,
+        if_exists=if_exists.value,
+        index=False,
+        chunksize=50_000,
+    )
+    return True
+
+
+
+ +
+ +
+ + +

+ augment_written_columns_from_sqlite + + +

+
augment_written_columns_from_sqlite(
+    conn: Connection,
+    datasets: set[Dataset],
+    written_columns: dict[Dataset, set[str]],
+) -> None
+
+ +
+ +

Add existing table columns to the written column map.

+ + +
+ Source code in mt5cli/sqlite_history.py + 
335
+336
+337
+338
+339
+340
+341
+342
+343
+344
+345
+346
+347
+348
def augment_written_columns_from_sqlite(
+    conn: sqlite3.Connection,
+    datasets: set[Dataset],
+    written_columns: dict[Dataset, set[str]],
+) -> None:
+    """Add existing table columns to the written column map."""
+    for dataset in datasets:
+        columns = get_table_columns(conn, dataset.table_name)
+        if not columns:
+            continue
+        if dataset in written_columns:
+            written_columns[dataset].update(columns)
+        else:
+            written_columns[dataset] = columns
+
+
+
+ +
+ +
+ + +

+ build_rate_view_name + + +

+
build_rate_view_name(
+    *,
+    symbol: str,
+    granularity: str,
+    granularity_count: int,
+    timeframe: int,
+) -> str
+
+ +
+ +

Return a collision-free offline optimize view name.

+

View names always include the timeframe integer after a __ separator so +a symbol such as EURUSD_M1 cannot collide with EURUSD at timeframe +M1.

+ + +
+ Source code in mt5cli/sqlite_history.py + 
107
+108
+109
+110
+111
+112
+113
+114
+115
+116
+117
+118
+119
+120
+121
+122
def build_rate_view_name(
+    *,
+    symbol: str,
+    granularity: str,
+    granularity_count: int,
+    timeframe: int,
+) -> str:
+    """Return a collision-free offline optimize view name.
+
+    View names always include the timeframe integer after a ``__`` separator so
+    a symbol such as ``EURUSD_M1`` cannot collide with ``EURUSD`` at timeframe
+    ``M1``.
+    """
+    if granularity_count == 1:
+        return f"rate_{symbol}__{timeframe}"
+    return f"rate_{symbol}__{granularity}_{timeframe}"
+
+
+
+ +
+ +
+ + +

+ create_cash_events_view + + +

+
create_cash_events_view(
+    conn: Connection, deals_columns: set[str]
+) -> bool
+
+ +
+ +

Create the cash_events SQLite view derived from history_deals.

+ + +

Returns:

+ + + + + + + + + + + + + +
TypeDescription
+ bool + +
+

True if the view was created, False if required columns are missing.

+
+
+ + +
+ Source code in mt5cli/sqlite_history.py + 
548
+549
+550
+551
+552
+553
+554
+555
+556
+557
+558
+559
+560
+561
+562
+563
+564
+565
def create_cash_events_view(
+    conn: sqlite3.Connection,
+    deals_columns: set[str],
+) -> bool:
+    """Create the cash_events SQLite view derived from history_deals.
+
+    Returns:
+        True if the view was created, False if required columns are missing.
+    """
+    if "type" not in deals_columns:
+        logger.warning("Skipping cash_events view: history_deals.type is missing")
+        return False
+    conn.execute("DROP VIEW IF EXISTS cash_events")
+    conn.execute(
+        "CREATE VIEW cash_events AS"  # noqa: S608
+        f" SELECT * FROM history_deals WHERE type NOT IN {_TRADE_DEAL_TYPES_SQL}",
+    )
+    return True
+
+
+
+ +
+ +
+ + +

+ create_history_indexes + + +

+
create_history_indexes(
+    conn: Connection,
+    written_columns: dict[Dataset, set[str]],
+) -> None
+
+ +
+ +

Create useful indexes for collected history tables when present.

+ + +
+ Source code in mt5cli/sqlite_history.py + 
459
+460
+461
+462
+463
+464
+465
+466
+467
+468
+469
+470
+471
+472
+473
+474
+475
+476
+477
+478
+479
+480
+481
def create_history_indexes(
+    conn: sqlite3.Connection,
+    written_columns: dict[Dataset, set[str]],
+) -> None:
+    """Create useful indexes for collected history tables when present."""
+    if {"symbol", "timeframe", "time"}.issubset(
+        written_columns.get(Dataset.rates, set()),
+    ):
+        conn.execute(
+            "CREATE INDEX IF NOT EXISTS idx_rates_symbol_timeframe_time"
+            " ON rates(symbol, timeframe, time)",
+        )
+    if {"symbol", "time"}.issubset(written_columns.get(Dataset.ticks, set())):
+        conn.execute(
+            "CREATE INDEX IF NOT EXISTS idx_ticks_symbol_time ON ticks(symbol, time)",
+        )
+    if {"position_id", "symbol"}.issubset(
+        written_columns.get(Dataset.history_deals, set()),
+    ):
+        conn.execute(
+            "CREATE INDEX IF NOT EXISTS idx_history_deals_position_symbol"
+            " ON history_deals(position_id, symbol)",
+        )
+
+
+
+ +
+ +
+ + +

+ create_positions_reconstructed_view + + +

+
create_positions_reconstructed_view(
+    conn: Connection, deals_columns: set[str]
+) -> bool
+
+ +
+ +

Create the positions_reconstructed SQLite view derived from history_deals.

+ + +

Returns:

+ + + + + + + + + + + + + +
TypeDescription
+ bool + +
+

True if the view was created, False if required columns are missing.

+
+
+ + +
+ Source code in mt5cli/sqlite_history.py + 
568
+569
+570
+571
+572
+573
+574
+575
+576
+577
+578
+579
+580
+581
+582
+583
+584
+585
+586
+587
+588
+589
+590
+591
+592
+593
+594
+595
+596
+597
+598
+599
+600
+601
+602
+603
+604
+605
+606
+607
+608
+609
+610
+611
+612
+613
+614
def create_positions_reconstructed_view(
+    conn: sqlite3.Connection,
+    deals_columns: set[str],
+) -> bool:
+    """Create the positions_reconstructed SQLite view derived from history_deals.
+
+    Returns:
+        True if the view was created, False if required columns are missing.
+    """
+    if not _POSITIONS_VIEW_REQUIRED_COLUMNS.issubset(deals_columns):
+        missing = ", ".join(sorted(_POSITIONS_VIEW_REQUIRED_COLUMNS - deals_columns))
+        logger.warning(
+            "Skipping positions_reconstructed view: history_deals missing columns: %s",
+            missing,
+        )
+        return False
+    conn.execute("DROP VIEW IF EXISTS positions_reconstructed")
+    conn.execute(
+        "CREATE VIEW positions_reconstructed AS"  # noqa: S608
+        " SELECT"
+        " position_id,"
+        " symbol,"
+        " MIN(CASE WHEN entry = 0 THEN time END) AS open_time,"
+        " MAX(CASE WHEN entry IN (1, 2, 3) THEN time END) AS close_time,"
+        " MIN(CASE WHEN entry = 0 THEN type END) AS direction,"
+        " SUM(CASE WHEN entry = 0 THEN volume ELSE 0 END) AS volume_open,"
+        " SUM(CASE WHEN entry IN (1, 2, 3) THEN volume ELSE 0 END) AS volume_close,"
+        " SUM(CASE WHEN entry = 2 THEN volume ELSE 0 END) AS volume_reversal,"
+        " CASE"
+        " WHEN SUM(CASE WHEN entry = 0 THEN volume ELSE 0 END) > 0"
+        " THEN SUM(CASE WHEN entry = 0 THEN price * volume ELSE 0 END)"
+        " / SUM(CASE WHEN entry = 0 THEN volume ELSE 0 END)"
+        " END AS open_price,"
+        " CASE"
+        " WHEN SUM(CASE WHEN entry IN (1, 2, 3) THEN volume ELSE 0 END) > 0"
+        " THEN SUM(CASE WHEN entry IN (1, 2, 3) THEN price * volume ELSE 0 END)"
+        " / SUM(CASE WHEN entry IN (1, 2, 3) THEN volume ELSE 0 END)"
+        " END AS close_price,"
+        " SUM(profit) AS total_profit,"
+        " SUM(CASE WHEN entry = 2 THEN 1 ELSE 0 END) AS reversal_count,"
+        " COUNT(*) AS deals_count"
+        " FROM history_deals"
+        f" WHERE type IN {_TRADE_DEAL_TYPES_SQL} AND position_id != 0"
+        " GROUP BY position_id, symbol"
+        " HAVING SUM(CASE WHEN entry IN (1, 2, 3) THEN 1 ELSE 0 END) > 0",
+    )
+    return True
+
+
+
+ +
+ +
+ + +

+ create_rate_compatibility_views + + +

+
create_rate_compatibility_views(conn: Connection) -> None
+
+ +
+ +

Create rate compatibility views from the normalized rates table.

+ + +
+ Source code in mt5cli/sqlite_history.py + 
627
+628
+629
+630
+631
+632
+633
+634
+635
+636
+637
+638
+639
+640
+641
+642
+643
+644
+645
+646
+647
+648
+649
+650
+651
+652
+653
+654
+655
+656
+657
def create_rate_compatibility_views(conn: sqlite3.Connection) -> None:
+    """Create rate compatibility views from the normalized rates table."""
+    columns = get_table_columns(conn, Dataset.rates.table_name)
+    if not {"symbol", "timeframe", "time"}.issubset(columns):
+        return
+    drop_rate_compatibility_views(conn)
+    select_columns = sorted(columns - {"symbol", "timeframe"})
+    quoted_columns = ", ".join(f'"{column}"' for column in select_columns)
+    rows = conn.execute(
+        "SELECT DISTINCT symbol, timeframe FROM rates ORDER BY symbol, timeframe",
+    ).fetchall()
+    timeframes_by_symbol: dict[str, list[int]] = {}
+    for symbol, timeframe in rows:
+        timeframes_by_symbol.setdefault(str(symbol), []).append(int(timeframe))
+    for symbol, timeframes in timeframes_by_symbol.items():
+        for timeframe in timeframes:
+            granularity = resolve_granularity_name(timeframe)
+            view_name = build_rate_view_name(
+                symbol=symbol,
+                granularity=granularity,
+                granularity_count=len(timeframes),
+                timeframe=timeframe,
+            )
+            quoted_view_name = quote_sqlite_identifier(view_name)
+            escaped_symbol = symbol.replace("'", "''")
+            conn.execute(
+                f"CREATE VIEW {quoted_view_name} AS"  # noqa: S608
+                f" SELECT {quoted_columns} FROM rates"
+                f" WHERE symbol = '{escaped_symbol}'"
+                f" AND timeframe = {timeframe}",
+            )
+
+
+
+ +
+ +
+ + +

+ deduplicate_history_tables + + +

+
deduplicate_history_tables(
+    conn: Connection,
+    written_columns: dict[Dataset, set[str]],
+    written_tables: set[Dataset],
+    dedup_scopes: dict[Dataset, list[DedupScope]]
+    | None = None,
+) -> None
+
+ +
+ +

Deduplicate appended history tables by stable identifiers.

+ + +
+ Source code in mt5cli/sqlite_history.py + 
419
+420
+421
+422
+423
+424
+425
+426
+427
+428
+429
+430
+431
+432
+433
+434
+435
+436
+437
+438
+439
+440
+441
+442
+443
+444
+445
+446
+447
+448
+449
+450
+451
+452
+453
+454
+455
+456
def deduplicate_history_tables(
+    conn: sqlite3.Connection,
+    written_columns: dict[Dataset, set[str]],
+    written_tables: set[Dataset],
+    dedup_scopes: dict[Dataset, list[DedupScope]] | None = None,
+) -> None:
+    """Deduplicate appended history tables by stable identifiers."""
+    cursor = conn.cursor()
+    for dataset in written_tables:
+        columns = written_columns.get(dataset, set())
+        table = dataset.table_name
+        keys = next(
+            (
+                candidate
+                for candidate in _HISTORY_DEDUP_KEYS[dataset]
+                if set(candidate).issubset(columns)
+            ),
+            None,
+        )
+        if keys is None:
+            logger.warning(
+                "Skipping %s deduplication: no supported key columns",
+                table,
+            )
+            continue
+        scopes = dedup_scopes.get(dataset, []) if dedup_scopes else []
+        if scopes:
+            for scope_where, scope_params in scopes:
+                drop_duplicates_in_table(
+                    cursor,
+                    table,
+                    list(keys),
+                    keep="last",
+                    scope_where=scope_where,
+                    scope_params=scope_params,
+                )
+            continue
+        drop_duplicates_in_table(cursor, table, list(keys), keep="last")
+
+
+
+ +
+ +
+ + +

+ drop_duplicates_in_table + + +

+
drop_duplicates_in_table(
+    cursor: Cursor,
+    table: str,
+    ids: list[str],
+    *,
+    keep: Literal["first", "last"] = "last",
+    scope_where: str | None = None,
+    scope_params: tuple[object, ...] = (),
+) -> None
+
+ +
+ +

Remove duplicate rows, keeping the first or last ROWID per key group.

+ + +

Raises:

+ + + + + + + + + + + + + +
TypeDescription
+ ValueError + +
+

If the table or column names are invalid.

+
+
+ + +
+ Source code in mt5cli/sqlite_history.py + 
371
+372
+373
+374
+375
+376
+377
+378
+379
+380
+381
+382
+383
+384
+385
+386
+387
+388
+389
+390
+391
+392
+393
+394
+395
+396
+397
+398
+399
+400
+401
+402
+403
+404
def drop_duplicates_in_table(
+    cursor: sqlite3.Cursor,
+    table: str,
+    ids: list[str],
+    *,
+    keep: Literal["first", "last"] = "last",
+    scope_where: str | None = None,
+    scope_params: tuple[object, ...] = (),
+) -> None:
+    """Remove duplicate rows, keeping the first or last ROWID per key group.
+
+    Raises:
+        ValueError: If the table or column names are invalid.
+    """
+    if not table.isidentifier():
+        msg = f"Invalid table name: {table}"
+        raise ValueError(msg)
+    if invalid := {column for column in ids if not column.isidentifier()}:
+        msg = f"Invalid column names: {', '.join(sorted(invalid))}"
+        raise ValueError(msg)
+    ids_csv = ", ".join(f'"{column}"' for column in ids)
+    rowid_selector = "MIN" if keep == "first" else "MAX"
+    if scope_where:
+        delete_sql = (
+            f"DELETE FROM {table} WHERE {scope_where} AND ROWID NOT IN"  # noqa: S608
+            f" (SELECT {rowid_selector}(ROWID) FROM {table} WHERE {scope_where}"
+            f" GROUP BY {ids_csv})"
+        )
+        cursor.execute(delete_sql, scope_params + scope_params)
+        return
+    cursor.execute(
+        f"DELETE FROM {table} WHERE ROWID NOT IN"  # noqa: S608
+        f" (SELECT {rowid_selector}(ROWID) FROM {table} GROUP BY {ids_csv})",
+    )
+
+
+
+ +
+ +
+ + +

+ drop_rate_compatibility_views + + +

+
drop_rate_compatibility_views(conn: Connection) -> None
+
+ +
+ +

Drop all mt5cli-managed rate_* compatibility views.

+ + +
+ Source code in mt5cli/sqlite_history.py + 
617
+618
+619
+620
+621
+622
+623
+624
def drop_rate_compatibility_views(conn: sqlite3.Connection) -> None:
+    """Drop all mt5cli-managed ``rate_*`` compatibility views."""
+    rows = conn.execute(
+        "SELECT name FROM sqlite_master WHERE type = 'view' AND name GLOB 'rate_*'",
+    ).fetchall()
+    for (view_name,) in rows:
+        quoted_view_name = quote_sqlite_identifier(str(view_name))
+        conn.execute(f"DROP VIEW IF EXISTS {quoted_view_name}")
+
+
+
+ +
+ +
+ + +

+ filter_incremental_history_deals_frame + + +

+
filter_incremental_history_deals_frame(
+    frame: DataFrame,
+    symbols: Sequence[str],
+    start_by_symbol: dict[str, datetime],
+    account_event_start: datetime,
+) -> DataFrame
+
+ +
+ +

Filter incrementally fetched history_deals by symbol and event start times.

+ + +

Returns:

+ + + + + + + + + + + + + + + + + +
TypeDescription
+ DataFrame + +
+

Rows for selected symbols at or after each symbol start, plus account

+
+
+ DataFrame + +
+

events at or after account_event_start.

+
+
+ + +
+ Source code in mt5cli/sqlite_history.py + 
498
+499
+500
+501
+502
+503
+504
+505
+506
+507
+508
+509
+510
+511
+512
+513
+514
+515
+516
+517
+518
+519
+520
+521
+522
+523
+524
+525
def filter_incremental_history_deals_frame(
+    frame: pd.DataFrame,
+    symbols: Sequence[str],
+    start_by_symbol: dict[str, datetime],
+    account_event_start: datetime,
+) -> pd.DataFrame:
+    """Filter incrementally fetched history_deals by symbol and event start times.
+
+    Returns:
+        Rows for selected symbols at or after each symbol start, plus account
+        events at or after ``account_event_start``.
+    """
+    if frame.empty:
+        return frame.copy()
+    parsed_times = _frame_parsed_times(frame)
+    time_valid = parsed_times.notna()
+    account_event_mask = _history_deals_account_event_mask(frame)
+    account_keep = account_event_mask & (parsed_times >= account_event_start)
+    trade_keep = pd.Series(data=False, index=frame.index)
+    if "symbol" in frame.columns:
+        for symbol in symbols:
+            trade_keep |= (
+                (frame["symbol"] == symbol)
+                & (parsed_times >= start_by_symbol[symbol])
+                & ~account_event_mask
+            )
+    keep = (account_keep | trade_keep) & time_valid
+    return frame.loc[keep].copy()
+
+
+
+ +
+ +
+ + +

+ filter_trade_history_frame + + +

+
filter_trade_history_frame(
+    frame: DataFrame,
+    symbols: Sequence[str],
+    *,
+    include_account_events: bool,
+) -> DataFrame
+
+ +
+ +

Filter trade history rows to selected symbols and account events.

+ + +

Returns:

+ + + + + + + + + + + + + +
TypeDescription
+ DataFrame + +
+

Filtered history rows.

+
+
+ + +
+ Source code in mt5cli/sqlite_history.py + 
528
+529
+530
+531
+532
+533
+534
+535
+536
+537
+538
+539
+540
+541
+542
+543
+544
+545
def filter_trade_history_frame(
+    frame: pd.DataFrame,
+    symbols: Sequence[str],
+    *,
+    include_account_events: bool,
+) -> pd.DataFrame:
+    """Filter trade history rows to selected symbols and account events.
+
+    Returns:
+        Filtered history rows.
+    """
+    if "symbol" not in frame.columns:
+        return frame
+    symbol_mask = frame["symbol"].isin(symbols)
+    if not include_account_events:
+        return frame.loc[symbol_mask].copy()
+    account_event_mask = _history_deals_account_event_mask(frame)
+    return frame.loc[symbol_mask | account_event_mask].copy()
+
+
+
+ +
+ +
+ + +

+ get_history_deals_account_event_start_datetime + + +

+
get_history_deals_account_event_start_datetime(
+    conn: Connection, *, fallback_start: datetime
+) -> datetime
+
+ +
+ +

Return the next update start for account-level history_deals rows.

+ + +
+ Source code in mt5cli/sqlite_history.py + 
160
+161
+162
+163
+164
+165
+166
+167
+168
+169
+170
+171
+172
+173
+174
+175
+176
+177
+178
+179
+180
def get_history_deals_account_event_start_datetime(
+    conn: sqlite3.Connection,
+    *,
+    fallback_start: datetime,
+) -> datetime:
+    """Return the next update start for account-level history_deals rows."""
+    table = Dataset.history_deals.table_name
+    columns = get_table_columns(conn, table)
+    if "time" not in columns:
+        return fallback_start
+    if "type" in columns:
+        where_clause = f"type NOT IN {_TRADE_DEAL_TYPES_SQL}"
+    elif "symbol" in columns:
+        where_clause = "symbol IS NULL OR symbol = ''"
+    else:
+        return fallback_start
+    row = conn.execute(
+        f"SELECT MAX(time) FROM {table} WHERE {where_clause}",  # noqa: S608
+    ).fetchone()
+    parsed = parse_sqlite_timestamp(row[0] if row else None)
+    return parsed if parsed is not None else fallback_start
+
+
+
+ +
+ +
+ + +

+ get_incremental_start_datetime + + +

+
get_incremental_start_datetime(
+    conn: Connection,
+    dataset: Dataset,
+    *,
+    symbol: str,
+    timeframe: int | None,
+    fallback_start: datetime,
+) -> datetime
+
+ +
+ +

Return the next update start datetime from existing MAX(time).

+ + +
+ Source code in mt5cli/sqlite_history.py + 
278
+279
+280
+281
+282
+283
+284
+285
+286
+287
+288
+289
+290
+291
+292
+293
+294
+295
def get_incremental_start_datetime(
+    conn: sqlite3.Connection,
+    dataset: Dataset,
+    *,
+    symbol: str,
+    timeframe: int | None,
+    fallback_start: datetime,
+) -> datetime:
+    """Return the next update start datetime from existing MAX(time)."""
+    timeframes = [timeframe] if timeframe is not None else None
+    starts = load_incremental_start_datetimes(
+        conn,
+        dataset,
+        symbols=[symbol],
+        timeframes=timeframes,
+        fallback_start=fallback_start,
+    )
+    return starts[symbol, timeframe]
+
+
+
+ +
+ +
+ + +

+ get_table_columns + + +

+
get_table_columns(conn: Connection, table: str) -> set[str]
+
+ +
+ +

Return existing SQLite columns for a table.

+ + +
+ Source code in mt5cli/sqlite_history.py + 
125
+126
+127
+128
def get_table_columns(conn: sqlite3.Connection, table: str) -> set[str]:
+    """Return existing SQLite columns for a table."""
+    rows = conn.execute(f"PRAGMA table_info({table})").fetchall()
+    return {str(row[1]) for row in rows}
+
+
+
+ +
+ +
+ + +

+ load_incremental_start_datetimes + + +

+
load_incremental_start_datetimes(
+    conn: Connection,
+    dataset: Dataset,
+    *,
+    symbols: Sequence[str],
+    timeframes: Sequence[int] | None = None,
+    fallback_start: datetime,
+) -> dict[tuple[str, int | None], datetime]
+
+ +
+ +

Return next update start datetimes keyed by symbol and optional timeframe.

+ + +
+ Source code in mt5cli/sqlite_history.py + 
202
+203
+204
+205
+206
+207
+208
+209
+210
+211
+212
+213
+214
+215
+216
+217
+218
+219
+220
+221
+222
+223
+224
+225
+226
+227
+228
+229
+230
+231
+232
+233
+234
+235
+236
+237
+238
+239
+240
+241
+242
+243
+244
+245
+246
+247
+248
+249
+250
+251
+252
+253
+254
+255
+256
+257
+258
+259
+260
+261
+262
+263
+264
+265
+266
+267
+268
+269
+270
+271
+272
+273
+274
+275
def load_incremental_start_datetimes(
+    conn: sqlite3.Connection,
+    dataset: Dataset,
+    *,
+    symbols: Sequence[str],
+    timeframes: Sequence[int] | None = None,
+    fallback_start: datetime,
+) -> dict[tuple[str, int | None], datetime]:
+    """Return next update start datetimes keyed by symbol and optional timeframe."""
+    table = dataset.table_name
+    columns = get_table_columns(conn, table)
+    if dataset is Dataset.rates and columns:
+        _validate_rates_schema(columns)
+
+    if "time" not in columns:
+        if dataset is Dataset.rates and timeframes is not None:
+            return {
+                (symbol, timeframe): fallback_start
+                for symbol in symbols
+                for timeframe in timeframes
+            }
+        return {(symbol, None): fallback_start for symbol in symbols}
+
+    parsed_by_key: dict[tuple[str, int | None], datetime] = {}
+    if (
+        dataset is Dataset.rates
+        and timeframes is not None
+        and {"symbol", "timeframe"}.issubset(columns)
+    ):
+        symbol_placeholders = ", ".join("?" for _ in symbols)
+        timeframe_placeholders = ", ".join("?" for _ in timeframes)
+        grouped_rates_query = (
+            "SELECT symbol, timeframe, MAX(time) FROM "  # noqa: S608
+            f"{table} WHERE symbol IN ({symbol_placeholders})"
+            f" AND timeframe IN ({timeframe_placeholders})"
+            " GROUP BY symbol, timeframe"
+        )
+        rows = conn.execute(
+            grouped_rates_query,
+            [*symbols, *timeframes],
+        ).fetchall()
+        for row_symbol, row_timeframe, max_time in rows:
+            parsed = parse_sqlite_timestamp(max_time)
+            if parsed is not None:
+                parsed_by_key[str(row_symbol), int(row_timeframe)] = parsed
+        return {
+            (symbol, timeframe): parsed_by_key.get(
+                (symbol, timeframe),
+                fallback_start,
+            )
+            for symbol in symbols
+            for timeframe in timeframes
+        }
+
+    if "symbol" in columns:
+        symbol_placeholders = ", ".join("?" for _ in symbols)
+        rows = conn.execute(
+            f"SELECT symbol, MAX(time) FROM {table}"  # noqa: S608
+            f" WHERE symbol IN ({symbol_placeholders}) GROUP BY symbol",
+            list(symbols),
+        ).fetchall()
+        for row_symbol, max_time in rows:
+            parsed = parse_sqlite_timestamp(max_time)
+            if parsed is not None:
+                parsed_by_key[str(row_symbol), None] = parsed
+        return {
+            (symbol, None): parsed_by_key.get((symbol, None), fallback_start)
+            for symbol in symbols
+        }
+
+    row = conn.execute(f"SELECT MAX(time) FROM {table}").fetchone()  # noqa: S608
+    parsed = parse_sqlite_timestamp(row[0] if row else None)
+    shared_start = parsed if parsed is not None else fallback_start
+    return {(symbol, None): shared_start for symbol in symbols}
+
+
+
+ +
+ +
+ + +

+ parse_sqlite_timestamp + + +

+
parse_sqlite_timestamp(value: object) -> datetime | None
+
+ +
+ +

Parse a SQLite history timestamp value.

+ + +

Returns:

+ + + + + + + + + + + + + +
TypeDescription
+ datetime | None + +
+

Parsed timezone-aware datetime, or None when parsing fails.

+
+
+ + +
+ Source code in mt5cli/sqlite_history.py + 
142
+143
+144
+145
+146
+147
+148
+149
+150
+151
+152
+153
+154
+155
+156
+157
def parse_sqlite_timestamp(value: object) -> datetime | None:
+    """Parse a SQLite history timestamp value.
+
+    Returns:
+        Parsed timezone-aware datetime, or None when parsing fails.
+    """
+    if value is None:
+        return None
+    if isinstance(value, datetime):
+        return value if value.tzinfo is not None else value.replace(tzinfo=UTC)
+    if isinstance(value, int | float):
+        return datetime.fromtimestamp(float(value), tz=UTC)
+    if isinstance(value, str):
+        return _parse_string_sqlite_timestamp(value)
+    logger.warning("Ignoring unsupported history timestamp type: %s", type(value))
+    return None
+
+
+
+ +
+ +
+ + +

+ quote_sqlite_identifier + + +

+
quote_sqlite_identifier(identifier: str) -> str
+
+ +
+ +

Return a safely quoted SQLite identifier using double quotes.

+ + +
+ Source code in mt5cli/sqlite_history.py + 
52
+53
+54
def quote_sqlite_identifier(identifier: str) -> str:
+    """Return a safely quoted SQLite identifier using double quotes."""
+    return '"' + identifier.replace('"', '""') + '"'
+
+
+
+ +
+ +
+ + +

+ record_written_columns + + +

+
record_written_columns(
+    written_columns: dict[Dataset, set[str]],
+    dataset: Dataset,
+    frame: DataFrame,
+) -> None
+
+ +
+ +

Remember columns for datasets written during collection.

+ + +
+ Source code in mt5cli/sqlite_history.py + 
322
+323
+324
+325
+326
+327
+328
+329
+330
+331
+332
def record_written_columns(
+    written_columns: dict[Dataset, set[str]],
+    dataset: Dataset,
+    frame: pd.DataFrame,
+) -> None:
+    """Remember columns for datasets written during collection."""
+    columns = set(frame.columns)
+    if dataset in written_columns:
+        written_columns[dataset].update(columns)
+    else:
+        written_columns[dataset] = columns
+
+
+
+ +
+ +
+ + +

+ resolve_granularity_name + + +

+
resolve_granularity_name(timeframe: int) -> str
+
+ +
+ +

Return a granularity name for a timeframe integer when known.

+ + +
+ Source code in mt5cli/sqlite_history.py + 
 99
+100
+101
+102
+103
+104
def resolve_granularity_name(timeframe: int) -> str:
+    """Return a granularity name for a timeframe integer when known."""
+    for name, value in TIMEFRAME_MAP.items():
+        if value == timeframe:
+            return name
+    return str(timeframe)
+
+
+
+ +
+ +
+ + +

+ resolve_history_datasets + + +

+
resolve_history_datasets(
+    datasets: set[Dataset] | None,
+) -> set[Dataset]
+
+ +
+ +

Resolve configured history datasets.

+ + +

Returns:

+ + + + + + + + + + + + + + + + + +
TypeDescription
+ set[Dataset] + +
+

All supported datasets when datasets is None, otherwise the

+
+
+ set[Dataset] + +
+

configured selection (which may be empty).

+
+
+ + +
+ Source code in mt5cli/sqlite_history.py + 
57
+58
+59
+60
+61
+62
+63
+64
+65
+66
def resolve_history_datasets(datasets: set[Dataset] | None) -> set[Dataset]:
+    """Resolve configured history datasets.
+
+    Returns:
+        All supported datasets when ``datasets`` is None, otherwise the
+        configured selection (which may be empty).
+    """
+    if datasets is None:
+        return set(Dataset)
+    return set(datasets)
+
+
+
+ +
+ +
+ + +

+ resolve_history_tick_flags + + +

+
resolve_history_tick_flags(flags: int | str) -> int
+
+ +
+ +

Resolve tick copy flags from an integer or name.

+ + +

Returns:

+ + + + + + + + + + + + + +
TypeDescription
+ int + +
+

Integer tick flag value.

+
+
+ + +
+ Source code in mt5cli/sqlite_history.py + 
88
+89
+90
+91
+92
+93
+94
+95
+96
def resolve_history_tick_flags(flags: int | str) -> int:
+    """Resolve tick copy flags from an integer or name.
+
+    Returns:
+        Integer tick flag value.
+    """
+    if isinstance(flags, int):
+        return flags
+    return parse_tick_flags(flags)
+
+
+
+ +
+ +
+ + +

+ resolve_history_timeframes + + +

+
resolve_history_timeframes(
+    timeframes: Sequence[int | str] | None,
+) -> list[int]
+
+ +
+ +

Resolve rate timeframes, deduplicating aliases for the same integer.

+ + +

Returns:

+ + + + + + + + + + + + + +
TypeDescription
+ list[int] + +
+

Ordered list of unique timeframe integers.

+
+
+ + +
+ Source code in mt5cli/sqlite_history.py + 
69
+70
+71
+72
+73
+74
+75
+76
+77
+78
+79
+80
+81
+82
+83
+84
+85
def resolve_history_timeframes(
+    timeframes: Sequence[int | str] | None,
+) -> list[int]:
+    """Resolve rate timeframes, deduplicating aliases for the same integer.
+
+    Returns:
+        Ordered list of unique timeframe integers.
+    """
+    raw = timeframes if timeframes is not None else DEFAULT_HISTORY_TIMEFRAMES
+    seen: set[int] = set()
+    resolved: list[int] = []
+    for value in raw:
+        tf = value if isinstance(value, int) else parse_timeframe(str(value))
+        if tf not in seen:
+            seen.add(tf)
+            resolved.append(tf)
+    return resolved
+
+
+
+ +
+ +
+ + +

+ write_collected_datasets + + +

+
write_collected_datasets(
+    conn: Connection,
+    client: Mt5DataClient,
+    symbols: Sequence[str],
+    datasets: set[Dataset],
+    timeframe: int,
+    flags: int,
+    date_from: datetime,
+    date_to: datetime,
+    if_exists: IfExists,
+) -> tuple[set[Dataset], dict[Dataset, set[str]]]
+
+ +
+ +

Collect selected datasets and stream each symbol frame into SQLite.

+ + +

Returns:

+ + + + + + + + + + + + + +
TypeDescription
+ tuple[set[Dataset], dict[Dataset, set[str]]] + +
+

Written datasets and their columns.

+
+
+ + +
+ Source code in mt5cli/sqlite_history.py + 
1112
+1113
+1114
+1115
+1116
+1117
+1118
+1119
+1120
+1121
+1122
+1123
+1124
+1125
+1126
+1127
+1128
+1129
+1130
+1131
+1132
+1133
+1134
+1135
+1136
+1137
+1138
+1139
+1140
+1141
+1142
+1143
+1144
+1145
+1146
+1147
+1148
+1149
+1150
+1151
+1152
+1153
+1154
+1155
+1156
+1157
+1158
+1159
+1160
+1161
+1162
+1163
+1164
+1165
+1166
+1167
+1168
+1169
+1170
+1171
+1172
+1173
+1174
+1175
+1176
def write_collected_datasets(
+    conn: sqlite3.Connection,
+    client: Mt5DataClient,
+    symbols: Sequence[str],
+    datasets: set[Dataset],
+    timeframe: int,
+    flags: int,
+    date_from: datetime,
+    date_to: datetime,
+    if_exists: IfExists,
+) -> tuple[set[Dataset], dict[Dataset, set[str]]]:
+    """Collect selected datasets and stream each symbol frame into SQLite.
+
+    Returns:
+        Written datasets and their columns.
+    """
+    written_columns: dict[Dataset, set[str]] = {}
+    written_tables: set[Dataset] = set()
+    if Dataset.rates in datasets and write_rates_dataset(
+        conn,
+        client,
+        symbols,
+        timeframe,
+        date_from,
+        date_to,
+        if_exists,
+        written_columns,
+    ):
+        written_tables.add(Dataset.rates)
+    if Dataset.ticks in datasets and write_ticks_dataset(
+        conn,
+        client,
+        symbols,
+        flags,
+        date_from,
+        date_to,
+        if_exists,
+        written_columns,
+    ):
+        written_tables.add(Dataset.ticks)
+    if Dataset.history_orders in datasets and write_history_dataset(
+        conn,
+        client.history_orders_get_as_df,
+        Dataset.history_orders,
+        symbols,
+        date_from,
+        date_to,
+        if_exists,
+        written_columns,
+        include_account_events=False,
+    ):
+        written_tables.add(Dataset.history_orders)
+    if Dataset.history_deals in datasets and write_history_dataset(
+        conn,
+        client.history_deals_get_as_df,
+        Dataset.history_deals,
+        symbols,
+        date_from,
+        date_to,
+        if_exists,
+        written_columns,
+        include_account_events=False,
+    ):
+        written_tables.add(Dataset.history_deals)
+    return written_tables, written_columns
+
+
+
+ +
+ +
+ + +

+ write_history_dataset + + +

+
write_history_dataset(
+    conn: Connection,
+    fetch: Callable[..., DataFrame],
+    dataset: Dataset,
+    symbols: Sequence[str],
+    date_from: datetime,
+    date_to: datetime,
+    if_exists: IfExists,
+    written_columns: dict[Dataset, set[str]],
+    *,
+    include_account_events: bool = False,
+) -> bool
+
+ +
+ +

Stream a history dataset into SQLite.

+ + +

Returns:

+ + + + + + + + + + + + + +
TypeDescription
+ bool + +
+

True if the target table was written.

+
+
+ + +
+ Source code in mt5cli/sqlite_history.py + 
733
+734
+735
+736
+737
+738
+739
+740
+741
+742
+743
+744
+745
+746
+747
+748
+749
+750
+751
+752
+753
+754
+755
+756
+757
+758
+759
+760
+761
+762
+763
+764
+765
+766
+767
+768
+769
+770
+771
+772
+773
+774
+775
+776
+777
+778
+779
+780
def write_history_dataset(
+    conn: sqlite3.Connection,
+    fetch: Callable[..., pd.DataFrame],
+    dataset: Dataset,
+    symbols: Sequence[str],
+    date_from: datetime,
+    date_to: datetime,
+    if_exists: IfExists,
+    written_columns: dict[Dataset, set[str]],
+    *,
+    include_account_events: bool = False,
+) -> bool:
+    """Stream a history dataset into SQLite.
+
+    Returns:
+        True if the target table was written.
+    """
+    table_exists = False
+    if include_account_events:
+        frame = filter_trade_history_frame(
+            fetch(date_from=date_from, date_to=date_to),
+            symbols,
+            include_account_events=True,
+        )
+        return write_streamed_frame(
+            conn,
+            frame,
+            dataset,
+            table_exists,
+            if_exists,
+            written_columns,
+        )
+    for sym in symbols:
+        frame = fetch(date_from=date_from, date_to=date_to, symbol=sym)
+        frame = filter_trade_history_frame(
+            frame,
+            [sym],
+            include_account_events=False,
+        )
+        table_exists = write_streamed_frame(
+            conn,
+            frame,
+            dataset,
+            table_exists,
+            if_exists,
+            written_columns,
+        )
+    return table_exists
+
+
+
+ +
+ +
+ + +

+ write_incremental_datasets + + +

+
write_incremental_datasets(
+    conn: Connection,
+    client: Mt5DataClient,
+    symbols: Sequence[str],
+    selected_datasets: set[Dataset],
+    resolved_timeframes: list[int],
+    resolved_tick_flags: int,
+    fallback_start: datetime,
+    end_date: datetime,
+    *,
+    deduplicate: bool,
+    create_rate_views: bool,
+    with_views: bool,
+    include_account_events: bool,
+) -> tuple[set[Dataset], dict[Dataset, set[str]]]
+
+ +
+ +

Append selected datasets incrementally and refresh indexes and views.

+ + +

Returns:

+ + + + + + + + + + + + + +
TypeDescription
+ tuple[set[Dataset], dict[Dataset, set[str]]] + +
+

Written datasets and their columns.

+
+
+ + +
+ Source code in mt5cli/sqlite_history.py + 
1029
+1030
+1031
+1032
+1033
+1034
+1035
+1036
+1037
+1038
+1039
+1040
+1041
+1042
+1043
+1044
+1045
+1046
+1047
+1048
+1049
+1050
+1051
+1052
+1053
+1054
+1055
+1056
+1057
+1058
+1059
+1060
+1061
+1062
+1063
+1064
+1065
+1066
+1067
+1068
+1069
+1070
+1071
+1072
+1073
+1074
+1075
+1076
+1077
+1078
+1079
+1080
+1081
+1082
+1083
+1084
+1085
+1086
+1087
+1088
+1089
+1090
+1091
+1092
+1093
+1094
+1095
+1096
+1097
+1098
+1099
+1100
+1101
+1102
+1103
+1104
+1105
+1106
+1107
+1108
+1109
def write_incremental_datasets(  # noqa: PLR0913
+    conn: sqlite3.Connection,
+    client: Mt5DataClient,
+    symbols: Sequence[str],
+    selected_datasets: set[Dataset],
+    resolved_timeframes: list[int],
+    resolved_tick_flags: int,
+    fallback_start: datetime,
+    end_date: datetime,
+    *,
+    deduplicate: bool,
+    create_rate_views: bool,
+    with_views: bool,
+    include_account_events: bool,
+) -> tuple[set[Dataset], dict[Dataset, set[str]]]:
+    """Append selected datasets incrementally and refresh indexes and views.
+
+    Returns:
+        Written datasets and their columns.
+    """
+    written_columns: dict[Dataset, set[str]] = {}
+    written_tables: set[Dataset] = set()
+    dedup_scopes: dict[Dataset, list[DedupScope]] = {}
+    if Dataset.rates in selected_datasets:
+        _write_incremental_rates(
+            conn,
+            client,
+            symbols,
+            resolved_timeframes,
+            fallback_start,
+            end_date,
+            written_columns,
+            written_tables,
+            dedup_scopes,
+        )
+    if Dataset.ticks in selected_datasets:
+        _write_incremental_ticks(
+            conn,
+            client,
+            symbols,
+            resolved_tick_flags,
+            fallback_start,
+            end_date,
+            written_columns,
+            written_tables,
+            dedup_scopes,
+        )
+    if Dataset.history_orders in selected_datasets:
+        _write_incremental_history_orders(
+            conn,
+            client,
+            symbols,
+            fallback_start,
+            end_date,
+            written_columns,
+            written_tables,
+            dedup_scopes,
+        )
+    if Dataset.history_deals in selected_datasets:
+        _write_incremental_history_deals(
+            conn,
+            client,
+            symbols,
+            fallback_start,
+            end_date,
+            written_columns,
+            written_tables,
+            dedup_scopes,
+            include_account_events=include_account_events,
+        )
+    _finalize_incremental_writes(
+        conn,
+        selected_datasets,
+        written_columns,
+        written_tables,
+        dedup_scopes,
+        deduplicate=deduplicate,
+        create_rate_views=create_rate_views,
+        with_views=with_views,
+    )
+    return written_tables, written_columns
+
+
+
+ +
+ +
+ + +

+ write_rates_dataset + + +

+
write_rates_dataset(
+    conn: Connection,
+    client: Mt5DataClient,
+    symbols: Sequence[str],
+    timeframe: int,
+    date_from: datetime,
+    date_to: datetime,
+    if_exists: IfExists,
+    written_columns: dict[Dataset, set[str]],
+) -> bool
+
+ +
+ +

Stream rates frames into SQLite.

+ + +

Returns:

+ + + + + + + + + + + + + +
TypeDescription
+ bool + +
+

True if the rates table was written.

+
+
+ + +
+ Source code in mt5cli/sqlite_history.py + 
660
+661
+662
+663
+664
+665
+666
+667
+668
+669
+670
+671
+672
+673
+674
+675
+676
+677
+678
+679
+680
+681
+682
+683
+684
+685
+686
+687
+688
+689
+690
+691
+692
+693
+694
def write_rates_dataset(
+    conn: sqlite3.Connection,
+    client: Mt5DataClient,
+    symbols: Sequence[str],
+    timeframe: int,
+    date_from: datetime,
+    date_to: datetime,
+    if_exists: IfExists,
+    written_columns: dict[Dataset, set[str]],
+) -> bool:
+    """Stream rates frames into SQLite.
+
+    Returns:
+        True if the rates table was written.
+    """
+    table_exists = False
+    for sym in symbols:
+        frame = client.copy_rates_range_as_df(
+            symbol=sym,
+            timeframe=timeframe,
+            date_from=date_from,
+            date_to=date_to,
+        ).drop(columns=["symbol", "timeframe"], errors="ignore")
+        if len(frame.columns) != 0:
+            frame.insert(0, "symbol", sym)
+            frame.insert(1, "timeframe", timeframe)
+        table_exists = write_streamed_frame(
+            conn,
+            frame,
+            Dataset.rates,
+            table_exists,
+            if_exists,
+            written_columns,
+        )
+    return table_exists
+
+
+
+ +
+ +
+ + +

+ write_streamed_frame + + +

+
write_streamed_frame(
+    conn: Connection,
+    frame: DataFrame,
+    dataset: Dataset,
+    table_exists: bool,
+    if_exists: IfExists,
+    written_columns: dict[Dataset, set[str]],
+) -> bool
+
+ +
+ +

Write one streamed dataset frame and track table state.

+ + +

Returns:

+ + + + + + + + + + + + + +
TypeDescription
+ bool + +
+

True if the dataset table exists after this write attempt.

+
+
+ + +
+ Source code in mt5cli/sqlite_history.py + 
351
+352
+353
+354
+355
+356
+357
+358
+359
+360
+361
+362
+363
+364
+365
+366
+367
+368
def write_streamed_frame(
+    conn: sqlite3.Connection,
+    frame: pd.DataFrame,
+    dataset: Dataset,
+    table_exists: bool,
+    if_exists: IfExists,
+    written_columns: dict[Dataset, set[str]],
+) -> bool:
+    """Write one streamed dataset frame and track table state.
+
+    Returns:
+        True if the dataset table exists after this write attempt.
+    """
+    write_mode = IfExists.APPEND if table_exists else if_exists
+    if append_dataframe(conn, frame, dataset.table_name, write_mode):
+        record_written_columns(written_columns, dataset, frame)
+        return True
+    return table_exists
+
+
+
+ +
+ +
+ + +

+ write_ticks_dataset + + +

+
write_ticks_dataset(
+    conn: Connection,
+    client: Mt5DataClient,
+    symbols: Sequence[str],
+    flags: int,
+    date_from: datetime,
+    date_to: datetime,
+    if_exists: IfExists,
+    written_columns: dict[Dataset, set[str]],
+) -> bool
+
+ +
+ +

Stream ticks frames into SQLite.

+ + +

Returns:

+ + + + + + + + + + + + + +
TypeDescription
+ bool + +
+

True if the ticks table was written.

+
+
+ + +
+ Source code in mt5cli/sqlite_history.py + 
697
+698
+699
+700
+701
+702
+703
+704
+705
+706
+707
+708
+709
+710
+711
+712
+713
+714
+715
+716
+717
+718
+719
+720
+721
+722
+723
+724
+725
+726
+727
+728
+729
+730
def write_ticks_dataset(
+    conn: sqlite3.Connection,
+    client: Mt5DataClient,
+    symbols: Sequence[str],
+    flags: int,
+    date_from: datetime,
+    date_to: datetime,
+    if_exists: IfExists,
+    written_columns: dict[Dataset, set[str]],
+) -> bool:
+    """Stream ticks frames into SQLite.
+
+    Returns:
+        True if the ticks table was written.
+    """
+    table_exists = False
+    for sym in symbols:
+        frame = client.copy_ticks_range_as_df(
+            symbol=sym,
+            date_from=date_from,
+            date_to=date_to,
+            flags=flags,
+        ).drop(columns=["symbol"], errors="ignore")
+        if len(frame.columns) != 0:
+            frame.insert(0, "symbol", sym)
+        table_exists = write_streamed_frame(
+            conn,
+            frame,
+            Dataset.ticks,
+            table_exists,
+            if_exists,
+            written_columns,
+        )
+    return table_exists
+
+
+
+ +
+ + + +
+ +
+ +

collect-history schema

+

The collect-history command (and the matching collect_history SDK function) writes +selected MT5 datasets into one SQLite database. Each dataset becomes a table; column +names and types mirror the pdmt5 DataFrame schema for that export, with two additions:

+
    +
  • symbol is prepended on every table.
    • +
  • timeframe is prepended on rates so appended runs at different bar sizes stay + distinguishable.
    • +
+

SQLite does not declare foreign keys. Rows are linked logically by symbol, time +windows, and (for deals) position_id / order. Duplicate rows are removed on +append using dataset-specific keys (for example ticket on history tables, or +(symbol, timeframe, time) on rates).

+

Optional views are created when --with-views is set and the history-deals dataset +was written.

+

Entity-relationship diagram

+

Sample layout for a full collection with --with-views:

+
erDiagram
+    rates {
+        TEXT symbol "dedup key"
+        INTEGER timeframe "dedup key"
+        TEXT time "dedup key"
+        REAL open
+        REAL high
+        REAL low
+        REAL close
+        INTEGER tick_volume
+        INTEGER spread
+        INTEGER real_volume
+    }
+
+    ticks {
+        TEXT symbol "dedup key"
+        TEXT time "dedup key"
+        INTEGER time_msc "dedup key (preferred)"
+        REAL bid
+        REAL ask
+        REAL last
+        INTEGER volume
+        INTEGER flags
+        REAL volume_real
+    }
+
+    history_orders {
+        INTEGER ticket "dedup key"
+        TEXT symbol
+        TEXT time
+        INTEGER type
+        INTEGER state
+        REAL volume_initial
+        REAL price_open
+        REAL price_current
+        INTEGER magic
+    }
+
+    history_deals {
+        INTEGER ticket "dedup key"
+        INTEGER order
+        INTEGER position_id "groups position view"
+        TEXT symbol
+        TEXT time
+        INTEGER type "0/1 trade, else cash event"
+        INTEGER entry "0 IN, 1 OUT, 2 INOUT, 3 OUT_BY"
+        REAL volume
+        REAL price
+        REAL profit
+        REAL commission
+        REAL swap
+        REAL fee
+    }
+
+    cash_events {
+        INTEGER ticket
+        TEXT symbol
+        TEXT time
+        INTEGER type
+        REAL profit
+    }
+
+    positions_reconstructed {
+        INTEGER position_id
+        TEXT symbol
+        TEXT open_time
+        TEXT close_time
+        INTEGER direction
+        REAL volume_open
+        REAL volume_close
+        REAL volume_reversal
+        REAL open_price
+        REAL close_price
+        REAL total_profit
+        INTEGER reversal_count
+        INTEGER deals_count
+    }
+
+    rates ||--o{ history_deals : "symbol (logical)"
+    ticks ||--o{ history_deals : "symbol (logical)"
+    history_orders ||--o{ history_deals : "order ~ ticket (logical)"
+    history_deals ||--|| cash_events : "VIEW: type NOT IN (0,1)"
+    history_deals ||--o{ positions_reconstructed : "VIEW: GROUP BY position_id"
+
+

Tables and views

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ObjectKindSourceNotes
ratestablecopy_rates_rangeIndexed on (symbol, timeframe, time) when columns exist.
tickstablecopy_ticks_rangeIndexed on (symbol, time) when columns exist.
history_orderstablehistory_orders_getFetched per --symbol, then concatenated.
history_dealstablehistory_deals_getFetched per --symbol, then concatenated. Indexed on (position_id, symbol) when present.
cash_eventsviewhistory_dealsNon-trade deal types (deposits, balance ops, etc.). Requires type column.
positions_reconstructedviewhistory_dealsOne row per closed position_id; volume-weighted prices and reversal stats.
+

Column sets can vary with terminal and pdmt5 version. Views are skipped with a warning +when required columns are missing.

+

Incremental collection

+

The update_history SDK path uses the same base tables and optional +cash_events / positions_reconstructed views. It additionally maintains +rate_<symbol>__<timeframe> compatibility views when create_rate_views=True.