| def main() -> None:
- """Run the mt5cli CLI."""
- app()
+ | def main() -> None:
+ """Run the mt5cli CLI."""
+ app()
|
+
+
+
+
+ DEFAULT_HISTORY_DATASETS
+
+
+
+ module-attribute
+
+
+
+
+
+
+
+
+
+
@@ -613,13 +657,13 @@ by an explicit table (for example a custom SQLite view).
Source code in mt5cli/history.py
- | def __post_init__(self) -> None:
- """Normalize accepted timeframe aliases to the stored integer value."""
- if not isinstance(self.timeframe, int):
- object.__setattr__(self, "timeframe", parse_timeframe(self.timeframe))
+ | def __post_init__(self) -> None:
+ """Normalize accepted timeframe aliases to the stored integer value."""
+ if not isinstance(self.timeframe, int):
+ object.__setattr__(self, "timeframe", parse_timeframe(self.timeframe))
|
Source code in mt5cli/history.py
- 1012
-1013
-1014
-1015
-1016
-1017
-1018
+ | def append_dataframe(
- conn: sqlite3.Connection,
- frame: pd.DataFrame,
- table_name: str,
- if_exists: IfExists,
-) -> bool:
- """Append a DataFrame to SQLite when it has a schema.
-
- Returns:
- True if a table was written, False if the frame had no columns.
- """
- if len(frame.columns) == 0:
- logger.warning("Skipping %s: dataset returned no columns", table_name)
- return False
- frame.to_sql( # type: ignore[reportUnknownMemberType]
- table_name,
- conn,
- if_exists=if_exists.value,
- index=False,
- chunksize=50_000,
- )
- return True
+1033
+1034
+1035
+1036
+1037
+1038
+1039
| def append_dataframe(
+ conn: sqlite3.Connection,
+ frame: pd.DataFrame,
+ table_name: str,
+ if_exists: IfExists,
+) -> bool:
+ """Append a DataFrame to SQLite when it has a schema.
+
+ Returns:
+ True if a table was written, False if the frame had no columns.
+ """
+ if len(frame.columns) == 0:
+ logger.warning("Skipping %s: dataset returned no columns", table_name)
+ return False
+ frame.to_sql( # type: ignore[reportUnknownMemberType]
+ table_name,
+ conn,
+ if_exists=if_exists.value,
+ index=False,
+ chunksize=50_000,
+ )
+ return True
|
Source code in mt5cli/history.py
- 1049
-1050
-1051
-1052
-1053
-1054
-1055
+ | def augment_written_columns_from_sqlite(
- conn: sqlite3.Connection,
- datasets: set[Dataset],
- written_columns: dict[Dataset, set[str]],
-) -> None:
- """Add existing table columns to the written column map."""
- for dataset in datasets:
- columns = get_table_columns(conn, dataset.table_name)
- if not columns:
- continue
- if dataset in written_columns:
- written_columns[dataset].update(columns)
- else:
- written_columns[dataset] = columns
+1062
+1063
+1064
+1065
+1066
+1067
+1068
| def augment_written_columns_from_sqlite(
+ conn: sqlite3.Connection,
+ datasets: set[Dataset],
+ written_columns: dict[Dataset, set[str]],
+) -> None:
+ """Add existing table columns to the written column map."""
+ for dataset in datasets:
+ columns = get_table_columns(conn, dataset.table_name)
+ if not columns:
+ continue
+ if dataset in written_columns:
+ written_columns[dataset].update(columns)
+ else:
+ written_columns[dataset] = columns
|
symbol=None for each timeframe instead of raising.
Source code in mt5cli/history.py
- 555
-556
-557
-558
-559
-560
-561
+ | def build_rate_targets(
- symbols: Sequence[str],
- timeframes: Sequence[int | str],
- *,
- allow_missing_symbol: bool = False,
-) -> list[RateTarget]:
- """Build rate targets for every symbol and timeframe combination.
-
- Args:
- symbols: MT5 symbol names. May be empty when ``allow_missing_symbol``.
- timeframes: MT5 timeframes as integers or names (for example ``M1``).
- allow_missing_symbol: When True and ``symbols`` is empty, build targets
- with ``symbol=None`` for each timeframe instead of raising.
+589
+590
+591
+592
+593
+594
+595
| def build_rate_targets(
+ symbols: Sequence[str],
+ timeframes: Sequence[int | str],
+ *,
+ allow_missing_symbol: bool = False,
+) -> list[RateTarget]:
+ """Build rate targets for every symbol and timeframe combination.
- Returns:
- Targets in row-major order: every timeframe for the first symbol, then
- every timeframe for the next symbol, and so on.
-
- Raises:
- ValueError: If ``timeframes`` is empty, or ``symbols`` is empty and
- ``allow_missing_symbol`` is False.
- """
- if not timeframes:
- msg = "At least one timeframe is required."
- raise ValueError(msg)
- if not symbols:
- if not allow_missing_symbol:
- msg = "At least one symbol is required."
- raise ValueError(msg)
- return [RateTarget(symbol=None, timeframe=tf) for tf in timeframes]
- return [
- RateTarget(symbol=symbol, timeframe=tf)
- for symbol in symbols
- for tf in timeframes
- ]
+ Args:
+ symbols: MT5 symbol names. May be empty when ``allow_missing_symbol``.
+ timeframes: MT5 timeframes as integers or names (for example ``M1``).
+ allow_missing_symbol: When True and ``symbols`` is empty, build targets
+ with ``symbol=None`` for each timeframe instead of raising.
+
+ Returns:
+ Targets in row-major order: every timeframe for the first symbol, then
+ every timeframe for the next symbol, and so on.
+
+ Raises:
+ ValueError: If ``timeframes`` is empty, or ``symbols`` is empty and
+ ``allow_missing_symbol`` is False.
+ """
+ if not timeframes:
+ msg = "At least one timeframe is required."
+ raise ValueError(msg)
+ if not symbols:
+ if not allow_missing_symbol:
+ msg = "At least one symbol is required."
+ raise ValueError(msg)
+ return [RateTarget(symbol=None, timeframe=tf) for tf in timeframes]
+ return [
+ RateTarget(symbol=symbol, timeframe=tf)
+ for symbol in symbols
+ for tf in timeframes
+ ]
|
EURUSD_M1 cannot collide with EURUSD
Source code in mt5cli/history.py
- 127
-128
-129
-130
-131
-132
-133
+ | def build_rate_view_name(
- *,
- symbol: str,
- granularity: str,
- granularity_count: int,
- timeframe: int,
-) -> str:
- """Return a collision-free offline optimize view name.
-
- View names always include the timeframe integer after a ``__`` separator so
- a symbol such as ``EURUSD_M1`` cannot collide with ``EURUSD`` at timeframe
- ``M1``.
- """
- if granularity_count == 1:
- return f"rate_{symbol}__{timeframe}"
- return f"rate_{symbol}__{granularity}_{timeframe}"
+142
+143
+144
+145
+146
+147
+148
| def build_rate_view_name(
+ *,
+ symbol: str,
+ granularity: str,
+ granularity_count: int,
+ timeframe: int,
+) -> str:
+ """Return a collision-free offline optimize view name.
+
+ View names always include the timeframe integer after a ``__`` separator so
+ a symbol such as ``EURUSD_M1`` cannot collide with ``EURUSD`` at timeframe
+ ``M1``.
+ """
+ if granularity_count == 1:
+ return f"rate_{symbol}__{timeframe}"
+ return f"rate_{symbol}__{granularity}_{timeframe}"
|
EURUSD_M1 cannot collide with EURUSD
Source code in mt5cli/history.py
- 1286
-1287
-1288
-1289
-1290
-1291
-1292
+ | def create_cash_events_view(
- conn: sqlite3.Connection,
- deals_columns: set[str],
-) -> bool:
- """Create the cash_events SQLite view derived from history_deals.
-
- Returns:
- True if the view was created, False if required columns are missing.
- """
- if "type" not in deals_columns:
- logger.warning("Skipping cash_events view: history_deals.type is missing")
- return False
- conn.execute("DROP VIEW IF EXISTS cash_events")
- conn.execute(
- "CREATE VIEW cash_events AS" # noqa: S608
- f" SELECT * FROM history_deals WHERE type NOT IN {_TRADE_DEAL_TYPES_SQL}",
- )
- return True
+1303
+1304
+1305
+1306
+1307
+1308
+1309
| def create_cash_events_view(
+ conn: sqlite3.Connection,
+ deals_columns: set[str],
+) -> bool:
+ """Create the cash_events SQLite view derived from history_deals.
+
+ Returns:
+ True if the view was created, False if required columns are missing.
+ """
+ if "type" not in deals_columns:
+ logger.warning("Skipping cash_events view: history_deals.type is missing")
+ return False
+ conn.execute("DROP VIEW IF EXISTS cash_events")
+ conn.execute(
+ "CREATE VIEW cash_events AS" # noqa: S608
+ f" SELECT * FROM history_deals WHERE type NOT IN {_TRADE_DEAL_TYPES_SQL}",
+ )
+ return True
|
EURUSD_M1 cannot collide with EURUSD
Source code in mt5cli/history.py
- 1197
-1198
-1199
-1200
-1201
-1202
-1203
+ | def create_history_indexes(
- conn: sqlite3.Connection,
- written_columns: dict[Dataset, set[str]],
-) -> None:
- """Create useful indexes for collected history tables when present."""
- if {"symbol", "timeframe", "time"}.issubset(
- written_columns.get(Dataset.rates, set()),
- ):
- conn.execute(
- "CREATE INDEX IF NOT EXISTS idx_rates_symbol_timeframe_time"
- " ON rates(symbol, timeframe, time)",
- )
- if {"symbol", "time"}.issubset(written_columns.get(Dataset.ticks, set())):
- conn.execute(
- "CREATE INDEX IF NOT EXISTS idx_ticks_symbol_time ON ticks(symbol, time)",
- )
- if {"position_id", "symbol"}.issubset(
- written_columns.get(Dataset.history_deals, set()),
- ):
+1219
+1220
+1221
+1222
+1223
+1224
+1225
| def create_history_indexes(
+ conn: sqlite3.Connection,
+ written_columns: dict[Dataset, set[str]],
+) -> None:
+ """Create useful indexes for collected history tables when present."""
+ if {"symbol", "timeframe", "time"}.issubset(
+ written_columns.get(Dataset.rates, set()),
+ ):
+ conn.execute(
+ "CREATE INDEX IF NOT EXISTS idx_rates_symbol_timeframe_time"
+ " ON rates(symbol, timeframe, time)",
+ )
+ if {"symbol", "time"}.issubset(written_columns.get(Dataset.ticks, set())):
conn.execute(
- "CREATE INDEX IF NOT EXISTS idx_history_deals_position_symbol"
- " ON history_deals(position_id, symbol)",
- )
+ "CREATE INDEX IF NOT EXISTS idx_ticks_symbol_time ON ticks(symbol, time)",
+ )
+ if {"position_id", "symbol"}.issubset(
+ written_columns.get(Dataset.history_deals, set()),
+ ):
+ conn.execute(
+ "CREATE INDEX IF NOT EXISTS idx_history_deals_position_symbol"
+ " ON history_deals(position_id, symbol)",
+ )
|
EURUSD_M1 cannot collide with EURUSD
Source code in mt5cli/history.py
- 1306
-1307
-1308
-1309
-1310
-1311
-1312
+ | def create_positions_reconstructed_view(
- conn: sqlite3.Connection,
- deals_columns: set[str],
-) -> bool:
- """Create the positions_reconstructed SQLite view derived from history_deals.
-
- Returns:
- True if the view was created, False if required columns are missing.
- """
- if not _POSITIONS_VIEW_REQUIRED_COLUMNS.issubset(deals_columns):
- missing = ", ".join(sorted(_POSITIONS_VIEW_REQUIRED_COLUMNS - deals_columns))
- logger.warning(
- "Skipping positions_reconstructed view: history_deals missing columns: %s",
- missing,
- )
- return False
- conn.execute("DROP VIEW IF EXISTS positions_reconstructed")
- conn.execute(
- "CREATE VIEW positions_reconstructed AS" # noqa: S608
- " SELECT"
- " position_id,"
- " symbol,"
- " MIN(CASE WHEN entry = 0 THEN time END) AS open_time,"
- " MAX(CASE WHEN entry IN (1, 2, 3) THEN time END) AS close_time,"
- " MIN(CASE WHEN entry = 0 THEN type END) AS direction,"
- " SUM(CASE WHEN entry = 0 THEN volume ELSE 0 END) AS volume_open,"
- " SUM(CASE WHEN entry IN (1, 2, 3) THEN volume ELSE 0 END) AS volume_close,"
- " SUM(CASE WHEN entry = 2 THEN volume ELSE 0 END) AS volume_reversal,"
- " CASE"
- " WHEN SUM(CASE WHEN entry = 0 THEN volume ELSE 0 END) > 0"
- " THEN SUM(CASE WHEN entry = 0 THEN price * volume ELSE 0 END)"
- " / SUM(CASE WHEN entry = 0 THEN volume ELSE 0 END)"
- " END AS open_price,"
- " CASE"
- " WHEN SUM(CASE WHEN entry IN (1, 2, 3) THEN volume ELSE 0 END) > 0"
- " THEN SUM(CASE WHEN entry IN (1, 2, 3) THEN price * volume ELSE 0 END)"
- " / SUM(CASE WHEN entry IN (1, 2, 3) THEN volume ELSE 0 END)"
- " END AS close_price,"
- " SUM(profit) AS total_profit,"
- " SUM(CASE WHEN entry = 2 THEN 1 ELSE 0 END) AS reversal_count,"
- " COUNT(*) AS deals_count"
- " FROM history_deals"
- f" WHERE type IN {_TRADE_DEAL_TYPES_SQL} AND position_id != 0"
- " GROUP BY position_id, symbol"
- " HAVING SUM(CASE WHEN entry IN (1, 2, 3) THEN 1 ELSE 0 END) > 0",
- )
- return True
+1352
+1353
+1354
+1355
+1356
+1357
+1358
| def create_positions_reconstructed_view(
+ conn: sqlite3.Connection,
+ deals_columns: set[str],
+) -> bool:
+ """Create the positions_reconstructed SQLite view derived from history_deals.
+
+ Returns:
+ True if the view was created, False if required columns are missing.
+ """
+ if not _POSITIONS_VIEW_REQUIRED_COLUMNS.issubset(deals_columns):
+ missing = ", ".join(sorted(_POSITIONS_VIEW_REQUIRED_COLUMNS - deals_columns))
+ logger.warning(
+ "Skipping positions_reconstructed view: history_deals missing columns: %s",
+ missing,
+ )
+ return False
+ conn.execute("DROP VIEW IF EXISTS positions_reconstructed")
+ conn.execute(
+ "CREATE VIEW positions_reconstructed AS" # noqa: S608
+ " SELECT"
+ " position_id,"
+ " symbol,"
+ " MIN(CASE WHEN entry = 0 THEN time END) AS open_time,"
+ " MAX(CASE WHEN entry IN (1, 2, 3) THEN time END) AS close_time,"
+ " MIN(CASE WHEN entry = 0 THEN type END) AS direction,"
+ " SUM(CASE WHEN entry = 0 THEN volume ELSE 0 END) AS volume_open,"
+ " SUM(CASE WHEN entry IN (1, 2, 3) THEN volume ELSE 0 END) AS volume_close,"
+ " SUM(CASE WHEN entry = 2 THEN volume ELSE 0 END) AS volume_reversal,"
+ " CASE"
+ " WHEN SUM(CASE WHEN entry = 0 THEN volume ELSE 0 END) > 0"
+ " THEN SUM(CASE WHEN entry = 0 THEN price * volume ELSE 0 END)"
+ " / SUM(CASE WHEN entry = 0 THEN volume ELSE 0 END)"
+ " END AS open_price,"
+ " CASE"
+ " WHEN SUM(CASE WHEN entry IN (1, 2, 3) THEN volume ELSE 0 END) > 0"
+ " THEN SUM(CASE WHEN entry IN (1, 2, 3) THEN price * volume ELSE 0 END)"
+ " / SUM(CASE WHEN entry IN (1, 2, 3) THEN volume ELSE 0 END)"
+ " END AS close_price,"
+ " SUM(profit) AS total_profit,"
+ " SUM(CASE WHEN entry = 2 THEN 1 ELSE 0 END) AS reversal_count,"
+ " COUNT(*) AS deals_count"
+ " FROM history_deals"
+ f" WHERE type IN {_TRADE_DEAL_TYPES_SQL} AND position_id != 0"
+ " GROUP BY position_id, symbol"
+ " HAVING SUM(CASE WHEN entry IN (1, 2, 3) THEN 1 ELSE 0 END) > 0",
+ )
+ return True
|
EURUSD_M1 cannot collide with EURUSD
Source code in mt5cli/history.py
- 1365
-1366
-1367
-1368
-1369
-1370
-1371
+ | def create_rate_compatibility_views(conn: sqlite3.Connection) -> None:
- """Create rate compatibility views from the normalized rates table."""
- columns = get_table_columns(conn, Dataset.rates.table_name)
- if not {"symbol", "timeframe", "time"}.issubset(columns):
- return
- drop_rate_compatibility_views(conn)
- select_columns = sorted(columns - {"symbol", "timeframe"})
- quoted_columns = ", ".join(f'"{column}"' for column in select_columns)
- rows = conn.execute(
- "SELECT DISTINCT symbol, timeframe FROM rates ORDER BY symbol, timeframe",
- ).fetchall()
- timeframes_by_symbol: dict[str, list[int]] = {}
- for symbol, timeframe in rows:
- timeframes_by_symbol.setdefault(str(symbol), []).append(int(timeframe))
- for symbol, timeframes in timeframes_by_symbol.items():
- for timeframe in timeframes:
- granularity = resolve_granularity_name(timeframe)
- view_name = build_rate_view_name(
- symbol=symbol,
- granularity=granularity,
- granularity_count=len(timeframes),
- timeframe=timeframe,
- )
- quoted_view_name = quote_sqlite_identifier(view_name)
- escaped_symbol = symbol.replace("'", "''")
- conn.execute(
- f"CREATE VIEW {quoted_view_name} AS" # noqa: S608
- f" SELECT {quoted_columns} FROM rates"
- f" WHERE symbol = '{escaped_symbol}'"
- f" AND timeframe = {timeframe}",
- )
+1395
+1396
+1397
+1398
+1399
+1400
+1401
| def create_rate_compatibility_views(conn: sqlite3.Connection) -> None:
+ """Create rate compatibility views from the normalized rates table."""
+ columns = get_table_columns(conn, Dataset.rates.table_name)
+ if not {"symbol", "timeframe", "time"}.issubset(columns):
+ return
+ drop_rate_compatibility_views(conn)
+ select_columns = sorted(columns - {"symbol", "timeframe"})
+ quoted_columns = ", ".join(f'"{column}"' for column in select_columns)
+ rows = conn.execute(
+ "SELECT DISTINCT symbol, timeframe FROM rates ORDER BY symbol, timeframe",
+ ).fetchall()
+ timeframes_by_symbol: dict[str, list[int]] = {}
+ for symbol, timeframe in rows:
+ timeframes_by_symbol.setdefault(str(symbol), []).append(int(timeframe))
+ for symbol, timeframes in timeframes_by_symbol.items():
+ for timeframe in timeframes:
+ granularity = resolve_granularity_name(timeframe)
+ view_name = build_rate_view_name(
+ symbol=symbol,
+ granularity=granularity,
+ granularity_count=len(timeframes),
+ timeframe=timeframe,
+ )
+ quoted_view_name = quote_sqlite_identifier(view_name)
+ escaped_symbol = symbol.replace("'", "''")
+ conn.execute(
+ f"CREATE VIEW {quoted_view_name} AS" # noqa: S608
+ f" SELECT {quoted_columns} FROM rates"
+ f" WHERE symbol = '{escaped_symbol}'"
+ f" AND timeframe = {timeframe}",
+ )
|
Source code in mt5cli/history.py
- 1149
-1150
-1151
-1152
-1153
-1154
-1155
+ | def deduplicate_history_tables(
- conn: sqlite3.Connection,
- written_columns: dict[Dataset, set[str]],
- written_tables: set[Dataset],
- dedup_scopes: Mapping[Dataset, Sequence[DedupScope]] | None = None,
-) -> None:
- """Deduplicate appended history tables by stable identifiers.
-
- Scopes whose required columns are not present in the written table are
- skipped. If all scopes for a dataset are skipped, the table receives one
- unscoped deduplication pass instead.
- """
- cursor = conn.cursor()
- for dataset in written_tables:
- columns = written_columns.get(dataset, set())
- table = dataset.table_name
- keys = next(
- (
- candidate
- for candidate in _HISTORY_DEDUP_KEYS[dataset]
- if set(candidate).issubset(columns)
- ),
- None,
- )
- if keys is None:
- logger.warning(
- "Skipping %s deduplication: no supported key columns",
- table,
- )
- continue
- raw_scopes: Sequence[DedupScope] = (
- dedup_scopes.get(dataset, ()) if dedup_scopes else ()
- )
- scopes = [scope for scope in raw_scopes if scope.required_columns <= columns]
- if scopes:
- for scope in scopes:
- drop_duplicates_in_table(
- cursor,
- table,
- list(keys),
- keep="last",
- scope_where=scope.where,
- scope_params=scope.params,
- )
- continue
- drop_duplicates_in_table(cursor, table, list(keys), keep="last")
+1194
+1195
+1196
+1197
+1198
+1199
+1200
| def deduplicate_history_tables(
+ conn: sqlite3.Connection,
+ written_columns: dict[Dataset, set[str]],
+ written_tables: set[Dataset],
+ dedup_scopes: Mapping[Dataset, Sequence[DedupScope]] | None = None,
+) -> None:
+ """Deduplicate appended history tables by stable identifiers.
+
+ Scopes whose required columns are not present in the written table are
+ skipped. If all scopes for a dataset are skipped, the table receives one
+ unscoped deduplication pass instead.
+ """
+ cursor = conn.cursor()
+ for dataset in written_tables:
+ columns = written_columns.get(dataset, set())
+ table = dataset.table_name
+ keys = next(
+ (
+ candidate
+ for candidate in _HISTORY_DEDUP_KEYS[dataset]
+ if set(candidate).issubset(columns)
+ ),
+ None,
+ )
+ if keys is None:
+ logger.warning(
+ "Skipping %s deduplication: no supported key columns",
+ table,
+ )
+ continue
+ raw_scopes: Sequence[DedupScope] = (
+ dedup_scopes.get(dataset, ()) if dedup_scopes else ()
+ )
+ scopes = [scope for scope in raw_scopes if scope.required_columns <= columns]
+ if scopes:
+ for scope in scopes:
+ drop_duplicates_in_table(
+ cursor,
+ table,
+ list(keys),
+ keep="last",
+ scope_where=scope.where,
+ scope_params=scope.params,
+ )
+ continue
+ drop_duplicates_in_table(cursor, table, list(keys), keep="last")
|
Source code in mt5cli/history.py
- 1085
-1086
-1087
-1088
-1089
-1090
-1091
+ | def drop_duplicates_in_table(
- cursor: sqlite3.Cursor,
- table: str,
- ids: list[str],
- *,
- keep: Literal["first", "last"] = "last",
- scope_where: str | None = None,
- scope_params: tuple[object, ...] = (),
-) -> None:
- """Remove duplicate rows, keeping the first or last ROWID per key group.
-
- Raises:
- ValueError: If the table or column names are invalid.
- """
- if not table.isidentifier():
- msg = f"Invalid table name: {table}"
- raise ValueError(msg)
- if invalid := {column for column in ids if not column.isidentifier()}:
- msg = f"Invalid column names: {', '.join(sorted(invalid))}"
- raise ValueError(msg)
- ids_csv = ", ".join(f'"{column}"' for column in ids)
- rowid_selector = "MIN" if keep == "first" else "MAX"
- if scope_where:
- delete_sql = (
- f"DELETE FROM {table} WHERE {scope_where} AND ROWID NOT IN" # noqa: S608
- f" (SELECT {rowid_selector}(ROWID) FROM {table} WHERE {scope_where}"
- f" GROUP BY {ids_csv})"
- )
- cursor.execute(delete_sql, scope_params + scope_params)
- return
- cursor.execute(
- f"DELETE FROM {table} WHERE ROWID NOT IN" # noqa: S608
- f" (SELECT {rowid_selector}(ROWID) FROM {table} GROUP BY {ids_csv})",
- )
+1118
+1119
+1120
+1121
+1122
+1123
+1124
| def drop_duplicates_in_table(
+ cursor: sqlite3.Cursor,
+ table: str,
+ ids: list[str],
+ *,
+ keep: Literal["first", "last"] = "last",
+ scope_where: str | None = None,
+ scope_params: tuple[object, ...] = (),
+) -> None:
+ """Remove duplicate rows, keeping the first or last ROWID per key group.
+
+ Raises:
+ ValueError: If the table or column names are invalid.
+ """
+ if not table.isidentifier():
+ msg = f"Invalid table name: {table}"
+ raise ValueError(msg)
+ if invalid := {column for column in ids if not column.isidentifier()}:
+ msg = f"Invalid column names: {', '.join(sorted(invalid))}"
+ raise ValueError(msg)
+ ids_csv = ", ".join(f'"{column}"' for column in ids)
+ rowid_selector = "MIN" if keep == "first" else "MAX"
+ if scope_where:
+ delete_sql = (
+ f"DELETE FROM {table} WHERE {scope_where} AND ROWID NOT IN" # noqa: S608
+ f" (SELECT {rowid_selector}(ROWID) FROM {table} WHERE {scope_where}"
+ f" GROUP BY {ids_csv})"
+ )
+ cursor.execute(delete_sql, scope_params + scope_params)
+ return
+ cursor.execute(
+ f"DELETE FROM {table} WHERE ROWID NOT IN" # noqa: S608
+ f" (SELECT {rowid_selector}(ROWID) FROM {table} GROUP BY {ids_csv})",
+ )
|
Source code in mt5cli/history.py
- 110
-111
-112
-113
-114
-115
-116
+ 116
117
118
119
@@ -1816,21 +1854,27 @@ completed bars. Empty frames and single-row frames return empty results.
121
122
123
-124 | def drop_forming_rate_bar(df_rate: pd.DataFrame) -> pd.DataFrame:
- """Return closed bars from chronologically ordered MT5 rate data.
-
- MetaTrader 5 ``copy_rates_from_pos(start_pos=0)`` includes the still-forming
- current bar as the last row. Slice it off so downstream logic only sees
- completed bars. Empty frames and single-row frames return empty results.
-
- Args:
- df_rate: Rate data ordered oldest-to-newest with the forming bar last.
-
- Returns:
- A new DataFrame with all rows except the last. Index and columns are
- preserved. The input frame is not modified.
- """
- return df_rate.iloc[:-1].copy()
+124
+125
+126
+127
+128
+129
+130
| def drop_forming_rate_bar(df_rate: pd.DataFrame) -> pd.DataFrame:
+ """Return closed bars from chronologically ordered MT5 rate data.
+
+ MetaTrader 5 ``copy_rates_from_pos(start_pos=0)`` includes the still-forming
+ current bar as the last row. Slice it off so downstream logic only sees
+ completed bars. Empty frames and single-row frames return empty results.
+
+ Args:
+ df_rate: Rate data ordered oldest-to-newest with the forming bar last.
+
+ Returns:
+ A new DataFrame with all rows except the last. Index and columns are
+ preserved. The input frame is not modified.
+ """
+ return df_rate.iloc[:-1].copy()
|
Source code in mt5cli/history.py
- | def drop_rate_compatibility_views(conn: sqlite3.Connection) -> None:
- """Drop all mt5cli-managed ``rate_*`` compatibility views."""
- rows = conn.execute(
- "SELECT name FROM sqlite_master WHERE type = 'view' AND name GLOB 'rate_*'",
- ).fetchall()
- for (view_name,) in rows:
- quoted_view_name = quote_sqlite_identifier(str(view_name))
- conn.execute(f"DROP VIEW IF EXISTS {quoted_view_name}")
+ | def drop_rate_compatibility_views(conn: sqlite3.Connection) -> None:
+ """Drop all mt5cli-managed ``rate_*`` compatibility views."""
+ rows = conn.execute(
+ "SELECT name FROM sqlite_master WHERE type = 'view' AND name GLOB 'rate_*'",
+ ).fetchall()
+ for (view_name,) in rows:
+ quoted_view_name = quote_sqlite_identifier(str(view_name))
+ conn.execute(f"DROP VIEW IF EXISTS {quoted_view_name}")
|
Source code in mt5cli/history.py
- 1236
-1237
-1238
-1239
-1240
-1241
-1242
+ | def filter_incremental_history_deals_frame(
- frame: pd.DataFrame,
- symbols: Sequence[str],
- start_by_symbol: dict[str, datetime],
- account_event_start: datetime,
-) -> pd.DataFrame:
- """Filter incrementally fetched history_deals by symbol and event start times.
-
- Returns:
- Rows for selected symbols at or after each symbol start, plus account
- events at or after ``account_event_start``.
- """
- if frame.empty:
- return frame.copy()
- parsed_times = _frame_parsed_times(frame)
- time_valid = parsed_times.notna()
- account_event_mask = _history_deals_account_event_mask(frame)
- account_keep = account_event_mask & (parsed_times >= account_event_start)
- trade_keep = pd.Series(data=False, index=frame.index)
- if "symbol" in frame.columns:
- for symbol in symbols:
- trade_keep |= (
- (frame["symbol"] == symbol)
- & (parsed_times >= start_by_symbol[symbol])
- & ~account_event_mask
- )
- keep = (account_keep | trade_keep) & time_valid
- return frame.loc[keep].copy()
+1263
+1264
+1265
+1266
+1267
+1268
+1269
| def filter_incremental_history_deals_frame(
+ frame: pd.DataFrame,
+ symbols: Sequence[str],
+ start_by_symbol: dict[str, datetime],
+ account_event_start: datetime,
+) -> pd.DataFrame:
+ """Filter incrementally fetched history_deals by symbol and event start times.
+
+ Returns:
+ Rows for selected symbols at or after each symbol start, plus account
+ events at or after ``account_event_start``.
+ """
+ if frame.empty:
+ return frame.copy()
+ parsed_times = _frame_parsed_times(frame)
+ time_valid = parsed_times.notna()
+ account_event_mask = _history_deals_account_event_mask(frame)
+ account_keep = account_event_mask & (parsed_times >= account_event_start)
+ trade_keep = pd.Series(data=False, index=frame.index)
+ if "symbol" in frame.columns:
+ for symbol in symbols:
+ trade_keep |= (
+ (frame["symbol"] == symbol)
+ & (parsed_times >= start_by_symbol[symbol])
+ & ~account_event_mask
+ )
+ keep = (account_keep | trade_keep) & time_valid
+ return frame.loc[keep].copy()
|
Source code in mt5cli/history.py
- 1266
-1267
-1268
-1269
-1270
-1271
-1272
+ | def filter_trade_history_frame(
- frame: pd.DataFrame,
- symbols: Sequence[str],
- *,
- include_account_events: bool,
-) -> pd.DataFrame:
- """Filter trade history rows to selected symbols and account events.
-
- Returns:
- Filtered history rows.
- """
- if "symbol" not in frame.columns:
- return frame
- symbol_mask = frame["symbol"].isin(symbols)
- if not include_account_events:
- return frame.loc[symbol_mask].copy()
- account_event_mask = _history_deals_account_event_mask(frame)
- return frame.loc[symbol_mask | account_event_mask].copy()
+1283
+1284
+1285
+1286
+1287
+1288
+1289
| def filter_trade_history_frame(
+ frame: pd.DataFrame,
+ symbols: Sequence[str],
+ *,
+ include_account_events: bool,
+) -> pd.DataFrame:
+ """Filter trade history rows to selected symbols and account events.
+
+ Returns:
+ Filtered history rows.
+ """
+ if "symbol" not in frame.columns:
+ return frame
+ symbol_mask = frame["symbol"].isin(symbols)
+ if not include_account_events:
+ return frame.loc[symbol_mask].copy()
+ account_event_mask = _history_deals_account_event_mask(frame)
+ return frame.loc[symbol_mask | account_event_mask].copy()
|
Source code in mt5cli/history.py
- 874
-875
-876
-877
-878
-879
-880
+ 880
881
882
883
@@ -2120,27 +2158,33 @@ completed bars. Empty frames and single-row frames return empty results.
891
892
893
-894 | def get_history_deals_account_event_start_datetime(
- conn: sqlite3.Connection,
- *,
- fallback_start: datetime,
-) -> datetime:
- """Return the next update start for account-level history_deals rows."""
- table = Dataset.history_deals.table_name
- columns = get_table_columns(conn, table)
- if "time" not in columns:
- return fallback_start
- if "type" in columns:
- where_clause = f"type NOT IN {_TRADE_DEAL_TYPES_SQL}"
- elif "symbol" in columns:
- where_clause = "symbol IS NULL OR symbol = ''"
- else:
+894
+895
+896
+897
+898
+899
+900
| def get_history_deals_account_event_start_datetime(
+ conn: sqlite3.Connection,
+ *,
+ fallback_start: datetime,
+) -> datetime:
+ """Return the next update start for account-level history_deals rows."""
+ table = Dataset.history_deals.table_name
+ columns = get_table_columns(conn, table)
+ if "time" not in columns:
return fallback_start
- row = conn.execute(
- f"SELECT MAX(time) FROM {table} WHERE {where_clause}", # noqa: S608
- ).fetchone()
- parsed = parse_sqlite_timestamp(row[0] if row else None)
- return parsed if parsed is not None else fallback_start
+ if "type" in columns:
+ where_clause = f"type NOT IN {_TRADE_DEAL_TYPES_SQL}"
+ elif "symbol" in columns:
+ where_clause = "symbol IS NULL OR symbol = ''"
+ else:
+ return fallback_start
+ row = conn.execute(
+ f"SELECT MAX(time) FROM {table} WHERE {where_clause}", # noqa: S608
+ ).fetchone()
+ parsed = parse_sqlite_timestamp(row[0] if row else None)
+ return parsed if parsed is not None else fallback_start
|
Source code in mt5cli/history.py
- 992
- 993
- 994
- 995
- 996
- 997
- 998
+ | def get_incremental_start_datetime(
- conn: sqlite3.Connection,
- dataset: Dataset,
- *,
- symbol: str,
- timeframe: int | None,
- fallback_start: datetime,
-) -> datetime:
- """Return the next update start datetime from existing MAX(time)."""
- timeframes = [timeframe] if timeframe is not None else None
- starts = load_incremental_start_datetimes(
- conn,
- dataset,
- symbols=[symbol],
- timeframes=timeframes,
- fallback_start=fallback_start,
- )
- return starts[symbol, timeframe]
+1009
+1010
+1011
+1012
+1013
+1014
+1015
| def get_incremental_start_datetime(
+ conn: sqlite3.Connection,
+ dataset: Dataset,
+ *,
+ symbol: str,
+ timeframe: int | None,
+ fallback_start: datetime,
+) -> datetime:
+ """Return the next update start datetime from existing MAX(time)."""
+ timeframes = [timeframe] if timeframe is not None else None
+ starts = load_incremental_start_datetimes(
+ conn,
+ dataset,
+ symbols=[symbol],
+ timeframes=timeframes,
+ fallback_start=fallback_start,
+ )
+ return starts[symbol, timeframe]
|
Source code in mt5cli/history.py
- | def get_table_columns(conn: sqlite3.Connection, table: str) -> set[str]:
- """Return existing SQLite columns for a table."""
- quoted_table = quote_sqlite_identifier(table)
- rows = conn.execute(f"PRAGMA table_info({quoted_table})").fetchall()
- return {str(row[1]) for row in rows}
+ | def get_table_columns(conn: sqlite3.Connection, table: str) -> set[str]:
+ """Return existing SQLite columns for a table."""
+ quoted_table = quote_sqlite_identifier(table)
+ rows = conn.execute(f"PRAGMA table_info({quoted_table})").fetchall()
+ return {str(row[1]) for row in rows}
|
Source code in mt5cli/history.py
- 916
-917
-918
-919
-920
-921
-922
+ 922
923
924
925
@@ -2344,80 +2382,86 @@ completed bars. Empty frames and single-row frames return empty results.
986
987
988
-989 | def load_incremental_start_datetimes(
- conn: sqlite3.Connection,
- dataset: Dataset,
- *,
- symbols: Sequence[str],
- timeframes: Sequence[int] | None = None,
- fallback_start: datetime,
-) -> dict[tuple[str, int | None], datetime]:
- """Return next update start datetimes keyed by symbol and optional timeframe."""
- table = dataset.table_name
- columns = get_table_columns(conn, table)
- if dataset is Dataset.rates and columns:
- _validate_rates_schema(columns)
-
- if "time" not in columns:
- if dataset is Dataset.rates and timeframes is not None:
- return {
- (symbol, timeframe): fallback_start
- for symbol in symbols
- for timeframe in timeframes
- }
- return {(symbol, None): fallback_start for symbol in symbols}
-
- parsed_by_key: dict[tuple[str, int | None], datetime] = {}
- if (
- dataset is Dataset.rates
- and timeframes is not None
- and {"symbol", "timeframe"}.issubset(columns)
- ):
- symbol_placeholders = ", ".join("?" for _ in symbols)
- timeframe_placeholders = ", ".join("?" for _ in timeframes)
- grouped_rates_query = (
- "SELECT symbol, timeframe, MAX(time) FROM " # noqa: S608
- f"{table} WHERE symbol IN ({symbol_placeholders})"
- f" AND timeframe IN ({timeframe_placeholders})"
- " GROUP BY symbol, timeframe"
- )
- rows = conn.execute(
- grouped_rates_query,
- [*symbols, *timeframes],
- ).fetchall()
- for row_symbol, row_timeframe, max_time in rows:
- parsed = parse_sqlite_timestamp(max_time)
- if parsed is not None:
- parsed_by_key[str(row_symbol), int(row_timeframe)] = parsed
- return {
- (symbol, timeframe): parsed_by_key.get(
- (symbol, timeframe),
- fallback_start,
- )
- for symbol in symbols
- for timeframe in timeframes
- }
-
- if "symbol" in columns:
- symbol_placeholders = ", ".join("?" for _ in symbols)
- rows = conn.execute(
- f"SELECT symbol, MAX(time) FROM {table}" # noqa: S608
- f" WHERE symbol IN ({symbol_placeholders}) GROUP BY symbol",
- list(symbols),
- ).fetchall()
- for row_symbol, max_time in rows:
- parsed = parse_sqlite_timestamp(max_time)
- if parsed is not None:
- parsed_by_key[str(row_symbol), None] = parsed
- return {
- (symbol, None): parsed_by_key.get((symbol, None), fallback_start)
- for symbol in symbols
- }
-
- row = conn.execute(f"SELECT MAX(time) FROM {table}").fetchone() # noqa: S608
- parsed = parse_sqlite_timestamp(row[0] if row else None)
- shared_start = parsed if parsed is not None else fallback_start
- return {(symbol, None): shared_start for symbol in symbols}
+989
+990
+991
+992
+993
+994
+995
| def load_incremental_start_datetimes(
+ conn: sqlite3.Connection,
+ dataset: Dataset,
+ *,
+ symbols: Sequence[str],
+ timeframes: Sequence[int] | None = None,
+ fallback_start: datetime,
+) -> dict[tuple[str, int | None], datetime]:
+ """Return next update start datetimes keyed by symbol and optional timeframe."""
+ table = dataset.table_name
+ columns = get_table_columns(conn, table)
+ if dataset is Dataset.rates and columns:
+ _validate_rates_schema(columns)
+
+ if "time" not in columns:
+ if dataset is Dataset.rates and timeframes is not None:
+ return {
+ (symbol, timeframe): fallback_start
+ for symbol in symbols
+ for timeframe in timeframes
+ }
+ return {(symbol, None): fallback_start for symbol in symbols}
+
+ parsed_by_key: dict[tuple[str, int | None], datetime] = {}
+ if (
+ dataset is Dataset.rates
+ and timeframes is not None
+ and {"symbol", "timeframe"}.issubset(columns)
+ ):
+ symbol_placeholders = ", ".join("?" for _ in symbols)
+ timeframe_placeholders = ", ".join("?" for _ in timeframes)
+ grouped_rates_query = (
+ "SELECT symbol, timeframe, MAX(time) FROM " # noqa: S608
+ f"{table} WHERE symbol IN ({symbol_placeholders})"
+ f" AND timeframe IN ({timeframe_placeholders})"
+ " GROUP BY symbol, timeframe"
+ )
+ rows = conn.execute(
+ grouped_rates_query,
+ [*symbols, *timeframes],
+ ).fetchall()
+ for row_symbol, row_timeframe, max_time in rows:
+ parsed = parse_sqlite_timestamp(max_time)
+ if parsed is not None:
+ parsed_by_key[str(row_symbol), int(row_timeframe)] = parsed
+ return {
+ (symbol, timeframe): parsed_by_key.get(
+ (symbol, timeframe),
+ fallback_start,
+ )
+ for symbol in symbols
+ for timeframe in timeframes
+ }
+
+ if "symbol" in columns:
+ symbol_placeholders = ", ".join("?" for _ in symbols)
+ rows = conn.execute(
+ f"SELECT symbol, MAX(time) FROM {table}" # noqa: S608
+ f" WHERE symbol IN ({symbol_placeholders}) GROUP BY symbol",
+ list(symbols),
+ ).fetchall()
+ for row_symbol, max_time in rows:
+ parsed = parse_sqlite_timestamp(max_time)
+ if parsed is not None:
+ parsed_by_key[str(row_symbol), None] = parsed
+ return {
+ (symbol, None): parsed_by_key.get((symbol, None), fallback_start)
+ for symbol in symbols
+ }
+
+ row = conn.execute(f"SELECT MAX(time) FROM {table}").fetchone() # noqa: S608
+ parsed = parse_sqlite_timestamp(row[0] if row else None)
+ shared_start = parsed if parsed is not None else fallback_start
+ return {(symbol, None): shared_start for symbol in symbols}
|
Source code in mt5cli/history.py
- 300
-301
-302
-303
-304
-305
-306
+ 306
307
308
309
@@ -2563,28 +2601,34 @@ completed bars. Empty frames and single-row frames return empty results.
318
319
320
-321 | def load_rate_data(
- conn_or_path: SqliteConnOrPath,
- table: str,
- count: int | None = None,
-) -> pd.DataFrame:
- """Load rate-like data from a SQLite database path or connection.
-
- Args:
- conn_or_path: SQLite database path or open connection.
- table: Source table or view name.
- count: Optional number of most recent rows to load.
-
- Returns:
- DataFrame indexed by ascending ``time``.
-
- """
- conn, should_close = _open_existing_sqlite_database(conn_or_path)
- try:
- return load_rate_data_from_connection(conn, table, count=count)
- finally:
- if should_close:
- conn.close()
+321
+322
+323
+324
+325
+326
+327
| def load_rate_data(
+ conn_or_path: SqliteConnOrPath,
+ table: str,
+ count: int | None = None,
+) -> pd.DataFrame:
+ """Load rate-like data from a SQLite database path or connection.
+
+ Args:
+ conn_or_path: SQLite database path or open connection.
+ table: Source table or view name.
+ count: Optional number of most recent rows to load.
+
+ Returns:
+ DataFrame indexed by ascending ``time``.
+
+ """
+ conn, should_close = _open_existing_sqlite_database(conn_or_path)
+ try:
+ return load_rate_data_from_connection(conn, table, count=count)
+ finally:
+ if should_close:
+ conn.close()
|
Source code in mt5cli/history.py
- 254
-255
-256
-257
-258
-259
-260
+ | def load_rate_data_from_connection(
- connection: sqlite3.Connection,
- table: str,
- count: int | None = None,
-) -> pd.DataFrame:
- """Load rate-like data from a SQLite table or view.
-
- Args:
- connection: Open SQLite connection.
- table: Source table or view name.
- count: Optional number of most recent rows to load.
-
- Returns:
- DataFrame indexed by ascending ``time``.
-
- Raises:
- ValueError: If inputs, schema, timestamps are invalid, or the table
- or view contains no rows.
- """
- table_name = _validate_rate_load_request(table, count)
- columns = get_table_columns(connection, table_name)
- _ensure_rate_columns(columns, table_name)
- quoted_table = quote_sqlite_identifier(table_name)
- if count is None:
- frame = cast(
- "pd.DataFrame",
- pd.read_sql_query( # type: ignore[reportUnknownMemberType]
- f"SELECT * FROM {quoted_table} ORDER BY time ASC", # noqa: S608
- connection,
- ),
- )
- else:
- frame = cast(
- "pd.DataFrame",
- pd.read_sql_query( # type: ignore[reportUnknownMemberType]
- f"SELECT * FROM {quoted_table} ORDER BY time DESC LIMIT ?", # noqa: S608
- connection,
- params=(count,),
- ),
- )
- if frame.empty:
- msg = f"SQLite table or view {table_name!r} contains no rows."
- raise ValueError(msg)
- return _parse_rate_time_index(frame, table_name)
+297
+298
+299
+300
+301
+302
+303
| def load_rate_data_from_connection(
+ connection: sqlite3.Connection,
+ table: str,
+ count: int | None = None,
+) -> pd.DataFrame:
+ """Load rate-like data from a SQLite table or view.
+
+ Args:
+ connection: Open SQLite connection.
+ table: Source table or view name.
+ count: Optional number of most recent rows to load.
+
+ Returns:
+ DataFrame indexed by ascending ``time``.
+
+ Raises:
+ ValueError: If inputs, schema, timestamps are invalid, or the table
+ or view contains no rows.
+ """
+ table_name = _validate_rate_load_request(table, count)
+ columns = get_table_columns(connection, table_name)
+ _ensure_rate_columns(columns, table_name)
+ quoted_table = quote_sqlite_identifier(table_name)
+ if count is None:
+ frame = cast(
+ "pd.DataFrame",
+ pd.read_sql_query( # type: ignore[reportUnknownMemberType]
+ f"SELECT * FROM {quoted_table} ORDER BY time ASC", # noqa: S608
+ connection,
+ ),
+ )
+ else:
+ frame = cast(
+ "pd.DataFrame",
+ pd.read_sql_query( # type: ignore[reportUnknownMemberType]
+ f"SELECT * FROM {quoted_table} ORDER BY time DESC LIMIT ?", # noqa: S608
+ connection,
+ params=(count,),
+ ),
+ )
+ if frame.empty:
+ msg = f"SQLite table or view {table_name!r} contains no rows."
+ raise ValueError(msg)
+ return _parse_rate_time_index(frame, table_name)
|
symbol=None for each granularity instead of raising.
Source code in mt5cli/history.py
- 789
-790
-791
-792
-793
-794
-795
+ | def load_rate_series_by_granularity(
- conn_or_path: SqliteConnOrPath,
- symbols: Sequence[str],
- granularities: Sequence[int | str],
- count: int,
- *,
- explicit_tables: Sequence[str] | None = None,
- allow_missing_symbol: bool = False,
-) -> dict[tuple[str | None, str], pd.DataFrame]:
- """Load rate series keyed by symbol and string granularity name.
-
- Builds targets with :func:`build_rate_targets` and loads them with
- :func:`load_rate_series_from_sqlite`, then rekeys the result by granularity
- name (for example ``M1``) instead of the integer timeframe to reduce
- downstream boilerplate.
-
- Args:
- conn_or_path: SQLite database path or open connection.
- symbols: MT5 symbol names. May be empty when ``allow_missing_symbol``.
- granularities: MT5 timeframes as integers or names (for example ``M1``).
- count: Number of most recent rows to load per series.
- explicit_tables: Optional explicit table or view names matching the
- built targets in row-major order. Required when symbols are omitted.
- allow_missing_symbol: When True and ``symbols`` is empty, build targets
- with ``symbol=None`` for each granularity instead of raising.
-
- Returns:
- Mapping keyed by ``(symbol | None, granularity_name)`` to each rate
- DataFrame. Propagates ``ValueError`` (via :func:`build_rate_targets` and
- :func:`load_rate_series_from_sqlite`) when inputs are empty or invalid,
- table resolution fails, or duplicate targets are present.
- """
- targets = build_rate_targets(
- symbols,
- granularities,
- allow_missing_symbol=allow_missing_symbol,
- )
- series = load_rate_series_from_sqlite(
- conn_or_path,
- targets,
- count,
- explicit_tables=explicit_tables,
+835
+836
+837
+838
+839
+840
+841
| def load_rate_series_by_granularity(
+ conn_or_path: SqliteConnOrPath,
+ symbols: Sequence[str],
+ granularities: Sequence[int | str],
+ count: int,
+ *,
+ explicit_tables: Sequence[str] | None = None,
+ allow_missing_symbol: bool = False,
+) -> dict[tuple[str | None, str], pd.DataFrame]:
+ """Load rate series keyed by symbol and string granularity name.
+
+ Builds targets with :func:`build_rate_targets` and loads them with
+ :func:`load_rate_series_from_sqlite`, then rekeys the result by granularity
+ name (for example ``M1``) instead of the integer timeframe to reduce
+ downstream boilerplate.
+
+ Args:
+ conn_or_path: SQLite database path or open connection.
+ symbols: MT5 symbol names. May be empty when ``allow_missing_symbol``.
+ granularities: MT5 timeframes as integers or names (for example ``M1``).
+ count: Number of most recent rows to load per series.
+ explicit_tables: Optional explicit table or view names matching the
+ built targets in row-major order. Required when symbols are omitted.
+ allow_missing_symbol: When True and ``symbols`` is empty, build targets
+ with ``symbol=None`` for each granularity instead of raising.
+
+ Returns:
+ Mapping keyed by ``(symbol | None, granularity_name)`` to each rate
+ DataFrame. Propagates ``ValueError`` (via :func:`build_rate_targets` and
+ :func:`load_rate_series_from_sqlite`) when inputs are empty or invalid,
+ table resolution fails, or duplicate targets are present.
+ """
+ targets = build_rate_targets(
+ symbols,
+ granularities,
+ allow_missing_symbol=allow_missing_symbol,
)
- return {
- (symbol, resolve_granularity_name(timeframe)): frame
- for (symbol, timeframe), frame in series.items()
- }
+ series = load_rate_series_from_sqlite(
+ conn_or_path,
+ targets,
+ count,
+ explicit_tables=explicit_tables,
+ )
+ return {
+ (symbol, resolve_granularity_name(timeframe)): frame
+ for (symbol, timeframe), frame in series.items()
+ }
|
Source code in mt5cli/history.py
- 709
-710
-711
-712
-713
-714
-715
+ | def load_rate_series_from_sqlite(
- conn_or_path: SqliteConnOrPath,
- targets: Sequence[RateTarget] | None = None,
- count: int | None = None,
- explicit_tables: Sequence[str] | None = None,
- *,
- table: str | None = None,
-) -> dict[tuple[str | None, int], pd.DataFrame] | pd.DataFrame:
- """Load one table/view or multiple rate series from a SQLite database.
-
- Args:
- conn_or_path: SQLite database path or open connection.
- targets: Rate targets to load. Each ``(symbol, timeframe_int)`` pair must
- be unique. Omit when loading a single explicit ``table``.
- count: Optional number of most recent rows to load per series.
- explicit_tables: Optional explicit table or view names matching targets.
- When omitted, managed ``rate_*`` compatibility views must already
- exist in the database.
- table: Optional single table or view name to load directly.
-
- Returns:
- A DataFrame when ``table`` is provided, otherwise a mapping keyed by
- ``(symbol, timeframe_int)`` to each rate DataFrame.
-
- Raises:
- ValueError: If ``count`` is not positive, targets are empty, duplicate
- ``(symbol, timeframe_int)`` pairs are present, or table resolution
- fails.
- """
- if table is not None:
- return load_rate_data(conn_or_path, table, count=count)
- if count is None or count <= 0:
- msg = "count must be positive."
- raise ValueError(msg)
- if targets is None:
- msg = "targets are required when table is not provided."
- raise ValueError(msg)
- target_list = list(targets)
- if not target_list:
- msg = "At least one rate target is required."
- raise ValueError(msg)
- if explicit_tables is None and any(target.symbol is None for target in target_list):
- msg = (
- "Cannot resolve a rate table for a target without a symbol; "
- "provide explicit_tables."
- )
+786
+787
+788
+789
+790
+791
+792
| def load_rate_series_from_sqlite(
+ conn_or_path: SqliteConnOrPath,
+ targets: Sequence[RateTarget] | None = None,
+ count: int | None = None,
+ explicit_tables: Sequence[str] | None = None,
+ *,
+ table: str | None = None,
+) -> dict[tuple[str | None, int], pd.DataFrame] | pd.DataFrame:
+ """Load one table/view or multiple rate series from a SQLite database.
+
+ Args:
+ conn_or_path: SQLite database path or open connection.
+ targets: Rate targets to load. Each ``(symbol, timeframe_int)`` pair must
+ be unique. Omit when loading a single explicit ``table``.
+ count: Optional number of most recent rows to load per series.
+ explicit_tables: Optional explicit table or view names matching targets.
+ When omitted, managed ``rate_*`` compatibility views must already
+ exist in the database.
+ table: Optional single table or view name to load directly.
+
+ Returns:
+ A DataFrame when ``table`` is provided, otherwise a mapping keyed by
+ ``(symbol, timeframe_int)`` to each rate DataFrame.
+
+ Raises:
+ ValueError: If ``count`` is not positive, targets are empty, duplicate
+ ``(symbol, timeframe_int)`` pairs are present, or table resolution
+ fails.
+ """
+ if table is not None:
+ return load_rate_data(conn_or_path, table, count=count)
+ if count is None or count <= 0:
+ msg = "count must be positive."
+ raise ValueError(msg)
+ if targets is None:
+ msg = "targets are required when table is not provided."
+ raise ValueError(msg)
+ target_list = list(targets)
+ if not target_list:
+ msg = "At least one rate target is required."
raise ValueError(msg)
- seen_keys: set[tuple[str | None, int]] = set()
- for target in target_list:
- key = (target.symbol, target.timeframe_int)
- if key in seen_keys:
- symbol_repr = repr(target.symbol)
- msg = f"Duplicate rate target: ({symbol_repr}, {target.timeframe_int})"
- raise ValueError(msg)
- seen_keys.add(key)
- tables = (
- resolve_rate_tables(None, target_list, explicit_tables)
- if explicit_tables is not None
- else None
- )
- conn, should_close = _open_existing_sqlite_database(conn_or_path)
- try:
- resolved_tables = tables or resolve_rate_tables(
- conn,
- target_list,
- require_existing=True,
- )
- return {
- (target.symbol, target.timeframe_int): load_rate_data_from_connection(
- conn,
- table,
- count=count,
- )
- for target, table in zip(target_list, resolved_tables, strict=True)
- }
- finally:
- if should_close:
- conn.close()
+ if explicit_tables is None and any(target.symbol is None for target in target_list):
+ msg = (
+ "Cannot resolve a rate table for a target without a symbol; "
+ "provide explicit_tables."
+ )
+ raise ValueError(msg)
+ seen_keys: set[tuple[str | None, int]] = set()
+ for target in target_list:
+ key = (target.symbol, target.timeframe_int)
+ if key in seen_keys:
+ symbol_repr = repr(target.symbol)
+ msg = f"Duplicate rate target: ({symbol_repr}, {target.timeframe_int})"
+ raise ValueError(msg)
+ seen_keys.add(key)
+ tables = (
+ resolve_rate_tables(None, target_list, explicit_tables)
+ if explicit_tables is not None
+ else None
+ )
+ conn, should_close = _open_existing_sqlite_database(conn_or_path)
+ try:
+ resolved_tables = tables or resolve_rate_tables(
+ conn,
+ target_list,
+ require_existing=True,
+ )
+ return {
+ (target.symbol, target.timeframe_int): load_rate_data_from_connection(
+ conn,
+ table,
+ count=count,
+ )
+ for target, table in zip(target_list, resolved_tables, strict=True)
+ }
+ finally:
+ if should_close:
+ conn.close()
|
Source code in mt5cli/history.py
- 856
-857
-858
-859
-860
-861
-862
+ | def parse_sqlite_timestamp(value: object) -> datetime | None:
- """Parse a SQLite history timestamp value.
-
- Returns:
- Parsed timezone-aware datetime, or None when parsing fails.
- """
- if value is None:
- return None
- if isinstance(value, datetime):
- return value if value.tzinfo is not None else value.replace(tzinfo=UTC)
- if isinstance(value, int | float):
- return datetime.fromtimestamp(float(value), tz=UTC)
- if isinstance(value, str):
- return _parse_string_sqlite_timestamp(value)
- logger.warning("Ignoring unsupported history timestamp type: %s", type(value))
- return None
+871
+872
+873
+874
+875
+876
+877
| def parse_sqlite_timestamp(value: object) -> datetime | None:
+ """Parse a SQLite history timestamp value.
+
+ Returns:
+ Parsed timezone-aware datetime, or None when parsing fails.
+ """
+ if value is None:
+ return None
+ if isinstance(value, datetime):
+ return value if value.tzinfo is not None else value.replace(tzinfo=UTC)
+ if isinstance(value, int | float):
+ return datetime.fromtimestamp(float(value), tz=UTC)
+ if isinstance(value, str):
+ return _parse_string_sqlite_timestamp(value)
+ logger.warning("Ignoring unsupported history timestamp type: %s", type(value))
+ return None
|
Source code in mt5cli/history.py
- | def quote_sqlite_identifier(identifier: str) -> str:
- """Return a safely quoted SQLite identifier using double quotes."""
- return '"' + identifier.replace('"', '""') + '"'
+ | def quote_sqlite_identifier(identifier: str) -> str:
+ """Return a safely quoted SQLite identifier using double quotes."""
+ return '"' + identifier.replace('"', '""') + '"'
|
Source code in mt5cli/history.py
- 1036
-1037
-1038
-1039
-1040
-1041
-1042
+ | def record_written_columns(
- written_columns: dict[Dataset, set[str]],
- dataset: Dataset,
- frame: pd.DataFrame,
-) -> None:
- """Remember columns for datasets written during collection."""
- columns = set(frame.columns)
- if dataset in written_columns:
- written_columns[dataset].update(columns)
- else:
- written_columns[dataset] = columns
+1046
+1047
+1048
+1049
+1050
+1051
+1052
| def record_written_columns(
+ written_columns: dict[Dataset, set[str]],
+ dataset: Dataset,
+ frame: pd.DataFrame,
+) -> None:
+ """Remember columns for datasets written during collection."""
+ columns = set(frame.columns)
+ if dataset in written_columns:
+ written_columns[dataset].update(columns)
+ else:
+ written_columns[dataset] = columns
|
Source code in mt5cli/history.py
- | def resolve_granularity_name(timeframe: int) -> str:
- """Return a granularity name for a timeframe integer when known."""
- try:
- name = _get_timeframe_name(timeframe)
- except ValueError:
- return str(timeframe)
- return name.removeprefix("TIMEFRAME_")
+ | def resolve_granularity_name(timeframe: int) -> str:
+ """Return a granularity name for a timeframe integer when known."""
+ try:
+ name = _get_timeframe_name(timeframe)
+ except ValueError:
+ return str(timeframe)
+ return name.removeprefix("TIMEFRAME_")
|
|
- All supported datasets when datasets is None, otherwise the
+ DEFAULT_HISTORY_DATASETS (rates, history-orders, history-deals)
|
@@ -3765,7 +3809,17 @@ fails.
- configured selection (which may be empty).
+ when datasets is None, otherwise the configured selection (which
+ |
+
+
+
+ set[Dataset]
+ |
+
+
+ may be empty or explicitly include Dataset.ticks).
|
@@ -3775,25 +3829,27 @@ fails.
Source code in mt5cli/history.py
- 61
-62
-63
-64
-65
-66
+ | def resolve_history_datasets(datasets: set[Dataset] | None) -> set[Dataset]:
- """Resolve configured history datasets.
-
- Returns:
- All supported datasets when ``datasets`` is None, otherwise the
- configured selection (which may be empty).
- """
- if datasets is None:
- return set(Dataset)
- return set(datasets)
+70
+71
+72
+73
+74
+75
+76
| def resolve_history_datasets(datasets: set[Dataset] | None) -> set[Dataset]:
+ """Resolve configured history datasets.
+
+ Returns:
+ ``DEFAULT_HISTORY_DATASETS`` (rates, history-orders, history-deals)
+ when ``datasets`` is None, otherwise the configured selection (which
+ may be empty or explicitly include ``Dataset.ticks``).
+ """
+ if datasets is None:
+ return set(DEFAULT_HISTORY_DATASETS)
+ return set(datasets)
|
Source code in mt5cli/history.py
- | def resolve_history_tick_flags(flags: int | str) -> int:
- """Resolve tick copy flags from an integer or name.
-
- Returns:
- Integer tick flag value.
- """
- return parse_tick_flags(flags)
+ | def resolve_history_tick_flags(flags: int | str) -> int:
+ """Resolve tick copy flags from an integer or name.
+
+ Returns:
+ Integer tick flag value.
+ """
+ return parse_tick_flags(flags)
|
Source code in mt5cli/history.py
- 73
-74
-75
-76
-77
-78
-79
+ | def resolve_history_timeframes(
- timeframes: Sequence[int | str] | None,
-) -> list[int]:
- """Resolve rate timeframes, deduplicating aliases for the same integer.
-
- Returns:
- Ordered list of unique timeframe integers.
- """
- raw = timeframes if timeframes is not None else DEFAULT_HISTORY_TIMEFRAMES
- seen: set[int] = set()
- resolved: list[int] = []
- for value in raw:
- tf = parse_timeframe(value)
- if tf not in seen:
- seen.add(tf)
- resolved.append(tf)
- return resolved
+89
+90
+91
+92
+93
+94
+95
| def resolve_history_timeframes(
+ timeframes: Sequence[int | str] | None,
+) -> list[int]:
+ """Resolve rate timeframes, deduplicating aliases for the same integer.
+
+ Returns:
+ Ordered list of unique timeframe integers.
+ """
+ raw = timeframes if timeframes is not None else DEFAULT_HISTORY_TIMEFRAMES
+ seen: set[int] = set()
+ resolved: list[int] = []
+ for value in raw:
+ tf = parse_timeframe(value)
+ if tf not in seen:
+ seen.add(tf)
+ resolved.append(tf)
+ return resolved
|
Source code in mt5cli/history.py
- 145
-146
-147
-148
-149
-150
-151
+ | def resolve_rate_table_name(symbol: str, granularity: str) -> str:
- """Return the canonical normalized SQLite rate table name.
-
- The normalized history table stores all symbols and timeframes in
- ``rates``; use :func:`resolve_rate_view_name` for per-symbol compatibility
- view names.
-
- Returns:
- Canonical normalized rates table name.
-
- Raises:
- ValueError: If ``symbol`` or ``granularity`` is invalid.
- """
- parse_timeframe(granularity)
- if not symbol.strip():
- msg = "symbol must not be empty."
- raise ValueError(msg)
- return Dataset.rates.table_name
+162
+163
+164
+165
+166
+167
+168
| def resolve_rate_table_name(symbol: str, granularity: str) -> str:
+ """Return the canonical normalized SQLite rate table name.
+
+ The normalized history table stores all symbols and timeframes in
+ ``rates``; use :func:`resolve_rate_view_name` for per-symbol compatibility
+ view names.
+
+ Returns:
+ Canonical normalized rates table name.
+
+ Raises:
+ ValueError: If ``symbol`` or ``granularity`` is invalid.
+ """
+ parse_timeframe(granularity)
+ if not symbol.strip():
+ msg = "symbol must not be empty."
+ raise ValueError(msg)
+ return Dataset.rates.table_name
|
Source code in mt5cli/history.py
- 592
-593
-594
-595
-596
-597
-598
+ | def resolve_rate_tables(
- conn_or_path: SqliteConnOrPath | None,
- targets: Sequence[RateTarget],
- explicit_tables: Sequence[str] | None = None,
- *,
- require_existing: bool = False,
-) -> list[str]:
- """Resolve SQLite table or view names for rate targets.
-
- Args:
- conn_or_path: SQLite database path or open connection. May be None when
- ``explicit_tables`` is provided, or when ``require_existing`` is
- False and deterministic default view names are sufficient.
- targets: Rate targets to resolve.
- explicit_tables: Optional explicit table or view names. When provided,
- they are used as-is and must match the number of targets.
- require_existing: When True, require the database and managed views to
- exist for each symbol target. Ignored when ``explicit_tables`` is
- provided.
-
- Returns:
- Table or view names aligned with ``targets``.
-
- Raises:
- ValueError: If ``targets`` is empty, ``explicit_tables`` length does not
- match the target count, a target without a symbol is resolved
- without an explicit table, or ``require_existing`` is True and the
- database or a managed view is missing.
- """
- target_list = list(targets)
- if not target_list:
- msg = "At least one rate target is required."
- raise ValueError(msg)
- if explicit_tables is not None:
- tables = list(explicit_tables)
- if len(tables) != len(target_list):
- msg = (
- f"Expected {len(target_list)} explicit table(s) "
- f"to match the targets, got {len(tables)}."
- )
- raise ValueError(msg)
- return tables
- if any(target.symbol is None for target in target_list):
- msg = (
- "Cannot resolve a rate table for a target without a symbol; "
- "provide explicit_tables."
- )
- raise ValueError(msg)
- conn, should_close = _open_history_connection(conn_or_path)
- try:
- if conn is None:
- if require_existing:
- path = (
- conn_or_path
- if isinstance(conn_or_path, (Path, str))
- else "database"
- )
- msg = f"SQLite database not found: {path}"
- raise ValueError(msg)
- timeframe_counts = None
- existing_views: set[str] = set()
- else:
- timeframe_counts = _load_rates_timeframe_counts(conn)
- existing_views = _load_existing_rate_views(conn)
- resolved: list[str] = []
- for target in target_list:
- symbol = cast("str", target.symbol)
- timeframe = target.timeframe_int
- resolved.append(
- _resolve_rate_view_name_from_context(
- symbol=symbol,
- timeframe=timeframe,
- granularity_name=resolve_granularity_name(timeframe),
- timeframe_counts=timeframe_counts,
- existing_views=existing_views,
- require_existing=require_existing,
- ),
- )
- return resolved
- finally:
- if should_close and conn is not None:
- conn.close()
+673
+674
+675
+676
+677
+678
+679
| def resolve_rate_tables(
+ conn_or_path: SqliteConnOrPath | None,
+ targets: Sequence[RateTarget],
+ explicit_tables: Sequence[str] | None = None,
+ *,
+ require_existing: bool = False,
+) -> list[str]:
+ """Resolve SQLite table or view names for rate targets.
+
+ Args:
+ conn_or_path: SQLite database path or open connection. May be None when
+ ``explicit_tables`` is provided, or when ``require_existing`` is
+ False and deterministic default view names are sufficient.
+ targets: Rate targets to resolve.
+ explicit_tables: Optional explicit table or view names. When provided,
+ they are used as-is and must match the number of targets.
+ require_existing: When True, require the database and managed views to
+ exist for each symbol target. Ignored when ``explicit_tables`` is
+ provided.
+
+ Returns:
+ Table or view names aligned with ``targets``.
+
+ Raises:
+ ValueError: If ``targets`` is empty, ``explicit_tables`` length does not
+ match the target count, a target without a symbol is resolved
+ without an explicit table, or ``require_existing`` is True and the
+ database or a managed view is missing.
+ """
+ target_list = list(targets)
+ if not target_list:
+ msg = "At least one rate target is required."
+ raise ValueError(msg)
+ if explicit_tables is not None:
+ tables = list(explicit_tables)
+ if len(tables) != len(target_list):
+ msg = (
+ f"Expected {len(target_list)} explicit table(s) "
+ f"to match the targets, got {len(tables)}."
+ )
+ raise ValueError(msg)
+ return tables
+ if any(target.symbol is None for target in target_list):
+ msg = (
+ "Cannot resolve a rate table for a target without a symbol; "
+ "provide explicit_tables."
+ )
+ raise ValueError(msg)
+ conn, should_close = _open_history_connection(conn_or_path)
+ try:
+ if conn is None:
+ if require_existing:
+ path = (
+ conn_or_path
+ if isinstance(conn_or_path, (Path, str))
+ else "database"
+ )
+ msg = f"SQLite database not found: {path}"
+ raise ValueError(msg)
+ timeframe_counts = None
+ existing_views: set[str] = set()
+ else:
+ timeframe_counts = _load_rates_timeframe_counts(conn)
+ existing_views = _load_existing_rate_views(conn)
+ resolved: list[str] = []
+ for target in target_list:
+ symbol = cast("str", target.symbol)
+ timeframe = target.timeframe_int
+ resolved.append(
+ _resolve_rate_view_name_from_context(
+ symbol=symbol,
+ timeframe=timeframe,
+ granularity_name=resolve_granularity_name(timeframe),
+ timeframe_counts=timeframe_counts,
+ existing_views=existing_views,
+ require_existing=require_existing,
+ ),
+ )
+ return resolved
+ finally:
+ if should_close and conn is not None:
+ conn.close()
|
Source code in mt5cli/history.py
- 420
-421
-422
-423
-424
-425
-426
+ 426
427
428
429
@@ -4611,59 +4661,65 @@ default view name is returned without creating a database file.
469
470
471
-472 | def resolve_rate_view_name(
- conn_or_path: SqliteConnOrPath | None,
- symbol: str,
- granularity: str,
- *,
- require_existing: bool = False,
-) -> str:
- """Resolve the mt5cli-managed rate compatibility view name.
-
- Args:
- conn_or_path: SQLite database path or open connection. When None or a
- non-existing path and ``require_existing`` is False, the deterministic
- default view name is returned without creating a database file.
- symbol: Symbol stored in the normalized ``rates`` table.
- granularity: Timeframe name (for example ``M1``) or integer string.
- require_existing: When True, require the database and a managed view to exist.
-
- Returns:
- View name such as ``rate_EURUSD__1`` or ``rate_EURUSD__M1_1``.
-
- Raises:
- ValueError: If ``require_existing`` is True and the database or view is missing.
- """
- timeframe = parse_timeframe(granularity)
- granularity_name = resolve_granularity_name(timeframe)
- conn, should_close = _open_history_connection(conn_or_path)
- try:
- if conn is None:
- if require_existing:
- path = (
- conn_or_path
- if isinstance(conn_or_path, (Path, str))
- else "database"
- )
- msg = f"SQLite database not found: {path}"
- raise ValueError(msg)
- return build_rate_view_name(
- symbol=symbol,
- granularity=granularity_name,
- granularity_count=1,
- timeframe=timeframe,
- )
- return _resolve_rate_view_name_from_context(
- symbol=symbol,
- timeframe=timeframe,
- granularity_name=granularity_name,
- timeframe_counts=_load_rates_timeframe_counts(conn),
- existing_views=_load_existing_rate_views(conn),
- require_existing=require_existing,
- )
- finally:
- if should_close and conn is not None:
- conn.close()
+472
+473
+474
+475
+476
+477
+478
| def resolve_rate_view_name(
+ conn_or_path: SqliteConnOrPath | None,
+ symbol: str,
+ granularity: str,
+ *,
+ require_existing: bool = False,
+) -> str:
+ """Resolve the mt5cli-managed rate compatibility view name.
+
+ Args:
+ conn_or_path: SQLite database path or open connection. When None or a
+ non-existing path and ``require_existing`` is False, the deterministic
+ default view name is returned without creating a database file.
+ symbol: Symbol stored in the normalized ``rates`` table.
+ granularity: Timeframe name (for example ``M1``) or integer string.
+ require_existing: When True, require the database and a managed view to exist.
+
+ Returns:
+ View name such as ``rate_EURUSD__1`` or ``rate_EURUSD__M1_1``.
+
+ Raises:
+ ValueError: If ``require_existing`` is True and the database or view is missing.
+ """
+ timeframe = parse_timeframe(granularity)
+ granularity_name = resolve_granularity_name(timeframe)
+ conn, should_close = _open_history_connection(conn_or_path)
+ try:
+ if conn is None:
+ if require_existing:
+ path = (
+ conn_or_path
+ if isinstance(conn_or_path, (Path, str))
+ else "database"
+ )
+ msg = f"SQLite database not found: {path}"
+ raise ValueError(msg)
+ return build_rate_view_name(
+ symbol=symbol,
+ granularity=granularity_name,
+ granularity_count=1,
+ timeframe=timeframe,
+ )
+ return _resolve_rate_view_name_from_context(
+ symbol=symbol,
+ timeframe=timeframe,
+ granularity_name=granularity_name,
+ timeframe_counts=_load_rates_timeframe_counts(conn),
+ existing_views=_load_existing_rate_views(conn),
+ require_existing=require_existing,
+ )
+ finally:
+ if should_close and conn is not None:
+ conn.close()
|
Source code in mt5cli/history.py
- 475
-476
-477
-478
-479
-480
-481
+ 481
482
483
484
@@ -4871,60 +4921,66 @@ default view names are returned without creating a database file.
525
526
527
-528 | def resolve_rate_view_names(
- conn_or_path: SqliteConnOrPath | None,
- symbols: Sequence[str],
- granularities: Sequence[str],
- *,
- require_existing: bool = False,
-) -> list[str]:
- """Resolve rate compatibility view names for symbol and granularity pairs.
-
- Args:
- conn_or_path: SQLite database path or open connection. When None or a
- non-existing path and ``require_existing`` is False, deterministic
- default view names are returned without creating a database file.
- symbols: Symbols stored in the normalized ``rates`` table.
- granularities: Timeframe names (for example ``M1``) or integer strings.
- require_existing: When True, require the database and managed views to exist.
-
- Returns:
- View names in row-major order: every ``granularity`` for the first
- symbol, then every granularity for the next symbol, and so on.
- """
- conn, should_close = _open_history_connection(conn_or_path)
- try:
- if conn is None:
- return [
- resolve_rate_view_name(
- conn_or_path,
- symbol,
- granularity,
- require_existing=require_existing,
- )
- for symbol in symbols
- for granularity in granularities
- ]
- timeframe_counts = _load_rates_timeframe_counts(conn)
- existing_views = _load_existing_rate_views(conn)
- resolved: list[str] = []
- for symbol in symbols:
- for granularity in granularities:
- timeframe = parse_timeframe(granularity)
- resolved.append(
- _resolve_rate_view_name_from_context(
- symbol=symbol,
- timeframe=timeframe,
- granularity_name=resolve_granularity_name(timeframe),
- timeframe_counts=timeframe_counts,
- existing_views=existing_views,
- require_existing=require_existing,
- ),
- )
- return resolved
- finally:
- if should_close and conn is not None:
- conn.close()
+528
+529
+530
+531
+532
+533
+534
| def resolve_rate_view_names(
+ conn_or_path: SqliteConnOrPath | None,
+ symbols: Sequence[str],
+ granularities: Sequence[str],
+ *,
+ require_existing: bool = False,
+) -> list[str]:
+ """Resolve rate compatibility view names for symbol and granularity pairs.
+
+ Args:
+ conn_or_path: SQLite database path or open connection. When None or a
+ non-existing path and ``require_existing`` is False, deterministic
+ default view names are returned without creating a database file.
+ symbols: Symbols stored in the normalized ``rates`` table.
+ granularities: Timeframe names (for example ``M1``) or integer strings.
+ require_existing: When True, require the database and managed views to exist.
+
+ Returns:
+ View names in row-major order: every ``granularity`` for the first
+ symbol, then every granularity for the next symbol, and so on.
+ """
+ conn, should_close = _open_history_connection(conn_or_path)
+ try:
+ if conn is None:
+ return [
+ resolve_rate_view_name(
+ conn_or_path,
+ symbol,
+ granularity,
+ require_existing=require_existing,
+ )
+ for symbol in symbols
+ for granularity in granularities
+ ]
+ timeframe_counts = _load_rates_timeframe_counts(conn)
+ existing_views = _load_existing_rate_views(conn)
+ resolved: list[str] = []
+ for symbol in symbols:
+ for granularity in granularities:
+ timeframe = parse_timeframe(granularity)
+ resolved.append(
+ _resolve_rate_view_name_from_context(
+ symbol=symbol,
+ timeframe=timeframe,
+ granularity_name=resolve_granularity_name(timeframe),
+ timeframe_counts=timeframe_counts,
+ existing_views=existing_views,
+ require_existing=require_existing,
+ ),
+ )
+ return resolved
+ finally:
+ if should_close and conn is not None:
+ conn.close()
|
Source code in mt5cli/history.py
- 1900
-1901
-1902
-1903
-1904
-1905
-1906
+ | def write_collected_datasets(
- conn: sqlite3.Connection,
- client: Mt5DataClient,
- symbols: Sequence[str],
- datasets: set[Dataset],
- timeframe: int,
- flags: int,
- date_from: datetime,
- date_to: datetime,
- if_exists: IfExists,
-) -> tuple[set[Dataset], dict[Dataset, set[str]]]:
- """Collect selected datasets and stream each symbol frame into SQLite.
-
- Returns:
- Written datasets and their columns.
- """
- written_columns: dict[Dataset, set[str]] = {}
- written_tables: set[Dataset] = set()
- if Dataset.rates in datasets and write_rates_dataset(
- conn,
- client,
- symbols,
- timeframe,
- date_from,
- date_to,
- if_exists,
- written_columns,
- ):
- written_tables.add(Dataset.rates)
- if Dataset.ticks in datasets and write_ticks_dataset(
- conn,
- client,
- symbols,
- flags,
- date_from,
- date_to,
- if_exists,
- written_columns,
- ):
- written_tables.add(Dataset.ticks)
- if Dataset.history_orders in datasets and write_history_dataset(
- conn,
- client.history_orders_get_as_df,
- Dataset.history_orders,
- symbols,
- date_from,
- date_to,
- if_exists,
- written_columns,
- include_account_events=False,
- ):
- written_tables.add(Dataset.history_orders)
- if Dataset.history_deals in datasets and write_history_dataset(
- conn,
- client.history_deals_get_as_df,
- Dataset.history_deals,
- symbols,
- date_from,
- date_to,
- if_exists,
- written_columns,
- include_account_events=False,
- ):
- written_tables.add(Dataset.history_deals)
- return written_tables, written_columns
+1964
+1965
+1966
+1967
+1968
+1969
+1970
| def write_collected_datasets(
+ conn: sqlite3.Connection,
+ client: Mt5DataClient,
+ symbols: Sequence[str],
+ datasets: set[Dataset],
+ timeframe: int,
+ flags: int,
+ date_from: datetime,
+ date_to: datetime,
+ if_exists: IfExists,
+) -> tuple[set[Dataset], dict[Dataset, set[str]]]:
+ """Collect selected datasets and stream each symbol frame into SQLite.
+
+ Returns:
+ Written datasets and their columns.
+ """
+ written_columns: dict[Dataset, set[str]] = {}
+ written_tables: set[Dataset] = set()
+ if Dataset.rates in datasets and write_rates_dataset(
+ conn,
+ client,
+ symbols,
+ timeframe,
+ date_from,
+ date_to,
+ if_exists,
+ written_columns,
+ ):
+ written_tables.add(Dataset.rates)
+ if Dataset.ticks in datasets and write_ticks_dataset(
+ conn,
+ client,
+ symbols,
+ flags,
+ date_from,
+ date_to,
+ if_exists,
+ written_columns,
+ ):
+ written_tables.add(Dataset.ticks)
+ if Dataset.history_orders in datasets and write_history_dataset(
+ conn,
+ client.history_orders_get_as_df,
+ Dataset.history_orders,
+ symbols,
+ date_from,
+ date_to,
+ if_exists,
+ written_columns,
+ include_account_events=False,
+ ):
+ written_tables.add(Dataset.history_orders)
+ if Dataset.history_deals in datasets and write_history_dataset(
+ conn,
+ client.history_deals_get_as_df,
+ Dataset.history_deals,
+ symbols,
+ date_from,
+ date_to,
+ if_exists,
+ written_columns,
+ include_account_events=False,
+ ):
+ written_tables.add(Dataset.history_deals)
+ return written_tables, written_columns
|
Source code in mt5cli/history.py
- 1517
-1518
-1519
-1520
-1521
-1522
-1523
+ | def write_history_dataset(
- conn: sqlite3.Connection,
- fetch: Callable[..., pd.DataFrame],
- dataset: Dataset,
- symbols: Sequence[str],
- date_from: datetime,
- date_to: datetime,
- if_exists: IfExists,
- written_columns: dict[Dataset, set[str]],
- *,
- include_account_events: bool = False,
-) -> bool:
- """Stream a history dataset into SQLite.
-
- Returns:
- True if the target table was written.
- """
- table_exists = False
- if include_account_events:
- frame = filter_trade_history_frame(
- fetch(date_from=date_from, date_to=date_to),
- symbols,
- include_account_events=True,
- )
- return write_streamed_frame(
- conn,
- frame,
- dataset,
- table_exists,
- if_exists,
- written_columns,
- )
-
- def _fetch_history_frame(sym: str) -> pd.DataFrame:
- return filter_trade_history_frame(
- fetch(date_from=date_from, date_to=date_to, symbol=sym),
- [sym],
- include_account_events=False,
- )
-
- return _stream_symbol_frames(
- conn,
- symbols,
- dataset,
- if_exists,
- written_columns,
- _fetch_history_frame,
- )
+1564
+1565
+1566
+1567
+1568
+1569
+1570
| def write_history_dataset(
+ conn: sqlite3.Connection,
+ fetch: Callable[..., pd.DataFrame],
+ dataset: Dataset,
+ symbols: Sequence[str],
+ date_from: datetime,
+ date_to: datetime,
+ if_exists: IfExists,
+ written_columns: dict[Dataset, set[str]],
+ *,
+ include_account_events: bool = False,
+) -> bool:
+ """Stream a history dataset into SQLite.
+
+ Returns:
+ True if the target table was written.
+ """
+ table_exists = False
+ if include_account_events:
+ frame = filter_trade_history_frame(
+ fetch(date_from=date_from, date_to=date_to),
+ symbols,
+ include_account_events=True,
+ )
+ return write_streamed_frame(
+ conn,
+ frame,
+ dataset,
+ table_exists,
+ if_exists,
+ written_columns,
+ )
+
+ def _fetch_history_frame(sym: str) -> pd.DataFrame:
+ return filter_trade_history_frame(
+ fetch(date_from=date_from, date_to=date_to, symbol=sym),
+ [sym],
+ include_account_events=False,
+ )
+
+ return _stream_symbol_frames(
+ conn,
+ symbols,
+ dataset,
+ if_exists,
+ written_columns,
+ _fetch_history_frame,
+ )
|
Source code in mt5cli/history.py
- 1817
-1818
-1819
-1820
-1821
-1822
-1823
+ | def write_incremental_datasets( # noqa: PLR0913
- conn: sqlite3.Connection,
- client: Mt5DataClient,
- symbols: Sequence[str],
- selected_datasets: set[Dataset],
- resolved_timeframes: list[int],
- resolved_tick_flags: int,
- fallback_start: datetime,
- end_date: datetime,
- *,
- deduplicate: bool,
- create_rate_views: bool,
- with_views: bool,
- include_account_events: bool,
-) -> tuple[set[Dataset], dict[Dataset, set[str]]]:
- """Append selected datasets incrementally and refresh indexes and views.
-
- Returns:
- Written datasets and their columns.
- """
- written_columns: dict[Dataset, set[str]] = {}
- written_tables: set[Dataset] = set()
- dedup_scopes: dict[Dataset, list[DedupScope]] = {}
- if Dataset.rates in selected_datasets:
- _write_incremental_rates(
- conn,
- client,
- symbols,
- resolved_timeframes,
- fallback_start,
- end_date,
- written_columns,
- written_tables,
- dedup_scopes,
- )
- if Dataset.ticks in selected_datasets:
- _write_incremental_ticks(
- conn,
- client,
- symbols,
- resolved_tick_flags,
- fallback_start,
- end_date,
- written_columns,
- written_tables,
- dedup_scopes,
- )
- if Dataset.history_orders in selected_datasets:
- _write_incremental_history_orders(
- conn,
- client,
- symbols,
- fallback_start,
- end_date,
- written_columns,
- written_tables,
- dedup_scopes,
- )
- if Dataset.history_deals in selected_datasets:
- _write_incremental_history_deals(
- conn,
- client,
- symbols,
- fallback_start,
- end_date,
- written_columns,
- written_tables,
- dedup_scopes,
- include_account_events=include_account_events,
- )
- _finalize_incremental_writes(
- conn,
- selected_datasets,
- written_columns,
- written_tables,
- dedup_scopes,
- deduplicate=deduplicate,
- create_rate_views=create_rate_views,
- with_views=with_views,
- )
- return written_tables, written_columns
+1897
+1898
+1899
+1900
+1901
+1902
+1903
| def write_incremental_datasets( # noqa: PLR0913
+ conn: sqlite3.Connection,
+ client: Mt5DataClient,
+ symbols: Sequence[str],
+ selected_datasets: set[Dataset],
+ resolved_timeframes: list[int],
+ resolved_tick_flags: int,
+ fallback_start: datetime,
+ end_date: datetime,
+ *,
+ deduplicate: bool,
+ create_rate_views: bool,
+ with_views: bool,
+ include_account_events: bool,
+) -> tuple[set[Dataset], dict[Dataset, set[str]]]:
+ """Append selected datasets incrementally and refresh indexes and views.
+
+ Returns:
+ Written datasets and their columns.
+ """
+ written_columns: dict[Dataset, set[str]] = {}
+ written_tables: set[Dataset] = set()
+ dedup_scopes: dict[Dataset, list[DedupScope]] = {}
+ if Dataset.rates in selected_datasets:
+ _write_incremental_rates(
+ conn,
+ client,
+ symbols,
+ resolved_timeframes,
+ fallback_start,
+ end_date,
+ written_columns,
+ written_tables,
+ dedup_scopes,
+ )
+ if Dataset.ticks in selected_datasets:
+ _write_incremental_ticks(
+ conn,
+ client,
+ symbols,
+ resolved_tick_flags,
+ fallback_start,
+ end_date,
+ written_columns,
+ written_tables,
+ dedup_scopes,
+ )
+ if Dataset.history_orders in selected_datasets:
+ _write_incremental_history_orders(
+ conn,
+ client,
+ symbols,
+ fallback_start,
+ end_date,
+ written_columns,
+ written_tables,
+ dedup_scopes,
+ )
+ if Dataset.history_deals in selected_datasets:
+ _write_incremental_history_deals(
+ conn,
+ client,
+ symbols,
+ fallback_start,
+ end_date,
+ written_columns,
+ written_tables,
+ dedup_scopes,
+ include_account_events=include_account_events,
+ )
+ _finalize_incremental_writes(
+ conn,
+ selected_datasets,
+ written_columns,
+ written_tables,
+ dedup_scopes,
+ deduplicate=deduplicate,
+ create_rate_views=create_rate_views,
+ with_views=with_views,
+ )
+ return written_tables, written_columns
|
Source code in mt5cli/history.py
- 1442
-1443
-1444
-1445
-1446
-1447
-1448
+ | def write_rates_dataset(
- conn: sqlite3.Connection,
- client: Mt5DataClient,
- symbols: Sequence[str],
- timeframe: int,
- date_from: datetime,
- date_to: datetime,
- if_exists: IfExists,
- written_columns: dict[Dataset, set[str]],
-) -> bool:
- """Stream rates frames into SQLite.
-
- Returns:
- True if the rates table was written.
- """
-
- def _fetch_rates_frame(sym: str) -> pd.DataFrame:
- frame = client.copy_rates_range_as_df(
- symbol=sym,
- timeframe=timeframe,
- date_from=date_from,
- date_to=date_to,
- ).drop(columns=["symbol", "timeframe"], errors="ignore")
- if len(frame.columns) != 0:
- frame.insert(0, "symbol", sym)
- frame.insert(1, "timeframe", timeframe)
- return frame
-
- return _stream_symbol_frames(
- conn,
- symbols,
- Dataset.rates,
- if_exists,
- written_columns,
- _fetch_rates_frame,
- )
+1477
+1478
+1479
+1480
+1481
+1482
+1483
| def write_rates_dataset(
+ conn: sqlite3.Connection,
+ client: Mt5DataClient,
+ symbols: Sequence[str],
+ timeframe: int,
+ date_from: datetime,
+ date_to: datetime,
+ if_exists: IfExists,
+ written_columns: dict[Dataset, set[str]],
+) -> bool:
+ """Stream rates frames into SQLite.
+
+ Returns:
+ True if the rates table was written.
+ """
+
+ def _fetch_rates_frame(sym: str) -> pd.DataFrame:
+ frame = client.copy_rates_range_as_df(
+ symbol=sym,
+ timeframe=timeframe,
+ date_from=date_from,
+ date_to=date_to,
+ ).drop(columns=["symbol", "timeframe"], errors="ignore")
+ if len(frame.columns) != 0:
+ frame.insert(0, "symbol", sym)
+ frame.insert(1, "timeframe", timeframe)
+ return frame
+
+ return _stream_symbol_frames(
+ conn,
+ symbols,
+ Dataset.rates,
+ if_exists,
+ written_columns,
+ _fetch_rates_frame,
+ )
|
Source code in mt5cli/history.py
- 1065
-1066
-1067
-1068
-1069
-1070
-1071
+ | def write_streamed_frame(
- conn: sqlite3.Connection,
- frame: pd.DataFrame,
- dataset: Dataset,
- table_exists: bool,
- if_exists: IfExists,
- written_columns: dict[Dataset, set[str]],
-) -> bool:
- """Write one streamed dataset frame and track table state.
-
- Returns:
- True if the dataset table exists after this write attempt.
- """
- write_mode = IfExists.APPEND if table_exists else if_exists
- if append_dataframe(conn, frame, dataset.table_name, write_mode):
- record_written_columns(written_columns, dataset, frame)
- return True
- return table_exists
+1082
+1083
+1084
+1085
+1086
+1087
+1088
| def write_streamed_frame(
+ conn: sqlite3.Connection,
+ frame: pd.DataFrame,
+ dataset: Dataset,
+ table_exists: bool,
+ if_exists: IfExists,
+ written_columns: dict[Dataset, set[str]],
+) -> bool:
+ """Write one streamed dataset frame and track table state.
+
+ Returns:
+ True if the dataset table exists after this write attempt.
+ """
+ write_mode = IfExists.APPEND if table_exists else if_exists
+ if append_dataframe(conn, frame, dataset.table_name, write_mode):
+ record_written_columns(written_columns, dataset, frame)
+ return True
+ return table_exists
|
Source code in mt5cli/history.py
- 1480
-1481
-1482
-1483
-1484
-1485
-1486
+ | def write_ticks_dataset(
- conn: sqlite3.Connection,
- client: Mt5DataClient,
- symbols: Sequence[str],
- flags: int,
- date_from: datetime,
- date_to: datetime,
- if_exists: IfExists,
- written_columns: dict[Dataset, set[str]],
-) -> bool:
- """Stream ticks frames into SQLite.
-
- Returns:
- True if the ticks table was written.
- """
-
- def _fetch_ticks_frame(sym: str) -> pd.DataFrame:
- frame = client.copy_ticks_range_as_df(
- symbol=sym,
- date_from=date_from,
- date_to=date_to,
- flags=flags,
- ).drop(columns=["symbol"], errors="ignore")
- if len(frame.columns) != 0:
- frame.insert(0, "symbol", sym)
- return frame
-
- return _stream_symbol_frames(
- conn,
- symbols,
- Dataset.ticks,
- if_exists,
- written_columns,
- _fetch_ticks_frame,
- )
+1514
+1515
+1516
+1517
+1518
+1519
+1520
| def write_ticks_dataset(
+ conn: sqlite3.Connection,
+ client: Mt5DataClient,
+ symbols: Sequence[str],
+ flags: int,
+ date_from: datetime,
+ date_to: datetime,
+ if_exists: IfExists,
+ written_columns: dict[Dataset, set[str]],
+) -> bool:
+ """Stream ticks frames into SQLite.
+
+ Returns:
+ True if the ticks table was written.
+ """
+
+ def _fetch_ticks_frame(sym: str) -> pd.DataFrame:
+ frame = client.copy_ticks_range_as_df(
+ symbol=sym,
+ date_from=date_from,
+ date_to=date_to,
+ flags=flags,
+ ).drop(columns=["symbol"], errors="ignore")
+ if len(frame.columns) != 0:
+ frame.insert(0, "symbol", sym)
+ return frame
+
+ return _stream_symbol_frames(
+ conn,
+ symbols,
+ Dataset.ticks,
+ if_exists,
+ written_columns,
+ _fetch_ticks_frame,
+ )
|
mt5cli.sdk.update_history.
|
- Datasets to include (defaults to all).
+ Datasets to include (defaults to rates, history-orders,
+history-deals; pass {Dataset.ticks} to opt in to ticks).
|
@@ -2904,8 +2905,7 @@ when :meth:update runs. Receives the same keyword arguments as
Source code in mt5cli/sdk.py
- 1095
-1096
+ | def __init__(
- self,
- *,
- output: Path | str,
- datasets: set[Dataset] | None = None,
- timeframes: Sequence[int | str] | None = None,
- flags: int | str = "ALL",
- lookback_hours: float = 24.0,
- with_views: bool = False,
- include_account_events: bool = True,
- interval_seconds: float = 0.0,
- suppress_errors: bool = False,
- update_backend: UpdateHistoryBackend | None = None,
-) -> None:
- """Initialize the throttled updater.
-
- Args:
- output: SQLite database path.
- datasets: Datasets to include (defaults to all).
- timeframes: Rate timeframes to update (defaults to all fixed MT5
- timeframes).
- flags: Tick copy flags as integer or name (e.g. ``ALL``).
- lookback_hours: First-run lookback when a table has no prior rows.
- with_views: Create ``cash_events`` and ``positions_reconstructed``
- views.
- include_account_events: Include account-level cash events.
- interval_seconds: Minimum seconds between successful updates. Values
- ``<= 0`` update on every call.
- suppress_errors: When True, recoverable errors (``Mt5TradingError``,
- ``Mt5RuntimeError``, ``sqlite3.Error``, ``ValueError``,
- ``OSError``, and MT5 client capability ``AttributeError`` /
- ``TypeError`` for history API methods) raised during an update
- are swallowed and :meth:`update` returns False without advancing
- the throttle. Other ``AttributeError`` / ``TypeError`` values
- always propagate. When False (default), recoverable errors
- propagate so callers control logging.
- update_backend: Callable invoked instead of :func:`update_history`
- when :meth:`update` runs. Receives the same keyword arguments as
- :func:`update_history` (``client``, ``output``, ``symbols``,
- ``datasets``, ``timeframes``, ``flags``, ``lookback_hours``,
- ``with_views``, ``include_account_events``). Defaults to
- :func:`update_history`.
- """
- self.output = output
- self.datasets = datasets
- self.timeframes = timeframes
- self.flags = flags
- self.lookback_hours = lookback_hours
- self.with_views = with_views
- self.include_account_events = include_account_events
- self.interval_seconds = interval_seconds
- self.suppress_errors = suppress_errors
- self.update_backend = (
- update_history if update_backend is None else update_backend
- )
- self._last_update_monotonic: float | None = None
+1150
+1151
+1152
| def __init__(
+ self,
+ *,
+ output: Path | str,
+ datasets: set[Dataset] | None = None,
+ timeframes: Sequence[int | str] | None = None,
+ flags: int | str = "ALL",
+ lookback_hours: float = 24.0,
+ with_views: bool = False,
+ include_account_events: bool = True,
+ interval_seconds: float = 0.0,
+ suppress_errors: bool = False,
+ update_backend: UpdateHistoryBackend | None = None,
+) -> None:
+ """Initialize the throttled updater.
+
+ Args:
+ output: SQLite database path.
+ datasets: Datasets to include (defaults to rates, history-orders,
+ history-deals; pass ``{Dataset.ticks}`` to opt in to ticks).
+ timeframes: Rate timeframes to update (defaults to all fixed MT5
+ timeframes).
+ flags: Tick copy flags as integer or name (e.g. ``ALL``).
+ lookback_hours: First-run lookback when a table has no prior rows.
+ with_views: Create ``cash_events`` and ``positions_reconstructed``
+ views.
+ include_account_events: Include account-level cash events.
+ interval_seconds: Minimum seconds between successful updates. Values
+ ``<= 0`` update on every call.
+ suppress_errors: When True, recoverable errors (``Mt5TradingError``,
+ ``Mt5RuntimeError``, ``sqlite3.Error``, ``ValueError``,
+ ``OSError``, and MT5 client capability ``AttributeError`` /
+ ``TypeError`` for history API methods) raised during an update
+ are swallowed and :meth:`update` returns False without advancing
+ the throttle. Other ``AttributeError`` / ``TypeError`` values
+ always propagate. When False (default), recoverable errors
+ propagate so callers control logging.
+ update_backend: Callable invoked instead of :func:`update_history`
+ when :meth:`update` runs. Receives the same keyword arguments as
+ :func:`update_history` (``client``, ``output``, ``symbols``,
+ ``datasets``, ``timeframes``, ``flags``, ``lookback_hours``,
+ ``with_views``, ``include_account_events``). Defaults to
+ :func:`update_history`.
+ """
+ self.output = output
+ self.datasets = datasets
+ self.timeframes = timeframes
+ self.flags = flags
+ self.lookback_hours = lookback_hours
+ self.with_views = with_views
+ self.include_account_events = include_account_events
+ self.interval_seconds = interval_seconds
+ self.suppress_errors = suppress_errors
+ self.update_backend = (
+ update_history if update_backend is None else update_backend
+ )
+ self._last_update_monotonic: float | None = None
|
update runs. Receives the same keyword arguments as
Source code in mt5cli/sdk.py
- 1157
-1158
-1159
+ | def should_update(self) -> bool:
- """Return whether enough time has elapsed to run another update.
-
- Returns:
- True when ``interval_seconds <= 0``, when no update has succeeded
- yet, or when at least ``interval_seconds`` have elapsed since the
- last successful update.
- """
- if self.interval_seconds <= 0 or self._last_update_monotonic is None:
- return True
- return (time.monotonic() - self._last_update_monotonic) >= self.interval_seconds
+1167
+1168
+1169
| def should_update(self) -> bool:
+ """Return whether enough time has elapsed to run another update.
+
+ Returns:
+ True when ``interval_seconds <= 0``, when no update has succeeded
+ yet, or when at least ``interval_seconds`` have elapsed since the
+ last successful update.
+ """
+ if self.interval_seconds <= 0 or self._last_update_monotonic is None:
+ return True
+ return (time.monotonic() - self._last_update_monotonic) >= self.interval_seconds
|
Source code in mt5cli/sdk.py
- 1169
-1170
-1171
+ | def update(self, client: Mt5DataClient, symbols: Sequence[str]) -> bool:
- """Run a throttled incremental history update.
-
- Args:
- client: Connected MT5 data client.
- symbols: Symbols to update.
-
- Returns:
- True if an update ran successfully, False if it was throttled or
- (when ``suppress_errors`` is True) failed with a recoverable error.
- When ``suppress_errors`` is False, recoverable update failures
- propagate to the caller.
-
- Raises:
- AttributeError: MT5 client capability mismatch when
- ``suppress_errors`` is False, or any other attribute error.
- TypeError: MT5 client capability mismatch when ``suppress_errors``
- is False, or any other type error.
- """
- if not self.should_update():
- return False
- try:
- _resolve_update_history_request(
- output=self.output,
- symbols=symbols,
- datasets=self.datasets,
- timeframes=self.timeframes,
- flags=self.flags,
- lookback_hours=self.lookback_hours,
- date_to=None,
- )
- self.update_backend(
- client=client,
- output=self.output,
- symbols=symbols,
- datasets=self.datasets,
- timeframes=self.timeframes,
- flags=self.flags,
- lookback_hours=self.lookback_hours,
- with_views=self.with_views,
- include_account_events=self.include_account_events,
- )
- except _RECOVERABLE_HISTORY_UPDATE_ERRORS:
- if self.suppress_errors:
- logger.warning("Suppressed history update error", exc_info=True)
- return False
- raise
- except (AttributeError, TypeError) as exc:
- if self.suppress_errors and _is_mt5_client_capability_error(exc):
- logger.warning("Suppressed history update error", exc_info=True)
- return False
- raise
- self._last_update_monotonic = time.monotonic()
- return True
+1222
+1223
+1224
| def update(self, client: Mt5DataClient, symbols: Sequence[str]) -> bool:
+ """Run a throttled incremental history update.
+
+ Args:
+ client: Connected MT5 data client.
+ symbols: Symbols to update.
+
+ Returns:
+ True if an update ran successfully, False if it was throttled or
+ (when ``suppress_errors`` is True) failed with a recoverable error.
+ When ``suppress_errors`` is False, recoverable update failures
+ propagate to the caller.
+
+ Raises:
+ AttributeError: MT5 client capability mismatch when
+ ``suppress_errors`` is False, or any other attribute error.
+ TypeError: MT5 client capability mismatch when ``suppress_errors``
+ is False, or any other type error.
+ """
+ if not self.should_update():
+ return False
+ try:
+ _resolve_update_history_request(
+ output=self.output,
+ symbols=symbols,
+ datasets=self.datasets,
+ timeframes=self.timeframes,
+ flags=self.flags,
+ lookback_hours=self.lookback_hours,
+ date_to=None,
+ )
+ self.update_backend(
+ client=client,
+ output=self.output,
+ symbols=symbols,
+ datasets=self.datasets,
+ timeframes=self.timeframes,
+ flags=self.flags,
+ lookback_hours=self.lookback_hours,
+ with_views=self.with_views,
+ include_account_events=self.include_account_events,
+ )
+ except _RECOVERABLE_HISTORY_UPDATE_ERRORS:
+ if self.suppress_errors:
+ logger.warning("Suppressed history update error", exc_info=True)
+ return False
+ raise
+ except (AttributeError, TypeError) as exc:
+ if self.suppress_errors and _is_mt5_client_capability_error(exc):
+ logger.warning("Suppressed history update error", exc_info=True)
+ return False
+ raise
+ self._last_update_monotonic = time.monotonic()
+ return True
|
Source code in mt5cli/sdk.py
- | def account_info(*, config: Mt5Config | None = None) -> pd.DataFrame:
- """Return account information."""
- return _make_client(config=config).account_info()
+ | def account_info(*, config: Mt5Config | None = None) -> pd.DataFrame:
+ """Return account information."""
+ return _make_client(config=config).account_info()
|
path, login, password, and serve
|
- Datasets to include (defaults to all).
+ Datasets to include (defaults to rates, history-orders,
+history-deals; pass {Dataset.ticks} to opt in to ticks).
|
@@ -4161,9 +4165,7 @@ to path, login, password, and serve
Source code in mt5cli/sdk.py
- 1225
-1226
-1227
+ | def collect_history(
- output: Path,
- symbols: list[str],
- date_from: datetime | str,
- date_to: datetime | str,
- *,
- datasets: set[Dataset] | None = None,
- timeframe: int | str = 1,
- flags: int | str = "ALL",
- if_exists: IfExists = IfExists.FAIL,
- with_views: bool = False,
- config: Mt5Config | None = None,
-) -> None:
- """Collect historical datasets into a single SQLite database.
-
- Args:
- output: SQLite database path.
- symbols: Symbols to collect.
- date_from: Start date.
- date_to: End date.
- datasets: Datasets to include (defaults to all).
- timeframe: Rates timeframe as integer or name (e.g. ``M1``).
- flags: Tick copy flags as integer or name (e.g. ``ALL``).
- if_exists: Behavior when a target table already exists.
- with_views: Create ``cash_events`` and ``positions_reconstructed`` views.
- config: MT5 connection configuration.
- """
- start = _require_datetime(date_from)
- end = _require_datetime(date_to)
- selected = datasets if datasets is not None else set(Dataset)
- tf = _coerce_timeframe(timeframe)
- tick_flags = _coerce_tick_flags(flags)
- mt5_config = config or build_config()
- with connected_client(mt5_config) as client, sqlite3.connect(output) as conn:
- conn.execute("PRAGMA journal_mode=WAL")
- conn.execute("PRAGMA synchronous=NORMAL")
- written_tables, written_columns = write_collected_datasets(
- conn,
- client,
- symbols,
- selected,
- tf,
- tick_flags,
- start,
- end,
- if_exists,
- )
- create_history_indexes(conn, written_columns)
- if with_views and Dataset.history_deals in written_tables:
- create_cash_events_view(conn, written_columns[Dataset.history_deals])
- create_positions_reconstructed_view(
- conn,
- written_columns[Dataset.history_deals],
- )
- elif with_views:
- logger.warning(
- "--with-views ignored: history_deals table was not written",
- )
- logger.info(
- "Collected %s for %d symbol(s) into %s",
- ", ".join(sorted(ds.value for ds in selected)),
- len(symbols),
- output,
- )
+1288
+1289
+1290
+1291
| def collect_history(
+ output: Path,
+ symbols: list[str],
+ date_from: datetime | str,
+ date_to: datetime | str,
+ *,
+ datasets: set[Dataset] | None = None,
+ timeframe: int | str = 1,
+ flags: int | str = "ALL",
+ if_exists: IfExists = IfExists.FAIL,
+ with_views: bool = False,
+ config: Mt5Config | None = None,
+) -> None:
+ """Collect historical datasets into a single SQLite database.
+
+ Args:
+ output: SQLite database path.
+ symbols: Symbols to collect.
+ date_from: Start date.
+ date_to: End date.
+ datasets: Datasets to include (defaults to rates, history-orders,
+ history-deals; pass ``{Dataset.ticks}`` to opt in to ticks).
+ timeframe: Rates timeframe as integer or name (e.g. ``M1``).
+ flags: Tick copy flags as integer or name (e.g. ``ALL``).
+ if_exists: Behavior when a target table already exists.
+ with_views: Create ``cash_events`` and ``positions_reconstructed`` views.
+ config: MT5 connection configuration.
+ """
+ start = _require_datetime(date_from)
+ end = _require_datetime(date_to)
+ selected = resolve_history_datasets(datasets)
+ tf = _coerce_timeframe(timeframe)
+ tick_flags = _coerce_tick_flags(flags)
+ mt5_config = config or build_config()
+ with connected_client(mt5_config) as client, sqlite3.connect(output) as conn:
+ conn.execute("PRAGMA journal_mode=WAL")
+ conn.execute("PRAGMA synchronous=NORMAL")
+ written_tables, written_columns = write_collected_datasets(
+ conn,
+ client,
+ symbols,
+ selected,
+ tf,
+ tick_flags,
+ start,
+ end,
+ if_exists,
+ )
+ create_history_indexes(conn, written_columns)
+ if with_views and Dataset.history_deals in written_tables:
+ create_cash_events_view(conn, written_columns[Dataset.history_deals])
+ create_positions_reconstructed_view(
+ conn,
+ written_columns[Dataset.history_deals],
+ )
+ elif with_views:
+ logger.warning(
+ "--with-views ignored: history_deals table was not written",
+ )
+ logger.info(
+ "Collected %s for %d symbol(s) into %s",
+ ", ".join(sorted(ds.value for ds in selected)),
+ len(symbols),
+ output,
+ )
|
Source code in mt5cli/sdk.py
- 1864
-1865
-1866
-1867
+ | def collect_latest_closed_rates_by_granularity(
- accounts: Sequence[AccountSpec],
- granularities: Sequence[int | str],
- count: int,
- *,
- start_pos: int = 0,
- base_config: Mt5Config | None = None,
- retry_count: int = 0,
- backoff_base: float = 2.0,
-) -> dict[tuple[str, str], pd.DataFrame]:
- """Collect latest closed rate bars keyed by symbol and granularity name.
-
- Thin wrapper around :func:`collect_latest_closed_rates_for_accounts` that
- rekeys the result by granularity name (for example ``M1``) instead of the
- integer timeframe.
-
- Args:
- accounts: Account groups to read. Each must define at least one symbol.
- granularities: MT5 timeframes as integers or names (for example ``M1``).
- count: Number of closed bars to return per symbol/timeframe.
- start_pos: Initial bar position offset passed to the underlying collector.
- base_config: Optional base configuration whose fields fill any value not
- set on an individual account.
- retry_count: Maximum number of retries after the first attempt. ``0``
- disables retries.
- backoff_base: Base for exponential backoff between retry attempts.
-
- Returns:
- Mapping keyed by ``(symbol, granularity_name)``. Propagates
- ``ValueError`` from :func:`collect_latest_closed_rates_for_accounts`.
- """
- loaded = collect_latest_closed_rates_for_accounts(
- accounts,
- granularities,
- count,
- start_pos=start_pos,
- base_config=base_config,
- retry_count=retry_count,
- backoff_base=backoff_base,
- )
- return {
- (symbol, resolve_granularity_name(timeframe)): frame
- for (symbol, timeframe), frame in loaded.items()
- }
+1907
+1908
+1909
+1910
| def collect_latest_closed_rates_by_granularity(
+ accounts: Sequence[AccountSpec],
+ granularities: Sequence[int | str],
+ count: int,
+ *,
+ start_pos: int = 0,
+ base_config: Mt5Config | None = None,
+ retry_count: int = 0,
+ backoff_base: float = 2.0,
+) -> dict[tuple[str, str], pd.DataFrame]:
+ """Collect latest closed rate bars keyed by symbol and granularity name.
+
+ Thin wrapper around :func:`collect_latest_closed_rates_for_accounts` that
+ rekeys the result by granularity name (for example ``M1``) instead of the
+ integer timeframe.
+
+ Args:
+ accounts: Account groups to read. Each must define at least one symbol.
+ granularities: MT5 timeframes as integers or names (for example ``M1``).
+ count: Number of closed bars to return per symbol/timeframe.
+ start_pos: Initial bar position offset passed to the underlying collector.
+ base_config: Optional base configuration whose fields fill any value not
+ set on an individual account.
+ retry_count: Maximum number of retries after the first attempt. ``0``
+ disables retries.
+ backoff_base: Base for exponential backoff between retry attempts.
+
+ Returns:
+ Mapping keyed by ``(symbol, granularity_name)``. Propagates
+ ``ValueError`` from :func:`collect_latest_closed_rates_for_accounts`.
+ """
+ loaded = collect_latest_closed_rates_for_accounts(
+ accounts,
+ granularities,
+ count,
+ start_pos=start_pos,
+ base_config=base_config,
+ retry_count=retry_count,
+ backoff_base=backoff_base,
+ )
+ return {
+ (symbol, resolve_granularity_name(timeframe)): frame
+ for (symbol, timeframe), frame in loaded.items()
+ }
|
start_pos is 0).
Source code in mt5cli/sdk.py
- 1801
-1802
-1803
-1804
+ | def collect_latest_closed_rates_for_accounts(
- accounts: Sequence[AccountSpec],
- timeframes: Sequence[int | str],
- count: int,
- *,
- start_pos: int = 0,
- base_config: Mt5Config | None = None,
- retry_count: int = 0,
- backoff_base: float = 2.0,
-) -> dict[tuple[str, int], pd.DataFrame]:
- """Collect latest closed rate bars across multiple MT5 account groups.
-
- When ``start_pos`` is ``0`` (the default), MetaTrader 5 includes the
- still-forming current bar as the last row. This helper fetches
- ``count + 1`` bars, drops that bar with :func:`drop_forming_rate_bar`, and
- validates that each resulting frame is non-empty. When ``start_pos`` is
- greater than zero the forming bar is not in range, so only ``count`` bars
- are fetched and no row is dropped.
-
- Wraps :func:`collect_latest_rates_for_accounts_with_retries` for transient
- MT5 error handling.
+1861
+1862
+1863
+1864
| def collect_latest_closed_rates_for_accounts(
+ accounts: Sequence[AccountSpec],
+ timeframes: Sequence[int | str],
+ count: int,
+ *,
+ start_pos: int = 0,
+ base_config: Mt5Config | None = None,
+ retry_count: int = 0,
+ backoff_base: float = 2.0,
+) -> dict[tuple[str, int], pd.DataFrame]:
+ """Collect latest closed rate bars across multiple MT5 account groups.
+
+ When ``start_pos`` is ``0`` (the default), MetaTrader 5 includes the
+ still-forming current bar as the last row. This helper fetches
+ ``count + 1`` bars, drops that bar with :func:`drop_forming_rate_bar`, and
+ validates that each resulting frame is non-empty. When ``start_pos`` is
+ greater than zero the forming bar is not in range, so only ``count`` bars
+ are fetched and no row is dropped.
- Args:
- accounts: Account groups to read. Each must define at least one symbol.
- timeframes: MT5 timeframes as integers or names (for example ``M1``).
- count: Number of closed bars to return per symbol/timeframe.
- start_pos: Initial bar position offset passed to the underlying collector.
- base_config: Optional base configuration whose fields fill any value not
- set on an individual account.
- retry_count: Maximum number of retries after the first attempt. ``0``
- disables retries.
- backoff_base: Base for exponential backoff between retry attempts.
-
- Returns:
- Mapping keyed by ``(symbol, timeframe_int)``.
+ Wraps :func:`collect_latest_rates_for_accounts_with_retries` for transient
+ MT5 error handling.
+
+ Args:
+ accounts: Account groups to read. Each must define at least one symbol.
+ timeframes: MT5 timeframes as integers or names (for example ``M1``).
+ count: Number of closed bars to return per symbol/timeframe.
+ start_pos: Initial bar position offset passed to the underlying collector.
+ base_config: Optional base configuration whose fields fill any value not
+ set on an individual account.
+ retry_count: Maximum number of retries after the first attempt. ``0``
+ disables retries.
+ backoff_base: Base for exponential backoff between retry attempts.
- Raises:
- ValueError: If inputs are invalid, or any series is empty (after
- dropping the still-forming bar when ``start_pos`` is ``0``).
- """
- _require_positive(count, "count")
- _require_non_negative(start_pos, "start_pos")
- fetch_count = count + 1 if start_pos == 0 else count
- loaded = collect_latest_rates_for_accounts_with_retries(
- accounts,
- timeframes,
- fetch_count,
- start_pos=start_pos,
- base_config=base_config,
- retry_count=retry_count,
- backoff_base=backoff_base,
- )
- result: dict[tuple[str, int], pd.DataFrame] = {}
- for key, df_rate in loaded.items():
- closed = drop_forming_rate_bar(df_rate) if start_pos == 0 else df_rate
- if closed.empty:
- symbol, timeframe = key
- msg = f"Rate data is empty for {symbol!r} at timeframe {timeframe}."
- raise ValueError(msg)
- result[key] = closed
- return result
+ Returns:
+ Mapping keyed by ``(symbol, timeframe_int)``.
+
+ Raises:
+ ValueError: If inputs are invalid, or any series is empty (after
+ dropping the still-forming bar when ``start_pos`` is ``0``).
+ """
+ _require_positive(count, "count")
+ _require_non_negative(start_pos, "start_pos")
+ fetch_count = count + 1 if start_pos == 0 else count
+ loaded = collect_latest_rates_for_accounts_with_retries(
+ accounts,
+ timeframes,
+ fetch_count,
+ start_pos=start_pos,
+ base_config=base_config,
+ retry_count=retry_count,
+ backoff_base=backoff_base,
+ )
+ result: dict[tuple[str, int], pd.DataFrame] = {}
+ for key, df_rate in loaded.items():
+ closed = drop_forming_rate_bar(df_rate) if start_pos == 0 else df_rate
+ if closed.empty:
+ symbol, timeframe = key
+ msg = f"Rate data is empty for {symbol!r} at timeframe {timeframe}."
+ raise ValueError(msg)
+ result[key] = closed
+ return result
|
start_pos is 0).
Source code in mt5cli/sdk.py
- |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|