Daichi Narushima
b2bb2ad0a0
* Add public helpers to resolve rate compatibility view names. Expose resolve_rate_view_name and resolve_rate_view_names in mt5cli.history so consumers can derive mt5cli-managed SQLite view names from stored rates metadata without reimplementing the naming rules. Co-authored-by: Cursor <cursoragent@cursor.com> * Add reusable export, tick-window, and margin helpers for downstream tools. Expose SQLite append/dedup export, recent tick retrieval, and minimum margin summary through the SDK and CLI so projects like mteor can depend on mt5cli instead of duplicating MT5 data plumbing. Co-authored-by: Cursor <cursoragent@cursor.com> * Bump version to 0.4.3. Co-authored-by: Cursor <cursoragent@cursor.com> * Address PR review feedback for rate view resolution and SDK helpers. Harden SQLite read-only connections, tighten view discovery, improve recent_ticks fetch efficiency, default SQLite export to append, and expand tests and docs. Co-authored-by: Cursor <cursoragent@cursor.com> * Fix read-only SQLite URI construction on Windows. Use Path.as_uri() so encoded file URIs work cross-platform with mode=ro. Co-authored-by: Cursor <cursoragent@cursor.com> --------- Co-authored-by: Cursor <cursoragent@cursor.com>
1775 lines
68 KiB
Python
1775 lines
68 KiB
Python
"""Tests for mt5cli.history module."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
import sqlite3
|
|
from datetime import UTC, datetime
|
|
from typing import TYPE_CHECKING
|
|
from unittest.mock import MagicMock
|
|
|
|
import pandas as pd
|
|
import pytest
|
|
|
|
if TYPE_CHECKING:
|
|
from pathlib import Path
|
|
|
|
from mt5cli.history import (
|
|
DEFAULT_HISTORY_TIMEFRAMES,
|
|
append_dataframe,
|
|
augment_written_columns_from_sqlite,
|
|
build_rate_view_name,
|
|
create_cash_events_view,
|
|
create_history_indexes,
|
|
create_positions_reconstructed_view,
|
|
create_rate_compatibility_views,
|
|
deduplicate_history_tables,
|
|
drop_duplicates_in_table,
|
|
filter_incremental_history_deals_frame,
|
|
filter_trade_history_frame,
|
|
get_history_deals_account_event_start_datetime,
|
|
get_incremental_start_datetime,
|
|
get_table_columns,
|
|
load_incremental_start_datetimes,
|
|
parse_sqlite_timestamp,
|
|
quote_sqlite_identifier,
|
|
record_written_columns,
|
|
resolve_granularity_name,
|
|
resolve_history_datasets,
|
|
resolve_history_tick_flags,
|
|
resolve_history_timeframes,
|
|
resolve_rate_view_name,
|
|
resolve_rate_view_names,
|
|
write_collected_datasets,
|
|
write_history_dataset,
|
|
write_incremental_datasets,
|
|
write_rates_dataset,
|
|
write_streamed_frame,
|
|
)
|
|
from mt5cli.utils import TIMEFRAME_MAP, Dataset, IfExists
|
|
|
|
|
|
class TestResolveRateViewName:
|
|
"""Tests for resolve_rate_view_name and resolve_rate_view_names."""
|
|
|
|
def test_missing_database_path_does_not_create_file(self, tmp_path: Path) -> None:
|
|
"""Test resolving against a missing path does not create a database."""
|
|
db_path = tmp_path / "missing.db"
|
|
assert resolve_rate_view_name(db_path, "EURUSD", "M1") == "rate_EURUSD__1"
|
|
assert not db_path.exists()
|
|
|
|
def test_no_rates_table_falls_back_to_single_timeframe_name(
|
|
self,
|
|
tmp_path: Path,
|
|
) -> None:
|
|
"""Test databases without a rates table use single-timeframe naming."""
|
|
db_path = tmp_path / "no-rates.db"
|
|
with sqlite3.connect(db_path) as conn:
|
|
conn.execute("CREATE TABLE ticks(symbol TEXT, time TEXT)")
|
|
assert resolve_rate_view_name(db_path, "EURUSD", "M1") == "rate_EURUSD__1"
|
|
|
|
def test_single_timeframe_for_one_symbol(self, tmp_path: Path) -> None:
|
|
"""Test one stored timeframe resolves to the short view name."""
|
|
db_path = tmp_path / "single-timeframe.db"
|
|
with sqlite3.connect(db_path) as conn:
|
|
conn.execute(
|
|
"CREATE TABLE rates("
|
|
" symbol TEXT, timeframe INTEGER, time TEXT, close REAL)",
|
|
)
|
|
conn.execute(
|
|
"INSERT INTO rates(symbol, timeframe, time, close) VALUES (?, ?, ?, ?)",
|
|
("EURUSD", 1, "2024-01-01T00:00:00+00:00", 1.0),
|
|
)
|
|
create_rate_compatibility_views(conn)
|
|
assert resolve_rate_view_name(db_path, "EURUSD", "M1") == "rate_EURUSD__1"
|
|
|
|
def test_multiple_timeframes_for_one_symbol(self, tmp_path: Path) -> None:
|
|
"""Test multiple stored timeframes resolve to disambiguated view names."""
|
|
db_path = tmp_path / "multi-timeframe.db"
|
|
with sqlite3.connect(db_path) as conn:
|
|
conn.execute(
|
|
"CREATE TABLE rates("
|
|
" symbol TEXT, timeframe INTEGER, time TEXT, close REAL)",
|
|
)
|
|
conn.executemany(
|
|
"INSERT INTO rates(symbol, timeframe, time, close) VALUES (?, ?, ?, ?)",
|
|
[
|
|
("EURUSD", 1, "2024-01-01T00:00:00+00:00", 1.0),
|
|
("EURUSD", TIMEFRAME_MAP["H1"], "2024-01-01T01:00:00+00:00", 1.1),
|
|
],
|
|
)
|
|
create_rate_compatibility_views(conn)
|
|
assert resolve_rate_view_name(db_path, "EURUSD", "M1") == "rate_EURUSD__M1_1"
|
|
assert (
|
|
resolve_rate_view_name(db_path, "EURUSD", "H1") == "rate_EURUSD__H1_16385"
|
|
)
|
|
|
|
def test_prefers_multi_name_when_both_candidate_views_exist(
|
|
self,
|
|
tmp_path: Path,
|
|
) -> None:
|
|
"""Test multi-timeframe metadata wins over stale single-timeframe views."""
|
|
db_path = tmp_path / "stale-and-current-views.db"
|
|
with sqlite3.connect(db_path) as conn:
|
|
conn.execute(
|
|
"CREATE TABLE rates("
|
|
" symbol TEXT, timeframe INTEGER, time TEXT, close REAL)",
|
|
)
|
|
conn.executemany(
|
|
"INSERT INTO rates(symbol, timeframe, time, close) VALUES (?, ?, ?, ?)",
|
|
[
|
|
("EURUSD", 1, "2024-01-01T00:00:00+00:00", 1.0),
|
|
("EURUSD", TIMEFRAME_MAP["H1"], "2024-01-01T01:00:00+00:00", 1.1),
|
|
],
|
|
)
|
|
conn.execute(
|
|
'CREATE VIEW "rate_EURUSD__1" AS'
|
|
" SELECT time, close FROM rates"
|
|
" WHERE symbol = 'EURUSD' AND timeframe = 1",
|
|
)
|
|
conn.execute(
|
|
'CREATE VIEW "rate_EURUSD__M1_1" AS'
|
|
" SELECT time, close FROM rates"
|
|
" WHERE symbol = 'EURUSD' AND timeframe = 1",
|
|
)
|
|
assert resolve_rate_view_name(db_path, "EURUSD", "M1") == "rate_EURUSD__M1_1"
|
|
|
|
def test_prefers_existing_view_when_metadata_unavailable(
|
|
self,
|
|
tmp_path: Path,
|
|
) -> None:
|
|
"""Test an existing managed view is preferred without rates metadata."""
|
|
db_path = tmp_path / "view-only.db"
|
|
with sqlite3.connect(db_path) as conn:
|
|
conn.execute("CREATE TABLE ticks(symbol TEXT, time TEXT)")
|
|
conn.execute('CREATE VIEW "rate_EURUSD__M1_1" AS SELECT 1 AS close')
|
|
assert resolve_rate_view_name(db_path, "EURUSD", "M1") == "rate_EURUSD__M1_1"
|
|
|
|
def test_symbol_absent_from_rates_metadata_uses_candidate_pair(
|
|
self,
|
|
tmp_path: Path,
|
|
) -> None:
|
|
"""Test symbols missing from rates metadata still resolve known views."""
|
|
db_path = tmp_path / "other-symbol-only.db"
|
|
with sqlite3.connect(db_path) as conn:
|
|
conn.execute(
|
|
"CREATE TABLE rates("
|
|
" symbol TEXT, timeframe INTEGER, time TEXT, close REAL)",
|
|
)
|
|
conn.execute(
|
|
"INSERT INTO rates(symbol, timeframe, time, close) VALUES (?, ?, ?, ?)",
|
|
("GBPUSD", 1, "2024-01-01T00:00:00+00:00", 1.0),
|
|
)
|
|
conn.execute(
|
|
'CREATE VIEW "rate_EURUSD__1" AS'
|
|
" SELECT time, close FROM rates"
|
|
" WHERE symbol = 'EURUSD' AND timeframe = 1",
|
|
)
|
|
assert resolve_rate_view_name(db_path, "EURUSD", "M1") == "rate_EURUSD__1"
|
|
|
|
def test_ignores_non_compatibility_rate_views(self, tmp_path: Path) -> None:
|
|
"""Test unrelated rate_* views without the __ separator are ignored."""
|
|
db_path = tmp_path / "summary-view.db"
|
|
with sqlite3.connect(db_path) as conn:
|
|
conn.execute("CREATE TABLE ticks(symbol TEXT, time TEXT)")
|
|
conn.execute('CREATE VIEW "rate_summary" AS SELECT 1 AS close')
|
|
assert resolve_rate_view_name(db_path, "EURUSD", "M1") == "rate_EURUSD__1"
|
|
|
|
def test_invalid_granularity_propagates_value_error(self, tmp_path: Path) -> None:
|
|
"""Test invalid granularities raise ValueError from parse_timeframe."""
|
|
with pytest.raises(ValueError, match="Invalid timeframe"):
|
|
resolve_rate_view_name(tmp_path / "unused.db", "EURUSD", "BAD")
|
|
with pytest.raises(ValueError, match="Invalid timeframe"):
|
|
resolve_rate_view_names(tmp_path / "unused.db", ["EURUSD"], ["BAD"])
|
|
|
|
def test_resolve_rate_view_names_for_multiple_pairs(self, tmp_path: Path) -> None:
|
|
"""Test batch resolution returns row-major symbol/granularity pairs."""
|
|
db_path = tmp_path / "batch-resolve.db"
|
|
with sqlite3.connect(db_path) as conn:
|
|
conn.execute(
|
|
"CREATE TABLE rates("
|
|
" symbol TEXT, timeframe INTEGER, time TEXT, close REAL)",
|
|
)
|
|
conn.executemany(
|
|
"INSERT INTO rates(symbol, timeframe, time, close) VALUES (?, ?, ?, ?)",
|
|
[
|
|
("EURUSD", 1, "2024-01-01T00:00:00+00:00", 1.0),
|
|
("EURUSD", TIMEFRAME_MAP["H1"], "2024-01-01T01:00:00+00:00", 1.1),
|
|
("GBPUSD", 1, "2024-01-01T00:00:00+00:00", 1.2),
|
|
],
|
|
)
|
|
create_rate_compatibility_views(conn)
|
|
assert resolve_rate_view_names(
|
|
db_path,
|
|
["EURUSD", "GBPUSD"],
|
|
["M1", "H1"],
|
|
) == [
|
|
"rate_EURUSD__M1_1",
|
|
"rate_EURUSD__H1_16385",
|
|
"rate_GBPUSD__1",
|
|
"rate_GBPUSD__16385",
|
|
]
|
|
|
|
@pytest.mark.parametrize(
|
|
"symbol",
|
|
["EUR/USD", "US500.cash", "#US500"],
|
|
)
|
|
def test_supports_broker_specific_symbols(
|
|
self,
|
|
tmp_path: Path,
|
|
symbol: str,
|
|
) -> None:
|
|
"""Test broker-specific symbols resolve to safely created view names."""
|
|
db_path = tmp_path / "broker-symbol-resolve.db"
|
|
with sqlite3.connect(db_path) as conn:
|
|
conn.execute(
|
|
"CREATE TABLE rates("
|
|
" symbol TEXT, timeframe INTEGER, time TEXT, close REAL)",
|
|
)
|
|
conn.execute(
|
|
"INSERT INTO rates(symbol, timeframe, time, close) VALUES (?, ?, ?, ?)",
|
|
(symbol, 1, "2024-01-01T00:00:00+00:00", 1.0),
|
|
)
|
|
create_rate_compatibility_views(conn)
|
|
assert resolve_rate_view_name(db_path, symbol, "M1") == build_rate_view_name(
|
|
symbol=symbol,
|
|
granularity="M1",
|
|
granularity_count=1,
|
|
timeframe=1,
|
|
)
|
|
|
|
def test_accepts_open_sqlite_connection(self, tmp_path: Path) -> None:
|
|
"""Test resolver accepts an already-open SQLite connection."""
|
|
db_path = tmp_path / "open-connection.db"
|
|
with sqlite3.connect(db_path) as conn:
|
|
conn.execute(
|
|
"CREATE TABLE rates("
|
|
" symbol TEXT, timeframe INTEGER, time TEXT, close REAL)",
|
|
)
|
|
conn.execute(
|
|
"INSERT INTO rates(symbol, timeframe, time, close) VALUES (?, ?, ?, ?)",
|
|
("EURUSD", 1, "2024-01-01T00:00:00+00:00", 1.0),
|
|
)
|
|
create_rate_compatibility_views(conn)
|
|
assert resolve_rate_view_name(conn, "EURUSD", "M1") == "rate_EURUSD__1"
|
|
|
|
def test_require_existing_raises_when_database_missing(
|
|
self,
|
|
tmp_path: Path,
|
|
) -> None:
|
|
"""Test strict mode rejects missing database paths."""
|
|
db_path = tmp_path / "missing.db"
|
|
with pytest.raises(ValueError, match="SQLite database not found"):
|
|
resolve_rate_view_name(
|
|
db_path,
|
|
"EURUSD",
|
|
"M1",
|
|
require_existing=True,
|
|
)
|
|
with pytest.raises(ValueError, match="SQLite database not found"):
|
|
resolve_rate_view_names(
|
|
db_path,
|
|
["EURUSD"],
|
|
["M1"],
|
|
require_existing=True,
|
|
)
|
|
|
|
def test_require_existing_raises_when_view_missing(self, tmp_path: Path) -> None:
|
|
"""Test strict mode rejects databases without matching rate views."""
|
|
db_path = tmp_path / "no-view.db"
|
|
with sqlite3.connect(db_path) as conn:
|
|
conn.execute("CREATE TABLE ticks(symbol TEXT, time TEXT)")
|
|
with pytest.raises(ValueError, match="No rate compatibility view exists"):
|
|
resolve_rate_view_name(
|
|
db_path,
|
|
"EURUSD",
|
|
"M1",
|
|
require_existing=True,
|
|
)
|
|
with pytest.raises(ValueError, match="No rate compatibility view exists"):
|
|
resolve_rate_view_names(
|
|
db_path,
|
|
["EURUSD"],
|
|
["M1"],
|
|
require_existing=True,
|
|
)
|
|
|
|
def test_require_existing_returns_existing_view(self, tmp_path: Path) -> None:
|
|
"""Test strict mode returns a view when one exists."""
|
|
db_path = tmp_path / "existing-view.db"
|
|
with sqlite3.connect(db_path) as conn:
|
|
conn.execute(
|
|
"CREATE TABLE rates("
|
|
" symbol TEXT, timeframe INTEGER, time TEXT, close REAL)",
|
|
)
|
|
conn.execute(
|
|
"INSERT INTO rates(symbol, timeframe, time, close) VALUES (?, ?, ?, ?)",
|
|
("EURUSD", 1, "2024-01-01T00:00:00+00:00", 1.0),
|
|
)
|
|
create_rate_compatibility_views(conn)
|
|
assert (
|
|
resolve_rate_view_name(
|
|
db_path,
|
|
"EURUSD",
|
|
"M1",
|
|
require_existing=True,
|
|
)
|
|
== "rate_EURUSD__1"
|
|
)
|
|
assert resolve_rate_view_names(
|
|
db_path,
|
|
["EURUSD"],
|
|
["M1"],
|
|
require_existing=True,
|
|
) == ["rate_EURUSD__1"]
|
|
|
|
|
|
class TestQuoteSqliteIdentifier:
|
|
"""Tests for quote_sqlite_identifier."""
|
|
|
|
@pytest.mark.parametrize(
|
|
"symbol",
|
|
["EUR/USD", "US500.cash", "#US500"],
|
|
)
|
|
def test_quotes_broker_specific_symbols(self, symbol: str) -> None:
|
|
"""Test broker-specific symbols are safely quoted."""
|
|
quoted = quote_sqlite_identifier(f"rate_{symbol}")
|
|
assert quoted.startswith('"')
|
|
assert quoted.endswith('"')
|
|
|
|
|
|
class TestResolveHistorySettings:
|
|
"""Tests for history dataset and timeframe resolution."""
|
|
|
|
def test_resolve_history_datasets_defaults_and_empty(self) -> None:
|
|
"""Test dataset resolution distinguishes None from empty selection."""
|
|
assert resolve_history_datasets(None) == set(Dataset)
|
|
assert resolve_history_datasets(set()) == set()
|
|
|
|
def test_resolve_history_timeframes_defaults(self) -> None:
|
|
"""Test default timeframes include all fixed MT5 values."""
|
|
resolved = resolve_history_timeframes(None)
|
|
assert len(resolved) == len(DEFAULT_HISTORY_TIMEFRAMES)
|
|
assert 1 in resolved
|
|
assert TIMEFRAME_MAP["H1"] in resolved
|
|
|
|
def test_resolve_history_timeframes_deduplicates_aliases(self) -> None:
|
|
"""Test duplicate aliases for the same timeframe are deduplicated."""
|
|
assert resolve_history_timeframes(["M1", "1", "H1"]) == [1, TIMEFRAME_MAP["H1"]]
|
|
|
|
def test_resolve_history_tick_flags(self) -> None:
|
|
"""Test tick flag resolution."""
|
|
assert resolve_history_tick_flags("ALL") == 1
|
|
assert resolve_history_tick_flags(2) == 2
|
|
|
|
def test_resolve_granularity_name_falls_back_to_integer(self) -> None:
|
|
"""Test unknown timeframe constants fall back to integer text."""
|
|
assert resolve_granularity_name(999) == "999"
|
|
assert resolve_granularity_name(1) == "M1"
|
|
|
|
|
|
class TestParseSqliteTimestamp:
|
|
"""Tests for parse_sqlite_timestamp."""
|
|
|
|
def test_parses_iso_and_pandas_strings(self) -> None:
|
|
"""Test ISO, pandas-compatible, numeric, and datetime values."""
|
|
assert parse_sqlite_timestamp(None) is None
|
|
assert parse_sqlite_timestamp(1_704_067_200) == datetime(2024, 1, 1, tzinfo=UTC)
|
|
assert parse_sqlite_timestamp(
|
|
datetime.fromisoformat("2024-01-01T00:00:00"),
|
|
) == datetime(2024, 1, 1, tzinfo=UTC)
|
|
assert parse_sqlite_timestamp("Jan 1 2024") == datetime(2024, 1, 1, tzinfo=UTC)
|
|
assert parse_sqlite_timestamp("not-a-datetime") is None
|
|
assert parse_sqlite_timestamp(object()) is None
|
|
|
|
|
|
class TestIncrementalStart:
|
|
"""Tests for get_incremental_start_datetime."""
|
|
|
|
def test_uses_max_time_scoped_by_symbol_and_timeframe(
|
|
self,
|
|
tmp_path: Path,
|
|
) -> None:
|
|
"""Test rates increment is scoped by symbol and timeframe."""
|
|
db_path = tmp_path / "scoped-rates.db"
|
|
fallback = datetime(2024, 1, 1, tzinfo=UTC)
|
|
with sqlite3.connect(db_path) as conn:
|
|
conn.execute(
|
|
"CREATE TABLE rates("
|
|
" symbol TEXT, timeframe INTEGER, time TEXT, open REAL)",
|
|
)
|
|
conn.executemany(
|
|
"INSERT INTO rates(symbol, timeframe, time, open) VALUES (?, ?, ?, ?)",
|
|
[
|
|
("EURUSD", 1, "2024-01-02T00:00:00+00:00", 1.0),
|
|
("EURUSD", 16385, "2024-01-03T00:00:00+00:00", 1.1),
|
|
("GBPUSD", 1, "2024-01-04T00:00:00+00:00", 1.2),
|
|
],
|
|
)
|
|
assert get_incremental_start_datetime(
|
|
conn,
|
|
Dataset.rates,
|
|
symbol="EURUSD",
|
|
timeframe=1,
|
|
fallback_start=fallback,
|
|
) == datetime(2024, 1, 2, tzinfo=UTC)
|
|
assert get_incremental_start_datetime(
|
|
conn,
|
|
Dataset.rates,
|
|
symbol="EURUSD",
|
|
timeframe=16385,
|
|
fallback_start=fallback,
|
|
) == datetime(2024, 1, 3, tzinfo=UTC)
|
|
|
|
def test_load_incremental_start_datetimes_batches_rates(
|
|
self, tmp_path: Path
|
|
) -> None:
|
|
"""Test grouped rates resume query returns all symbol/timeframe pairs."""
|
|
fallback = datetime(2024, 1, 1, tzinfo=UTC)
|
|
with sqlite3.connect(tmp_path / "batch-rates.db") as conn:
|
|
conn.execute(
|
|
"CREATE TABLE rates("
|
|
" symbol TEXT, timeframe INTEGER, time TEXT, open REAL)",
|
|
)
|
|
conn.executemany(
|
|
"INSERT INTO rates(symbol, timeframe, time, open) VALUES (?, ?, ?, ?)",
|
|
[
|
|
("EURUSD", 1, "2024-01-02T00:00:00+00:00", 1.0),
|
|
("GBPUSD", 1, "2024-01-03T00:00:00+00:00", 1.1),
|
|
],
|
|
)
|
|
starts = load_incremental_start_datetimes(
|
|
conn,
|
|
Dataset.rates,
|
|
symbols=["EURUSD", "GBPUSD"],
|
|
timeframes=[1],
|
|
fallback_start=fallback,
|
|
)
|
|
assert starts["EURUSD", 1] == datetime(2024, 1, 2, tzinfo=UTC)
|
|
assert starts["GBPUSD", 1] == datetime(2024, 1, 3, tzinfo=UTC)
|
|
|
|
def test_load_incremental_start_datetimes_requires_timeframe_column(
|
|
self,
|
|
tmp_path: Path,
|
|
) -> None:
|
|
"""Test rates tables without timeframe fail fast during incremental resume."""
|
|
fallback = datetime(2024, 1, 1, tzinfo=UTC)
|
|
with sqlite3.connect(tmp_path / "legacy-rates.db") as conn:
|
|
conn.execute("CREATE TABLE rates(symbol TEXT, time TEXT, open REAL)")
|
|
conn.execute(
|
|
"INSERT INTO rates(symbol, time, open) VALUES (?, ?, ?)",
|
|
("EURUSD", "2024-01-02T00:00:00+00:00", 1.0),
|
|
)
|
|
with pytest.raises(ValueError, match="missing: timeframe") as exc_info:
|
|
load_incremental_start_datetimes(
|
|
conn,
|
|
Dataset.rates,
|
|
symbols=["EURUSD"],
|
|
timeframes=[1],
|
|
fallback_start=fallback,
|
|
)
|
|
assert "timeframe" in str(exc_info.value)
|
|
|
|
def test_load_incremental_start_datetimes_requires_symbol_column(
|
|
self,
|
|
tmp_path: Path,
|
|
) -> None:
|
|
"""Test rates tables without symbol fail fast during incremental resume."""
|
|
fallback = datetime(2024, 1, 1, tzinfo=UTC)
|
|
with sqlite3.connect(tmp_path / "rates-no-symbol.db") as conn:
|
|
conn.execute(
|
|
"CREATE TABLE rates(timeframe INTEGER, time TEXT, open REAL)",
|
|
)
|
|
with pytest.raises(ValueError, match="missing: symbol") as exc_info:
|
|
load_incremental_start_datetimes(
|
|
conn,
|
|
Dataset.rates,
|
|
symbols=["EURUSD"],
|
|
timeframes=[1],
|
|
fallback_start=fallback,
|
|
)
|
|
assert "symbol" in str(exc_info.value)
|
|
|
|
def test_load_incremental_start_datetimes_requires_time_column(
|
|
self,
|
|
tmp_path: Path,
|
|
) -> None:
|
|
"""Test rates tables without time fail fast during incremental resume."""
|
|
fallback = datetime(2024, 1, 1, tzinfo=UTC)
|
|
with sqlite3.connect(tmp_path / "rates-no-time.db") as conn:
|
|
conn.execute(
|
|
"CREATE TABLE rates(symbol TEXT, timeframe INTEGER, open REAL)",
|
|
)
|
|
with pytest.raises(ValueError, match="missing: time") as exc_info:
|
|
load_incremental_start_datetimes(
|
|
conn,
|
|
Dataset.rates,
|
|
symbols=["EURUSD"],
|
|
timeframes=[1],
|
|
fallback_start=fallback,
|
|
)
|
|
assert "time" in str(exc_info.value)
|
|
|
|
def test_load_incremental_start_datetimes_rejects_unrelated_rates_columns(
|
|
self,
|
|
tmp_path: Path,
|
|
) -> None:
|
|
"""Test rates tables with only unrelated columns fail fast."""
|
|
fallback = datetime(2024, 1, 1, tzinfo=UTC)
|
|
with sqlite3.connect(tmp_path / "rates-open-only.db") as conn:
|
|
conn.execute("CREATE TABLE rates(open REAL)")
|
|
with pytest.raises(ValueError, match="missing:") as exc_info:
|
|
load_incremental_start_datetimes(
|
|
conn,
|
|
Dataset.rates,
|
|
symbols=["EURUSD"],
|
|
timeframes=[1],
|
|
fallback_start=fallback,
|
|
)
|
|
message = str(exc_info.value)
|
|
assert "symbol" in message
|
|
assert "timeframe" in message
|
|
assert "time" in message
|
|
|
|
def test_load_incremental_start_skips_unparseable_max_time(
|
|
self,
|
|
tmp_path: Path,
|
|
) -> None:
|
|
"""Test grouped resume ignores rows whose MAX(time) cannot be parsed."""
|
|
fallback = datetime(2024, 1, 1, tzinfo=UTC)
|
|
with sqlite3.connect(tmp_path / "bad-max-time.db") as conn:
|
|
conn.execute("CREATE TABLE ticks(symbol TEXT, time TEXT)")
|
|
conn.execute(
|
|
"INSERT INTO ticks(symbol, time) VALUES (?, ?)",
|
|
("EURUSD", "not-a-datetime"),
|
|
)
|
|
starts = load_incremental_start_datetimes(
|
|
conn,
|
|
Dataset.ticks,
|
|
symbols=["EURUSD"],
|
|
fallback_start=fallback,
|
|
)
|
|
assert starts["EURUSD", None] == fallback
|
|
|
|
def test_load_incremental_start_uses_table_max_without_symbol_column(
|
|
self,
|
|
tmp_path: Path,
|
|
) -> None:
|
|
"""Test grouped resume uses table-wide MAX(time) without symbol column."""
|
|
fallback = datetime(2024, 1, 1, tzinfo=UTC)
|
|
with sqlite3.connect(tmp_path / "batch-no-symbol.db") as conn:
|
|
conn.execute("CREATE TABLE ticks(time TEXT)")
|
|
conn.execute(
|
|
"INSERT INTO ticks(time) VALUES (?)",
|
|
("2024-01-02T00:00:00+00:00",),
|
|
)
|
|
starts = load_incremental_start_datetimes(
|
|
conn,
|
|
Dataset.ticks,
|
|
symbols=["EURUSD", "GBPUSD"],
|
|
fallback_start=fallback,
|
|
)
|
|
expected = datetime(2024, 1, 2, tzinfo=UTC)
|
|
assert starts["EURUSD", None] == expected
|
|
assert starts["GBPUSD", None] == expected
|
|
|
|
def test_load_incremental_start_skips_unparseable_rates_max_time(
|
|
self,
|
|
tmp_path: Path,
|
|
) -> None:
|
|
"""Test grouped rates resume ignores unparseable MAX(time) values."""
|
|
fallback = datetime(2024, 1, 1, tzinfo=UTC)
|
|
with sqlite3.connect(tmp_path / "bad-rates-max-time.db") as conn:
|
|
conn.execute(
|
|
"CREATE TABLE rates("
|
|
" symbol TEXT, timeframe INTEGER, time TEXT, open REAL)",
|
|
)
|
|
conn.execute(
|
|
"INSERT INTO rates(symbol, timeframe, time, open) VALUES (?, ?, ?, ?)",
|
|
("EURUSD", 1, "not-a-datetime", 1.0),
|
|
)
|
|
starts = load_incremental_start_datetimes(
|
|
conn,
|
|
Dataset.rates,
|
|
symbols=["EURUSD"],
|
|
timeframes=[1],
|
|
fallback_start=fallback,
|
|
)
|
|
assert starts["EURUSD", 1] == fallback
|
|
|
|
|
|
class TestDeduplication:
|
|
"""Tests for SQLite deduplication helpers."""
|
|
|
|
def test_append_dedup_keeps_latest_rowid(self, tmp_path: Path) -> None:
|
|
"""Test deduplication keeps the latest ROWID for stable keys."""
|
|
db_path = tmp_path / "dedup.db"
|
|
with sqlite3.connect(db_path) as conn:
|
|
conn.execute(
|
|
"CREATE TABLE rates("
|
|
" symbol TEXT, timeframe INTEGER, time TEXT, open REAL)",
|
|
)
|
|
conn.executemany(
|
|
"INSERT INTO rates(symbol, timeframe, time, open) VALUES (?, ?, ?, ?)",
|
|
[
|
|
("EURUSD", 1, "2024-01-01T00:00:00+00:00", 1.0),
|
|
("EURUSD", 1, "2024-01-01T00:00:00+00:00", 9.9),
|
|
],
|
|
)
|
|
deduplicate_history_tables(
|
|
conn,
|
|
{Dataset.rates: {"symbol", "timeframe", "time", "open"}},
|
|
{Dataset.rates},
|
|
)
|
|
assert conn.execute("SELECT COUNT(*) FROM rates").fetchone() == (1,)
|
|
assert conn.execute("SELECT open FROM rates").fetchone() == (9.9,)
|
|
|
|
def test_drop_duplicates_rejects_invalid_identifiers(self) -> None:
|
|
"""Test invalid table or column names raise ValueError."""
|
|
cursor = sqlite3.connect(":memory:").cursor()
|
|
with pytest.raises(ValueError, match="Invalid table name"):
|
|
drop_duplicates_in_table(cursor, "bad table", ["id"])
|
|
with pytest.raises(ValueError, match="Invalid column names"):
|
|
drop_duplicates_in_table(cursor, "rates", ["bad column"])
|
|
|
|
@pytest.mark.parametrize(
|
|
("dataset", "table_sql", "insert_sql", "rows", "columns"),
|
|
[
|
|
(
|
|
Dataset.ticks,
|
|
(
|
|
"CREATE TABLE ticks("
|
|
" symbol TEXT, time_msc INTEGER, time TEXT, bid REAL)"
|
|
),
|
|
"INSERT INTO ticks(symbol, time_msc, time, bid) VALUES (?, ?, ?, ?)",
|
|
[
|
|
("EURUSD", 1, "2024-01-01T00:00:00+00:00", 1.0),
|
|
("EURUSD", 1, "2024-01-01T00:00:00+00:00", 9.9),
|
|
],
|
|
{"symbol", "time_msc", "time", "bid"},
|
|
),
|
|
(
|
|
Dataset.history_orders,
|
|
(
|
|
"CREATE TABLE history_orders("
|
|
" ticket INTEGER, symbol TEXT, time TEXT, type INTEGER)"
|
|
),
|
|
(
|
|
"INSERT INTO history_orders(ticket, symbol, time, type)"
|
|
" VALUES (?, ?, ?, ?)"
|
|
),
|
|
[
|
|
(1, "EURUSD", "2024-01-01T00:00:00+00:00", 0),
|
|
(1, "EURUSD", "2024-01-01T00:00:00+00:00", 1),
|
|
],
|
|
{"ticket", "symbol", "time", "type"},
|
|
),
|
|
(
|
|
Dataset.history_deals,
|
|
(
|
|
"CREATE TABLE history_deals("
|
|
" ticket INTEGER, symbol TEXT, time TEXT, type INTEGER,"
|
|
" entry INTEGER)"
|
|
),
|
|
(
|
|
"INSERT INTO history_deals(ticket, symbol, time, type, entry)"
|
|
" VALUES (?, ?, ?, ?, ?)"
|
|
),
|
|
[
|
|
(1, "EURUSD", "2024-01-01T00:00:00+00:00", 0, 0),
|
|
(1, "EURUSD", "2024-01-01T00:00:00+00:00", 0, 1),
|
|
],
|
|
{"ticket", "symbol", "time", "type", "entry"},
|
|
),
|
|
],
|
|
)
|
|
def test_deduplicates_non_rate_datasets_by_stable_keys(
|
|
self,
|
|
tmp_path: Path,
|
|
dataset: Dataset,
|
|
table_sql: str,
|
|
insert_sql: str,
|
|
rows: list[tuple[object, ...]],
|
|
columns: set[str],
|
|
) -> None:
|
|
"""Test deduplication keys for ticks, orders, and deals."""
|
|
with sqlite3.connect(tmp_path / f"{dataset.value}-dedup.db") as conn:
|
|
conn.execute(table_sql)
|
|
conn.executemany(insert_sql, rows)
|
|
deduplicate_history_tables(conn, {dataset: columns}, {dataset})
|
|
assert conn.execute(
|
|
f"SELECT COUNT(*) FROM {dataset.table_name}", # noqa: S608
|
|
).fetchone() == (1,)
|
|
|
|
def test_scoped_dedup_preserves_older_rows(self, tmp_path: Path) -> None:
|
|
"""Test scoped deduplication only rewrites the appended boundary."""
|
|
boundary = datetime(2024, 1, 2, tzinfo=UTC)
|
|
with sqlite3.connect(tmp_path / "scoped-dedup.db") as conn:
|
|
conn.execute(
|
|
"CREATE TABLE rates("
|
|
" symbol TEXT, timeframe INTEGER, time TEXT, open REAL)",
|
|
)
|
|
conn.executemany(
|
|
"INSERT INTO rates(symbol, timeframe, time, open) VALUES (?, ?, ?, ?)",
|
|
[
|
|
("EURUSD", 1, "2024-01-01T00:00:00+00:00", 1.0),
|
|
("EURUSD", 1, "2024-01-02T00:00:00+00:00", 2.0),
|
|
("EURUSD", 1, "2024-01-02T00:00:00+00:00", 9.9),
|
|
],
|
|
)
|
|
deduplicate_history_tables(
|
|
conn,
|
|
{Dataset.rates: {"symbol", "timeframe", "time", "open"}},
|
|
{Dataset.rates},
|
|
{
|
|
Dataset.rates: [
|
|
(
|
|
"symbol = ? AND timeframe = ? AND time >= ?",
|
|
("EURUSD", 1, boundary),
|
|
),
|
|
],
|
|
},
|
|
)
|
|
rows = conn.execute(
|
|
"SELECT time, open FROM rates ORDER BY time, open",
|
|
).fetchall()
|
|
assert rows == [
|
|
("2024-01-01T00:00:00+00:00", 1.0),
|
|
("2024-01-02T00:00:00+00:00", 9.9),
|
|
]
|
|
|
|
|
|
class TestRateCompatibilityViews:
|
|
"""Tests for rate compatibility view creation."""
|
|
|
|
def test_creates_views_for_multiple_timeframes(self, tmp_path: Path) -> None:
|
|
"""Test rate views are created per symbol and timeframe."""
|
|
db_path = tmp_path / "rate-views.db"
|
|
with sqlite3.connect(db_path) as conn:
|
|
conn.execute(
|
|
"CREATE TABLE rates("
|
|
" symbol TEXT, timeframe INTEGER, time TEXT, close REAL)",
|
|
)
|
|
conn.executemany(
|
|
"INSERT INTO rates(symbol, timeframe, time, close) VALUES (?, ?, ?, ?)",
|
|
[
|
|
("EURUSD", 1, "2024-01-01T00:00:00+00:00", 1.0),
|
|
("EURUSD", 16385, "2024-01-01T01:00:00+00:00", 1.1),
|
|
],
|
|
)
|
|
create_rate_compatibility_views(conn)
|
|
views = {
|
|
row[0]
|
|
for row in conn.execute(
|
|
"SELECT name FROM sqlite_master WHERE type='view'"
|
|
" AND name LIKE 'rate_EURUSD%'",
|
|
).fetchall()
|
|
}
|
|
assert views == {"rate_EURUSD__M1_1", "rate_EURUSD__H1_16385"}
|
|
assert conn.execute('SELECT close FROM "rate_EURUSD__M1_1"').fetchone() == (
|
|
1.0,
|
|
)
|
|
|
|
def test_drops_stale_single_timeframe_view(self, tmp_path: Path) -> None:
|
|
"""Test stale rate_<symbol> views are removed when timeframes change."""
|
|
db_path = tmp_path / "stale-rate-view.db"
|
|
with sqlite3.connect(db_path) as conn:
|
|
conn.execute(
|
|
"CREATE TABLE rates("
|
|
" symbol TEXT, timeframe INTEGER, time TEXT, close REAL)",
|
|
)
|
|
conn.execute(
|
|
"INSERT INTO rates(symbol, timeframe, time, close) VALUES (?, ?, ?, ?)",
|
|
("EURUSD", 1, "2024-01-01T00:00:00+00:00", 1.0),
|
|
)
|
|
create_rate_compatibility_views(conn)
|
|
assert conn.execute(
|
|
"SELECT 1 FROM sqlite_master"
|
|
" WHERE type='view' AND name = 'rate_EURUSD__1'",
|
|
).fetchone() == (1,)
|
|
|
|
conn.execute(
|
|
"INSERT INTO rates(symbol, timeframe, time, close) VALUES (?, ?, ?, ?)",
|
|
("EURUSD", TIMEFRAME_MAP["H1"], "2024-01-01T01:00:00+00:00", 1.1),
|
|
)
|
|
create_rate_compatibility_views(conn)
|
|
views = {
|
|
row[0]
|
|
for row in conn.execute(
|
|
"SELECT name FROM sqlite_master WHERE type='view'"
|
|
" AND name LIKE 'rate_EURUSD%'",
|
|
).fetchall()
|
|
}
|
|
assert views == {"rate_EURUSD__M1_1", "rate_EURUSD__H1_16385"}
|
|
assert "rate_EURUSD__1" not in views
|
|
|
|
def test_does_not_drop_views_outside_rate_prefix(self, tmp_path: Path) -> None:
|
|
"""Test only literal rate_* views are dropped during recreation."""
|
|
db_path = tmp_path / "rate-prefix-views.db"
|
|
with sqlite3.connect(db_path) as conn:
|
|
conn.execute(
|
|
"CREATE TABLE rates("
|
|
" symbol TEXT, timeframe INTEGER, time TEXT, close REAL)",
|
|
)
|
|
conn.execute(
|
|
"INSERT INTO rates(symbol, timeframe, time, close) VALUES (?, ?, ?, ?)",
|
|
("EURUSD", 1, "2024-01-01T00:00:00+00:00", 1.0),
|
|
)
|
|
conn.execute("CREATE VIEW rate_EURUSD AS SELECT 1 AS close")
|
|
conn.execute("CREATE VIEW rateX_custom AS SELECT 2 AS close")
|
|
create_rate_compatibility_views(conn)
|
|
views = {
|
|
row[0]
|
|
for row in conn.execute(
|
|
"SELECT name FROM sqlite_master WHERE type='view'",
|
|
).fetchall()
|
|
}
|
|
assert "rateX_custom" in views
|
|
assert "rate_EURUSD__1" in views
|
|
assert conn.execute("SELECT close FROM rate_EURUSD__1").fetchone() == (1.0,)
|
|
|
|
@pytest.mark.parametrize(
|
|
"symbol",
|
|
["EUR/USD", "US500.cash", "#US500"],
|
|
)
|
|
def test_supports_broker_specific_symbols(
|
|
self,
|
|
tmp_path: Path,
|
|
symbol: str,
|
|
) -> None:
|
|
"""Test broker-specific symbols get readable rate views."""
|
|
db_path = tmp_path / "broker-symbol.db"
|
|
view_name = build_rate_view_name(
|
|
symbol=symbol,
|
|
granularity="M1",
|
|
granularity_count=1,
|
|
timeframe=1,
|
|
)
|
|
with sqlite3.connect(db_path) as conn:
|
|
conn.execute(
|
|
"CREATE TABLE rates("
|
|
" symbol TEXT, timeframe INTEGER, time TEXT, close REAL)",
|
|
)
|
|
conn.execute(
|
|
"INSERT INTO rates(symbol, timeframe, time, close) VALUES (?, ?, ?, ?)",
|
|
(symbol, 1, "2024-01-01T00:00:00+00:00", 1.0),
|
|
)
|
|
create_rate_compatibility_views(conn)
|
|
assert conn.execute(
|
|
"SELECT 1 FROM sqlite_master WHERE type='view' AND name = ?",
|
|
(view_name,),
|
|
).fetchone() == (1,)
|
|
quoted = quote_sqlite_identifier(view_name)
|
|
query = "SELECT close FROM " + quoted # noqa: S608
|
|
assert conn.execute(query).fetchone() == (1.0,)
|
|
|
|
|
|
class TestDerivedViews:
|
|
"""Tests for cash_events and positions_reconstructed views."""
|
|
|
|
def test_creates_views_when_columns_present(self, tmp_path: Path) -> None:
|
|
"""Test cash_events and positions_reconstructed views are created."""
|
|
db_path = tmp_path / "derived-views.db"
|
|
columns = {
|
|
"ticket",
|
|
"position_id",
|
|
"symbol",
|
|
"time",
|
|
"type",
|
|
"entry",
|
|
"volume",
|
|
"price",
|
|
"profit",
|
|
}
|
|
with sqlite3.connect(db_path) as conn:
|
|
conn.execute(
|
|
"CREATE TABLE history_deals("
|
|
" ticket INTEGER, position_id INTEGER, symbol TEXT, time INTEGER,"
|
|
" type INTEGER, entry INTEGER, volume REAL, price REAL, profit REAL)",
|
|
)
|
|
conn.executemany(
|
|
"INSERT INTO history_deals VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)",
|
|
[
|
|
(1, 100, "EURUSD", 1, 0, 0, 1.0, 1.1, 0.0),
|
|
(2, 100, "EURUSD", 2, 0, 1, 1.0, 1.2, 5.0),
|
|
(3, 0, "", 3, 2, 0, 0.0, 0.0, 10.0),
|
|
],
|
|
)
|
|
assert create_cash_events_view(conn, columns)
|
|
assert create_positions_reconstructed_view(conn, columns)
|
|
assert conn.execute("SELECT COUNT(*) FROM cash_events").fetchone() == (1,)
|
|
assert conn.execute(
|
|
"SELECT position_id FROM positions_reconstructed",
|
|
).fetchall() == [(100,)]
|
|
|
|
|
|
class TestFilterTradeHistoryFrame:
|
|
"""Tests for filter_trade_history_frame."""
|
|
|
|
def test_includes_account_events_when_requested(self) -> None:
|
|
"""Test account events are kept alongside selected symbols."""
|
|
frame = pd.DataFrame({
|
|
"symbol": ["EURUSD", None, "OTHER"],
|
|
"type": [0, 2, 0],
|
|
})
|
|
filtered = filter_trade_history_frame(
|
|
frame,
|
|
["EURUSD"],
|
|
include_account_events=True,
|
|
)
|
|
assert filtered["symbol"].tolist()[0] == "EURUSD"
|
|
assert pd.isna(filtered["symbol"].tolist()[1])
|
|
|
|
|
|
class TestIncrementalHistoryDealsHelpers:
|
|
"""Tests for incremental history_deals helper functions."""
|
|
|
|
def test_account_event_start_uses_type_column(self, tmp_path: Path) -> None:
|
|
"""Test account-event start uses non-trade deal types when available."""
|
|
fallback = datetime(2024, 1, 1, tzinfo=UTC)
|
|
with sqlite3.connect(tmp_path / "account-start-type.db") as conn:
|
|
conn.execute(
|
|
"CREATE TABLE history_deals("
|
|
" ticket INTEGER, symbol TEXT, time TEXT, type INTEGER)",
|
|
)
|
|
conn.executemany(
|
|
"INSERT INTO history_deals(ticket, symbol, time, type)"
|
|
" VALUES (?, ?, ?, ?)",
|
|
[
|
|
(1, "EURUSD", "2024-01-05T00:00:00+00:00", 0),
|
|
(2, "", "2024-01-08T00:00:00+00:00", 2),
|
|
],
|
|
)
|
|
assert get_history_deals_account_event_start_datetime(
|
|
conn,
|
|
fallback_start=fallback,
|
|
) == datetime(2024, 1, 8, tzinfo=UTC)
|
|
|
|
def test_account_event_start_falls_back_to_empty_symbol(
|
|
self, tmp_path: Path
|
|
) -> None:
|
|
"""Test account-event start uses empty symbols when type is missing."""
|
|
fallback = datetime(2024, 1, 1, tzinfo=UTC)
|
|
with sqlite3.connect(tmp_path / "account-start-symbol.db") as conn:
|
|
conn.execute(
|
|
"CREATE TABLE history_deals(ticket INTEGER, symbol TEXT, time TEXT)",
|
|
)
|
|
conn.executemany(
|
|
"INSERT INTO history_deals(ticket, symbol, time) VALUES (?, ?, ?)",
|
|
[
|
|
(1, "EURUSD", "2024-01-05T00:00:00+00:00"),
|
|
(2, "", "2024-01-07T00:00:00+00:00"),
|
|
],
|
|
)
|
|
assert get_history_deals_account_event_start_datetime(
|
|
conn,
|
|
fallback_start=fallback,
|
|
) == datetime(2024, 1, 7, tzinfo=UTC)
|
|
|
|
def test_account_event_start_without_identifying_columns(
|
|
self,
|
|
tmp_path: Path,
|
|
) -> None:
|
|
"""Test account-event start falls back when type and symbol are missing."""
|
|
fallback = datetime(2024, 1, 1, tzinfo=UTC)
|
|
with sqlite3.connect(tmp_path / "account-start-fallback.db") as conn:
|
|
conn.execute("CREATE TABLE history_deals(ticket INTEGER, time TEXT)")
|
|
conn.execute(
|
|
"INSERT INTO history_deals(ticket, time) VALUES (?, ?)",
|
|
(1, "2024-01-05T00:00:00+00:00"),
|
|
)
|
|
assert (
|
|
get_history_deals_account_event_start_datetime(
|
|
conn,
|
|
fallback_start=fallback,
|
|
)
|
|
== fallback
|
|
)
|
|
|
|
def test_filter_incremental_history_deals_frame(self) -> None:
|
|
"""Test incremental deal filtering applies per-symbol and account starts."""
|
|
frame = pd.DataFrame({
|
|
"ticket": [1, 2, 3, 4, 5],
|
|
"symbol": ["EURUSD", "EURUSD", "GBPUSD", "OTHER", ""],
|
|
"time": [
|
|
"2024-01-05T00:00:00+00:00",
|
|
"2024-01-11T00:00:00+00:00",
|
|
"2024-01-02T00:00:00+00:00",
|
|
"2024-01-02T00:00:00+00:00",
|
|
"2024-01-03T00:00:00+00:00",
|
|
],
|
|
"type": [0, 0, 0, 0, 2],
|
|
})
|
|
start_by_symbol = {
|
|
"EURUSD": datetime(2024, 1, 10, tzinfo=UTC),
|
|
"GBPUSD": datetime(2024, 1, 1, tzinfo=UTC),
|
|
}
|
|
filtered = filter_incremental_history_deals_frame(
|
|
frame,
|
|
["EURUSD", "GBPUSD"],
|
|
start_by_symbol,
|
|
datetime(2024, 1, 2, tzinfo=UTC),
|
|
)
|
|
assert filtered["ticket"].tolist() == [2, 3, 5]
|
|
|
|
def test_filter_incremental_excludes_symbolized_account_events_from_trade_cursor(
|
|
self,
|
|
) -> None:
|
|
"""Test symbolized account events follow only account_event_start.
|
|
|
|
An account-event row with a matching symbol must not be kept by the
|
|
EURUSD trade cursor when its time is after the trade start but before
|
|
account_event_start.
|
|
"""
|
|
trade_cursor = datetime(2024, 1, 1, tzinfo=UTC)
|
|
account_event_start = datetime(2024, 1, 10, tzinfo=UTC)
|
|
row_time = datetime(2024, 1, 5, tzinfo=UTC)
|
|
frame = pd.DataFrame({
|
|
"ticket": [1],
|
|
"symbol": ["EURUSD"],
|
|
"time": [row_time.isoformat()],
|
|
"type": [2],
|
|
})
|
|
filtered = filter_incremental_history_deals_frame(
|
|
frame,
|
|
["EURUSD"],
|
|
{"EURUSD": trade_cursor},
|
|
account_event_start,
|
|
)
|
|
assert filtered.empty
|
|
|
|
def test_filter_incremental_rejects_rows_without_parseable_time(self) -> None:
|
|
"""Test incremental filtering drops rows when time cannot be parsed."""
|
|
frame = pd.DataFrame({
|
|
"ticket": [1],
|
|
"symbol": ["EURUSD"],
|
|
"time": ["not-a-datetime"],
|
|
"type": [0],
|
|
})
|
|
filtered = filter_incremental_history_deals_frame(
|
|
frame,
|
|
["EURUSD"],
|
|
{"EURUSD": datetime(2024, 1, 1, tzinfo=UTC)},
|
|
datetime(2024, 1, 1, tzinfo=UTC),
|
|
)
|
|
assert filtered.empty
|
|
|
|
def test_filter_incremental_keeps_account_events_without_symbol_column(
|
|
self,
|
|
) -> None:
|
|
"""Test account events are kept when history_deals has no symbol column."""
|
|
frame = pd.DataFrame({
|
|
"ticket": [1],
|
|
"time": ["2024-01-03T00:00:00+00:00"],
|
|
"type": [2],
|
|
})
|
|
filtered = filter_incremental_history_deals_frame(
|
|
frame,
|
|
["EURUSD"],
|
|
{"EURUSD": datetime(2024, 1, 10, tzinfo=UTC)},
|
|
datetime(2024, 1, 2, tzinfo=UTC),
|
|
)
|
|
assert filtered["ticket"].tolist() == [1]
|
|
|
|
def test_filter_incremental_skips_trade_rows_without_symbol_column(self) -> None:
|
|
"""Test trade rows are excluded when symbol column is unavailable."""
|
|
frame = pd.DataFrame({
|
|
"ticket": [1],
|
|
"time": ["2024-01-03T00:00:00+00:00"],
|
|
})
|
|
filtered = filter_incremental_history_deals_frame(
|
|
frame,
|
|
["EURUSD"],
|
|
{"EURUSD": datetime(2024, 1, 1, tzinfo=UTC)},
|
|
datetime(2024, 1, 1, tzinfo=UTC),
|
|
)
|
|
assert filtered.empty
|
|
|
|
def test_filter_incremental_rejects_rows_without_time_column(self) -> None:
|
|
"""Test incremental filtering drops rows when time column is missing."""
|
|
frame = pd.DataFrame({
|
|
"ticket": [1],
|
|
"symbol": ["EURUSD"],
|
|
"type": [0],
|
|
})
|
|
filtered = filter_incremental_history_deals_frame(
|
|
frame,
|
|
["EURUSD"],
|
|
{"EURUSD": datetime(2024, 1, 1, tzinfo=UTC)},
|
|
datetime(2024, 1, 1, tzinfo=UTC),
|
|
)
|
|
assert filtered.empty
|
|
|
|
|
|
class TestIncrementalIntegration:
|
|
"""Integration tests for incremental write helpers."""
|
|
|
|
def test_write_incremental_datasets_end_to_end(
|
|
self,
|
|
tmp_path: Path,
|
|
) -> None:
|
|
"""Test incremental dataset writer covers finalize branches."""
|
|
client = MagicMock()
|
|
client.copy_rates_range_as_df.return_value = pd.DataFrame({
|
|
"time": ["2024-01-01T00:00:00+00:00"],
|
|
"open": [1.0],
|
|
})
|
|
client.copy_ticks_range_as_df.return_value = pd.DataFrame()
|
|
client.history_orders_get_as_df.return_value = pd.DataFrame({
|
|
"ticket": [1],
|
|
"symbol": ["EURUSD"],
|
|
"time": ["2024-01-01T00:00:00+00:00"],
|
|
"type": [0],
|
|
})
|
|
client.history_deals_get_as_df.return_value = pd.DataFrame({
|
|
"ticket": [2],
|
|
"position_id": [100],
|
|
"symbol": ["EURUSD"],
|
|
"time": ["2024-01-01T00:00:00+00:00"],
|
|
"type": [0],
|
|
"entry": [0],
|
|
"volume": [1.0],
|
|
"price": [1.1],
|
|
"profit": [0.0],
|
|
})
|
|
db_path = tmp_path / "incremental-integration.db"
|
|
start = datetime(2024, 1, 1, tzinfo=UTC)
|
|
end = datetime(2024, 1, 2, tzinfo=UTC)
|
|
with sqlite3.connect(db_path) as conn:
|
|
write_incremental_datasets(
|
|
conn,
|
|
client,
|
|
["EURUSD"],
|
|
set(Dataset),
|
|
[1],
|
|
1,
|
|
start,
|
|
end,
|
|
deduplicate=True,
|
|
create_rate_views=True,
|
|
with_views=True,
|
|
include_account_events=True,
|
|
)
|
|
tables = {
|
|
row[0]
|
|
for row in conn.execute(
|
|
"SELECT name FROM sqlite_master WHERE type='table'",
|
|
).fetchall()
|
|
}
|
|
views = {
|
|
row[0]
|
|
for row in conn.execute(
|
|
"SELECT name FROM sqlite_master WHERE type='view'",
|
|
).fetchall()
|
|
}
|
|
assert {"rates", "history_orders", "history_deals"} <= tables
|
|
assert "rate_EURUSD__1" in views
|
|
assert "cash_events" in views
|
|
|
|
def test_rate_view_names_do_not_collide_for_symbol_suffix(
|
|
self,
|
|
tmp_path: Path,
|
|
) -> None:
|
|
"""Test symbols resembling generated view suffixes stay distinct."""
|
|
db_path = tmp_path / "rate-view-collision.db"
|
|
with sqlite3.connect(db_path) as conn:
|
|
conn.execute(
|
|
"CREATE TABLE rates("
|
|
" symbol TEXT, timeframe INTEGER, time TEXT, close REAL)",
|
|
)
|
|
conn.executemany(
|
|
"INSERT INTO rates(symbol, timeframe, time, close) VALUES (?, ?, ?, ?)",
|
|
[
|
|
("EURUSD", 1, "2024-01-01T00:00:00+00:00", 1.0),
|
|
("EURUSD", 16385, "2024-01-01T01:00:00+00:00", 1.1),
|
|
("EURUSD_M1", 1, "2024-01-01T02:00:00+00:00", 1.2),
|
|
],
|
|
)
|
|
create_rate_compatibility_views(conn)
|
|
views = {
|
|
row[0]
|
|
for row in conn.execute(
|
|
"SELECT name FROM sqlite_master WHERE type='view'",
|
|
).fetchall()
|
|
}
|
|
assert views == {
|
|
"rate_EURUSD__M1_1",
|
|
"rate_EURUSD__H1_16385",
|
|
"rate_EURUSD_M1__1",
|
|
}
|
|
|
|
def test_write_collected_datasets_and_edge_branches(
|
|
self,
|
|
tmp_path: Path,
|
|
) -> None:
|
|
"""Test collected dataset writer and helper edge branches."""
|
|
client = MagicMock()
|
|
client.copy_rates_range_as_df.return_value = pd.DataFrame()
|
|
client.copy_ticks_range_as_df.return_value = pd.DataFrame({
|
|
"time": ["2024-01-01T00:00:00+00:00"],
|
|
"bid": [1.0],
|
|
})
|
|
client.history_orders_get_as_df.return_value = pd.DataFrame({
|
|
"ticket": [1],
|
|
"symbol": ["EURUSD"],
|
|
"time": [1],
|
|
"type": [0],
|
|
})
|
|
client.history_deals_get_as_df.return_value = pd.DataFrame({
|
|
"ticket": [1],
|
|
"symbol": ["EURUSD"],
|
|
"time": [1],
|
|
"type": [0],
|
|
})
|
|
db_path = tmp_path / "collected-integration.db"
|
|
start = datetime(2024, 1, 1, tzinfo=UTC)
|
|
end = datetime(2024, 1, 2, tzinfo=UTC)
|
|
with sqlite3.connect(db_path) as conn:
|
|
write_collected_datasets(
|
|
conn,
|
|
client,
|
|
["EURUSD"],
|
|
{Dataset.ticks, Dataset.history_orders, Dataset.history_deals},
|
|
1,
|
|
1,
|
|
start,
|
|
end,
|
|
IfExists.APPEND,
|
|
)
|
|
assert (
|
|
get_incremental_start_datetime(
|
|
conn,
|
|
Dataset.ticks,
|
|
symbol="EURUSD",
|
|
timeframe=None,
|
|
fallback_start=start,
|
|
)
|
|
== start
|
|
)
|
|
deduplicate_history_tables(
|
|
conn,
|
|
{Dataset.ticks: {"time"}},
|
|
{Dataset.ticks},
|
|
)
|
|
filtered = filter_trade_history_frame(
|
|
pd.DataFrame({"ticket": [1]}),
|
|
["EURUSD"],
|
|
include_account_events=False,
|
|
)
|
|
assert len(filtered) == 1
|
|
account_filtered = filter_trade_history_frame(
|
|
pd.DataFrame({"symbol": ["", "EURUSD"], "type": [2, 0]}),
|
|
["EURUSD"],
|
|
include_account_events=True,
|
|
)
|
|
assert len(account_filtered) == 2
|
|
no_type_filtered = filter_trade_history_frame(
|
|
pd.DataFrame({"symbol": ["", "EURUSD"]}),
|
|
["EURUSD"],
|
|
include_account_events=True,
|
|
)
|
|
assert len(no_type_filtered) == 2
|
|
|
|
def test_get_incremental_start_without_symbol_column(
|
|
self,
|
|
tmp_path: Path,
|
|
) -> None:
|
|
"""Test incremental start ignores missing symbol column filters."""
|
|
db_path = tmp_path / "no-symbol-column.db"
|
|
fallback = datetime(2024, 1, 1, tzinfo=UTC)
|
|
with sqlite3.connect(db_path) as conn:
|
|
conn.execute("CREATE TABLE ticks(time TEXT)")
|
|
conn.execute(
|
|
"INSERT INTO ticks(time) VALUES (?)",
|
|
("2024-01-02T00:00:00+00:00",),
|
|
)
|
|
assert get_incremental_start_datetime(
|
|
conn,
|
|
Dataset.ticks,
|
|
symbol="EURUSD",
|
|
timeframe=None,
|
|
fallback_start=fallback,
|
|
) == datetime(2024, 1, 2, tzinfo=UTC)
|
|
|
|
def test_deduplicate_skips_unsupported_keys(
|
|
self,
|
|
tmp_path: Path,
|
|
caplog: pytest.LogCaptureFixture,
|
|
) -> None:
|
|
"""Test deduplication logs when no stable key columns exist."""
|
|
with (
|
|
sqlite3.connect(tmp_path / "no-keys.db") as conn,
|
|
caplog.at_level(
|
|
logging.WARNING,
|
|
logger="mt5cli.history",
|
|
),
|
|
):
|
|
deduplicate_history_tables(conn, {Dataset.ticks: {"time"}}, {Dataset.ticks})
|
|
assert "Skipping ticks deduplication" in caplog.text
|
|
|
|
def test_write_rates_skips_empty_schema(
|
|
self,
|
|
tmp_path: Path,
|
|
) -> None:
|
|
"""Test rates writer skips frames with no columns after normalization."""
|
|
client = MagicMock()
|
|
client.copy_rates_range_as_df.return_value = pd.DataFrame()
|
|
written_columns: dict[Dataset, set[str]] = {}
|
|
with sqlite3.connect(tmp_path / "empty-rates.db") as conn:
|
|
assert not write_rates_dataset(
|
|
conn,
|
|
client,
|
|
["EURUSD"],
|
|
1,
|
|
datetime(2024, 1, 1, tzinfo=UTC),
|
|
datetime(2024, 1, 2, tzinfo=UTC),
|
|
IfExists.APPEND,
|
|
written_columns,
|
|
)
|
|
|
|
def test_finalize_with_views_warning_when_deals_missing(
|
|
self,
|
|
tmp_path: Path,
|
|
caplog: pytest.LogCaptureFixture,
|
|
) -> None:
|
|
"""Test with_views warning when history_deals was not written."""
|
|
client = MagicMock()
|
|
client.copy_rates_range_as_df.return_value = pd.DataFrame()
|
|
with (
|
|
caplog.at_level(logging.WARNING, logger="mt5cli.history"),
|
|
sqlite3.connect(tmp_path / "views-warning.db") as conn,
|
|
):
|
|
write_incremental_datasets(
|
|
conn,
|
|
client,
|
|
["EURUSD"],
|
|
{Dataset.rates, Dataset.history_deals},
|
|
[1],
|
|
0,
|
|
datetime(2024, 1, 1, tzinfo=UTC),
|
|
datetime(2024, 1, 2, tzinfo=UTC),
|
|
deduplicate=False,
|
|
create_rate_views=False,
|
|
with_views=True,
|
|
include_account_events=True,
|
|
)
|
|
assert "with_views ignored" in caplog.text
|
|
|
|
def test_augment_written_columns_creates_new_entry(
|
|
self,
|
|
tmp_path: Path,
|
|
) -> None:
|
|
"""Test augment helper initializes dataset column maps."""
|
|
db_path = tmp_path / "augment-new.db"
|
|
written_columns: dict[Dataset, set[str]] = {}
|
|
with sqlite3.connect(db_path) as conn:
|
|
conn.execute("CREATE TABLE rates(time TEXT)")
|
|
augment_written_columns_from_sqlite(
|
|
conn,
|
|
{Dataset.rates},
|
|
written_columns,
|
|
)
|
|
assert written_columns == {Dataset.rates: {"time"}}
|
|
|
|
def test_create_rate_views_noop_without_required_columns(
|
|
self,
|
|
tmp_path: Path,
|
|
) -> None:
|
|
"""Test rate view creation skips tables missing required columns."""
|
|
with sqlite3.connect(tmp_path / "missing-rate-columns.db") as conn:
|
|
conn.execute("CREATE TABLE rates(open REAL)")
|
|
create_rate_compatibility_views(conn)
|
|
views = conn.execute(
|
|
"SELECT name FROM sqlite_master WHERE type='view'",
|
|
).fetchall()
|
|
assert views == []
|
|
|
|
def test_incremental_history_noops_when_fetch_returns_empty(
|
|
self,
|
|
tmp_path: Path,
|
|
) -> None:
|
|
"""Test incremental history skips table tracking for empty writes."""
|
|
client = MagicMock()
|
|
client.history_orders_get_as_df.return_value = pd.DataFrame()
|
|
client.history_deals_get_as_df.return_value = pd.DataFrame()
|
|
start = datetime(2024, 1, 1, tzinfo=UTC)
|
|
end = datetime(2024, 1, 2, tzinfo=UTC)
|
|
with sqlite3.connect(tmp_path / "empty-history.db") as conn:
|
|
written_tables, _ = write_incremental_datasets(
|
|
conn,
|
|
client,
|
|
["EURUSD"],
|
|
{Dataset.history_orders, Dataset.history_deals},
|
|
[],
|
|
0,
|
|
start,
|
|
end,
|
|
deduplicate=False,
|
|
create_rate_views=False,
|
|
with_views=False,
|
|
include_account_events=True,
|
|
)
|
|
assert written_tables == set()
|
|
|
|
def test_resolve_history_tick_flags_invalid(self) -> None:
|
|
"""Test invalid tick flags raise ValueError."""
|
|
with pytest.raises(ValueError, match="Invalid tick flags"):
|
|
resolve_history_tick_flags("BAD")
|
|
|
|
def test_resolve_history_timeframes_invalid(self) -> None:
|
|
"""Test invalid timeframes raise ValueError."""
|
|
with pytest.raises(ValueError, match="Invalid timeframe"):
|
|
resolve_history_timeframes(["BAD"])
|
|
|
|
|
|
class TestIncrementalHistoryDeals:
|
|
"""Tests for incremental history_deals account-event handling."""
|
|
|
|
def test_fetches_per_symbol_when_account_events_disabled(
|
|
self,
|
|
tmp_path: Path,
|
|
) -> None:
|
|
"""Test history_deals are fetched per symbol without account events."""
|
|
client = MagicMock()
|
|
client.history_deals_get_as_df.return_value = pd.DataFrame({
|
|
"ticket": [1],
|
|
"symbol": ["EURUSD"],
|
|
"time": [1],
|
|
"type": [0],
|
|
"entry": [0],
|
|
})
|
|
start = datetime(2024, 1, 1, tzinfo=UTC)
|
|
end = datetime(2024, 1, 2, tzinfo=UTC)
|
|
with sqlite3.connect(tmp_path / "per-symbol-deals.db") as conn:
|
|
write_incremental_datasets(
|
|
conn,
|
|
client,
|
|
["EURUSD", "GBPUSD"],
|
|
{Dataset.history_deals},
|
|
[],
|
|
0,
|
|
start,
|
|
end,
|
|
deduplicate=False,
|
|
create_rate_views=False,
|
|
with_views=False,
|
|
include_account_events=False,
|
|
)
|
|
assert client.history_deals_get_as_df.call_count == 2
|
|
|
|
def test_skips_history_deals_when_fetch_returns_empty(
|
|
self,
|
|
tmp_path: Path,
|
|
) -> None:
|
|
"""Test incremental history_deals skips tracking when writes are empty."""
|
|
client = MagicMock()
|
|
client.history_deals_get_as_df.return_value = pd.DataFrame()
|
|
start = datetime(2024, 1, 1, tzinfo=UTC)
|
|
end = datetime(2024, 1, 2, tzinfo=UTC)
|
|
with sqlite3.connect(tmp_path / "empty-per-symbol-deals.db") as conn:
|
|
written_tables, _ = write_incremental_datasets(
|
|
conn,
|
|
client,
|
|
["EURUSD", "GBPUSD"],
|
|
{Dataset.history_deals},
|
|
[],
|
|
0,
|
|
start,
|
|
end,
|
|
deduplicate=False,
|
|
create_rate_views=False,
|
|
with_views=False,
|
|
include_account_events=False,
|
|
)
|
|
assert written_tables == set()
|
|
|
|
def test_write_history_dataset_fetches_account_events_once(
|
|
self,
|
|
tmp_path: Path,
|
|
) -> None:
|
|
"""Test write_history_dataset account-event path fetches once."""
|
|
client = MagicMock()
|
|
client.history_deals_get_as_df.return_value = pd.DataFrame({
|
|
"ticket": [1, 2],
|
|
"symbol": ["EURUSD", ""],
|
|
"time": [1, 2],
|
|
"type": [0, 2],
|
|
"entry": [0, 0],
|
|
})
|
|
start = datetime(2024, 1, 1, tzinfo=UTC)
|
|
end = datetime(2024, 1, 2, tzinfo=UTC)
|
|
written_columns: dict[Dataset, set[str]] = {}
|
|
with sqlite3.connect(tmp_path / "history-dataset-account.db") as conn:
|
|
assert write_history_dataset(
|
|
conn,
|
|
client.history_deals_get_as_df,
|
|
Dataset.history_deals,
|
|
["EURUSD"],
|
|
start,
|
|
end,
|
|
IfExists.APPEND,
|
|
written_columns,
|
|
include_account_events=True,
|
|
)
|
|
rows = conn.execute(
|
|
"SELECT ticket, symbol, type FROM history_deals ORDER BY ticket",
|
|
).fetchall()
|
|
client.history_deals_get_as_df.assert_called_once()
|
|
assert rows == [(1, "EURUSD", 0), (2, "", 2)]
|
|
|
|
def test_fetches_account_events_once_for_multiple_symbols(
|
|
self,
|
|
tmp_path: Path,
|
|
) -> None:
|
|
"""Test account events are fetched once and not duplicated."""
|
|
client = MagicMock()
|
|
client.history_deals_get_as_df.return_value = pd.DataFrame({
|
|
"ticket": [1, 2, 3, 4],
|
|
"symbol": ["EURUSD", "GBPUSD", "OTHER", ""],
|
|
"time": [
|
|
"2024-01-01T00:00:00+00:00",
|
|
"2024-01-01T01:00:00+00:00",
|
|
"2024-01-01T02:00:00+00:00",
|
|
"2024-01-01T03:00:00+00:00",
|
|
],
|
|
"type": [0, 0, 0, 2],
|
|
"entry": [0, 0, 0, 0],
|
|
})
|
|
start = datetime(2024, 1, 1, tzinfo=UTC)
|
|
end = datetime(2024, 1, 2, tzinfo=UTC)
|
|
with sqlite3.connect(tmp_path / "account-events.db") as conn:
|
|
write_incremental_datasets(
|
|
conn,
|
|
client,
|
|
["EURUSD", "GBPUSD"],
|
|
{Dataset.history_deals},
|
|
[],
|
|
0,
|
|
start,
|
|
end,
|
|
deduplicate=False,
|
|
create_rate_views=False,
|
|
with_views=False,
|
|
include_account_events=True,
|
|
)
|
|
rows = conn.execute(
|
|
"SELECT ticket, symbol, type FROM history_deals ORDER BY ticket",
|
|
).fetchall()
|
|
row_count = conn.execute("SELECT COUNT(*) FROM history_deals").fetchone()
|
|
client.history_deals_get_as_df.assert_called_once()
|
|
assert rows == [(1, "EURUSD", 0), (2, "GBPUSD", 0), (4, "", 2)]
|
|
assert row_count == (3,)
|
|
|
|
def test_records_account_event_dedup_scope_without_type_column(
|
|
self,
|
|
tmp_path: Path,
|
|
) -> None:
|
|
"""Test account-event dedup scope falls back to empty symbols."""
|
|
client = MagicMock()
|
|
client.history_deals_get_as_df.return_value = pd.DataFrame({
|
|
"ticket": [1],
|
|
"symbol": [""],
|
|
"time": ["2024-01-02T00:00:00+00:00"],
|
|
})
|
|
start = datetime(2024, 1, 1, tzinfo=UTC)
|
|
end = datetime(2024, 1, 3, tzinfo=UTC)
|
|
with sqlite3.connect(tmp_path / "legacy-deals.db") as conn:
|
|
conn.execute(
|
|
"CREATE TABLE history_deals( ticket INTEGER, symbol TEXT, time TEXT)",
|
|
)
|
|
write_incremental_datasets(
|
|
conn,
|
|
client,
|
|
["EURUSD"],
|
|
{Dataset.history_deals},
|
|
[],
|
|
0,
|
|
start,
|
|
end,
|
|
deduplicate=True,
|
|
create_rate_views=False,
|
|
with_views=False,
|
|
include_account_events=True,
|
|
)
|
|
assert conn.execute("SELECT COUNT(*) FROM history_deals").fetchone() == (1,)
|
|
|
|
def test_skips_symbol_dedup_scope_when_symbol_column_missing(
|
|
self,
|
|
tmp_path: Path,
|
|
) -> None:
|
|
"""Test account-event writes skip per-symbol dedup without symbol column."""
|
|
client = MagicMock()
|
|
client.history_deals_get_as_df.return_value = pd.DataFrame({
|
|
"ticket": [1],
|
|
"time": ["2024-01-02T00:00:00+00:00"],
|
|
"type": [2],
|
|
})
|
|
start = datetime(2024, 1, 1, tzinfo=UTC)
|
|
end = datetime(2024, 1, 3, tzinfo=UTC)
|
|
with sqlite3.connect(tmp_path / "no-symbol-deals.db") as conn:
|
|
conn.execute(
|
|
"CREATE TABLE history_deals(ticket INTEGER, time TEXT, type INTEGER)",
|
|
)
|
|
write_incremental_datasets(
|
|
conn,
|
|
client,
|
|
["EURUSD"],
|
|
{Dataset.history_deals},
|
|
[],
|
|
0,
|
|
start,
|
|
end,
|
|
deduplicate=True,
|
|
create_rate_views=False,
|
|
with_views=False,
|
|
include_account_events=True,
|
|
)
|
|
assert conn.execute("SELECT COUNT(*) FROM history_deals").fetchone() == (1,)
|
|
|
|
def test_creates_views_when_history_deals_written_with_with_views(
|
|
self,
|
|
tmp_path: Path,
|
|
) -> None:
|
|
"""Test derived views are rebuilt when history_deals is written."""
|
|
client = MagicMock()
|
|
client.history_deals_get_as_df.return_value = pd.DataFrame({
|
|
"ticket": [1, 2],
|
|
"position_id": [100, 100],
|
|
"symbol": ["EURUSD", "EURUSD"],
|
|
"time": ["2024-01-01T00:00:00+00:00", "2024-01-02T00:00:00+00:00"],
|
|
"type": [0, 1],
|
|
"entry": [0, 1],
|
|
"volume": [1.0, 1.0],
|
|
"price": [1.1, 1.2],
|
|
"profit": [0.0, 5.0],
|
|
})
|
|
start = datetime(2024, 1, 1, tzinfo=UTC)
|
|
end = datetime(2024, 1, 3, tzinfo=UTC)
|
|
with sqlite3.connect(tmp_path / "deals-with-views.db") as conn:
|
|
write_incremental_datasets(
|
|
conn,
|
|
client,
|
|
["EURUSD"],
|
|
{Dataset.history_deals},
|
|
[],
|
|
0,
|
|
start,
|
|
end,
|
|
deduplicate=False,
|
|
create_rate_views=False,
|
|
with_views=True,
|
|
include_account_events=False,
|
|
)
|
|
views = {
|
|
row[0]
|
|
for row in conn.execute(
|
|
"SELECT name FROM sqlite_master WHERE type='view'",
|
|
).fetchall()
|
|
}
|
|
assert {"cash_events", "positions_reconstructed"} <= views
|
|
|
|
def test_applies_per_symbol_start_when_account_events_enabled(
|
|
self,
|
|
tmp_path: Path,
|
|
) -> None:
|
|
"""Test account-event fetch keeps per-symbol incremental boundaries."""
|
|
client = MagicMock()
|
|
client.history_deals_get_as_df.return_value = pd.DataFrame({
|
|
"ticket": [1, 2, 3, 4, 5],
|
|
"symbol": ["EURUSD", "EURUSD", "GBPUSD", "OTHER", ""],
|
|
"time": [
|
|
"2024-01-05T00:00:00+00:00",
|
|
"2024-01-11T00:00:00+00:00",
|
|
"2024-01-02T00:00:00+00:00",
|
|
"2024-01-02T00:00:00+00:00",
|
|
"2024-01-03T00:00:00+00:00",
|
|
],
|
|
"type": [0, 0, 0, 0, 2],
|
|
"entry": [0, 0, 0, 0, 0],
|
|
})
|
|
fallback = datetime(2024, 1, 1, tzinfo=UTC)
|
|
end = datetime(2024, 1, 15, tzinfo=UTC)
|
|
with sqlite3.connect(tmp_path / "per-symbol-account-events.db") as conn:
|
|
conn.execute(
|
|
"CREATE TABLE history_deals("
|
|
" ticket INTEGER, symbol TEXT, time TEXT, type INTEGER, entry INTEGER)",
|
|
)
|
|
conn.executemany(
|
|
"INSERT INTO history_deals(ticket, symbol, time, type, entry)"
|
|
" VALUES (?, ?, ?, ?, ?)",
|
|
[
|
|
(100, "EURUSD", "2024-01-10T00:00:00+00:00", 0, 0),
|
|
(200, "GBPUSD", "2024-01-01T00:00:00+00:00", 0, 0),
|
|
(300, "", "2024-01-02T00:00:00+00:00", 2, 0),
|
|
],
|
|
)
|
|
write_incremental_datasets(
|
|
conn,
|
|
client,
|
|
["EURUSD", "GBPUSD"],
|
|
{Dataset.history_deals},
|
|
[],
|
|
0,
|
|
fallback,
|
|
end,
|
|
deduplicate=False,
|
|
create_rate_views=False,
|
|
with_views=False,
|
|
include_account_events=True,
|
|
)
|
|
rows = conn.execute(
|
|
"SELECT ticket, symbol, type FROM history_deals"
|
|
" WHERE ticket IN (1, 2, 3, 4, 5) ORDER BY ticket",
|
|
).fetchall()
|
|
client.history_deals_get_as_df.assert_called_once_with(
|
|
date_from=datetime(2024, 1, 1, tzinfo=UTC),
|
|
date_to=end,
|
|
)
|
|
assert rows == [(2, "EURUSD", 0), (3, "GBPUSD", 0), (5, "", 2)]
|
|
|
|
|
|
class TestWriteHelpers:
|
|
"""Tests for SQLite write helper branches."""
|
|
|
|
def test_append_dataframe_handles_wide_frames(self, tmp_path: Path) -> None:
|
|
"""Test wide DataFrames append without exceeding SQLite variable limits."""
|
|
db_path = tmp_path / "wide-frame.db"
|
|
columns = {f"col_{index}": [float(index)] for index in range(80)}
|
|
frame = pd.DataFrame(columns)
|
|
with sqlite3.connect(db_path) as conn:
|
|
assert append_dataframe(conn, frame, "wide_rates", IfExists.APPEND)
|
|
assert get_table_columns(conn, "wide_rates") == set(columns)
|
|
|
|
def test_write_streamed_frame_and_column_tracking(self, tmp_path: Path) -> None:
|
|
"""Test append helpers track columns and skip empty frames."""
|
|
db_path = tmp_path / "write-helpers.db"
|
|
written_columns: dict[Dataset, set[str]] = {}
|
|
with sqlite3.connect(db_path) as conn:
|
|
assert not write_streamed_frame(
|
|
conn,
|
|
pd.DataFrame(),
|
|
Dataset.rates,
|
|
table_exists=False,
|
|
if_exists=IfExists.APPEND,
|
|
written_columns=written_columns,
|
|
)
|
|
assert append_dataframe(
|
|
conn,
|
|
pd.DataFrame({"time": [1], "open": [1.0]}),
|
|
"rates",
|
|
IfExists.APPEND,
|
|
)
|
|
record_written_columns(
|
|
written_columns,
|
|
Dataset.rates,
|
|
pd.DataFrame({"close": [1.1]}),
|
|
)
|
|
assert "close" in written_columns[Dataset.rates]
|
|
augment_written_columns_from_sqlite(
|
|
conn,
|
|
{Dataset.rates},
|
|
written_columns,
|
|
)
|
|
assert get_table_columns(conn, "rates") == {"time", "open"}
|
|
create_history_indexes(conn, written_columns)
|