Fix mt5cli issues #92-#95 and #97 (#102)

* feat: add MT5 order metadata, coverage report, and env-backed CLI config * fix: align review-driven trading and history contracts * Bump pdmt5 to 1.1.0 * test: stabilize history gaps CLI assertion * fix: address review follow-ups for gaps and filling mode
2026-07-04 14:19:56 +09:00
parent c2cf0656dd
commit 74e3754a80
15 changed files with 1372 additions and 80 deletions
@@ -48,6 +48,7 @@ from mt5cli.history import (
    parse_sqlite_timestamp,
    quote_sqlite_identifier,
    record_written_columns,
+    report_rate_gaps,
    resolve_granularity_name,
    resolve_history_datasets,
    resolve_history_tick_flags,
@@ -3075,6 +3076,299 @@ class TestRateSourceHelpers:

        assert set(result) == {("EURUSD", "M1"), ("EURUSD", "H1")}

+    def test_report_rate_gaps_reports_one_row_per_gap(self, tmp_path: Path) -> None:
+        """Gap reports emit one row for each detected missing interval run."""
+        db_path = tmp_path / "gaps.db"
+        with sqlite3.connect(db_path) as conn:
+            conn.execute(
+                "CREATE TABLE rates("
+                "symbol TEXT, timeframe INTEGER, time TEXT, close REAL"
+                ")",
+            )
+            conn.executemany(
+                "INSERT INTO rates(symbol, timeframe, time, close) VALUES (?, ?, ?, ?)",
+                [
+                    ("EURUSD", 1, "2024-01-01T00:00:00+00:00", 1.0),
+                    ("EURUSD", 1, "2024-01-01T00:01:00+00:00", 1.1),
+                    ("EURUSD", 1, "2024-01-01T00:03:00+00:00", 1.2),
+                ],
+            )
+            conn.execute(
+                'CREATE VIEW "rate_EURUSD__M1_1" AS '
+                "SELECT time, close FROM rates "
+                "WHERE symbol = 'EURUSD' AND timeframe = 1",
+            )
+            result = report_rate_gaps(
+                conn,
+                "rate_EURUSD__M1_1",
+                granularity_seconds=60,
+            )
+
+        records = cast("list[dict[str, object]]", result.to_dict("records"))
+        assert records == [
+            {
+                "table": "rate_EURUSD__M1_1",
+                "symbol": "EURUSD",
+                "timeframe": 1,
+                "granularity": "M1",
+                "granularity_seconds": 60,
+                "gap_start": datetime(2024, 1, 1, 0, 2, tzinfo=UTC),
+                "gap_end": datetime(2024, 1, 1, 0, 2, tzinfo=UTC),
+                "missing_intervals": 1,
+            },
+        ]
+
+    def test_report_rate_gaps_parses_numeric_sqlite_times_as_epoch_seconds(
+        self,
+        tmp_path: Path,
+    ) -> None:
+        """Numeric SQLite timestamps must be interpreted as Unix seconds."""
+        db_path = tmp_path / "numeric-gaps.db"
+        with sqlite3.connect(db_path) as conn:
+            conn.execute("CREATE TABLE custom_rates(time INTEGER, close REAL)")
+            conn.executemany(
+                "INSERT INTO custom_rates(time, close) VALUES (?, ?)",
+                [
+                    (1704067200, 1.0),
+                    (1704067320, 1.1),
+                ],
+            )
+            result = report_rate_gaps(
+                conn,
+                "custom_rates",
+                granularity_seconds=60,
+            )
+
+        assert len(result) == 1
+        assert result.iloc[0]["missing_intervals"] == 1
+
+    def test_report_rate_gaps_empty_schema_for_zero_gap_tables(
+        self,
+        tmp_path: Path,
+    ) -> None:
+        """Tables without gaps return the stable empty result schema."""
+        db_path = tmp_path / "no-gaps.db"
+        with sqlite3.connect(db_path) as conn:
+            conn.execute("CREATE TABLE custom_rates(time TEXT, close REAL)")
+            conn.executemany(
+                "INSERT INTO custom_rates(time, close) VALUES (?, ?)",
+                [
+                    ("2024-01-01T00:00:00+00:00", 1.0),
+                    ("2024-01-01T00:01:00+00:00", 1.1),
+                ],
+            )
+            result = report_rate_gaps(
+                conn,
+                "custom_rates",
+                granularity_seconds=60,
+            )
+
+        assert list(result.columns) == [
+            "table",
+            "symbol",
+            "timeframe",
+            "granularity",
+            "granularity_seconds",
+            "gap_start",
+            "gap_end",
+            "missing_intervals",
+        ]
+        assert result.empty
+
+    def test_report_rate_gaps_filters_by_min_gap_intervals(
+        self,
+        tmp_path: Path,
+    ) -> None:
+        """Small gaps are filtered out when min_gap_intervals is raised."""
+        db_path = tmp_path / "filtered-gaps.db"
+        with sqlite3.connect(db_path) as conn:
+            conn.execute("CREATE TABLE custom_rates(time TEXT, close REAL)")
+            conn.executemany(
+                "INSERT INTO custom_rates(time, close) VALUES (?, ?)",
+                [
+                    ("2024-01-01T00:00:00+00:00", 1.0),
+                    ("2024-01-01T00:02:00+00:00", 1.1),
+                ],
+            )
+            result = report_rate_gaps(
+                conn,
+                "custom_rates",
+                granularity_seconds=60,
+                min_gap_intervals=2,
+            )
+
+        assert result.empty
+
+    def test_report_rate_gaps_computes_gaps_per_series_key(
+        self,
+        tmp_path: Path,
+    ) -> None:
+        """Managed rates tables must detect gaps within each symbol/timeframe series."""
+        db_path = tmp_path / "series-gaps.db"
+        with sqlite3.connect(db_path) as conn:
+            conn.execute(
+                "CREATE TABLE rates("
+                "symbol TEXT, timeframe INTEGER, time TEXT, close REAL"
+                ")",
+            )
+            conn.executemany(
+                "INSERT INTO rates(symbol, timeframe, time, close) VALUES (?, ?, ?, ?)",
+                [
+                    ("EURUSD", 1, "2024-01-01T00:00:00+00:00", 1.0),
+                    ("GBPUSD", 1, "2024-01-01T00:01:00+00:00", 1.1),
+                    ("EURUSD", 1, "2024-01-01T00:02:00+00:00", 1.2),
+                ],
+            )
+
+            result = report_rate_gaps(
+                conn,
+                "rates",
+                granularity_seconds=60,
+            )
+
+        records = cast("list[dict[str, object]]", result.to_dict("records"))
+        assert records == [
+            {
+                "table": "rates",
+                "symbol": "EURUSD",
+                "timeframe": 1,
+                "granularity": "M1",
+                "granularity_seconds": 60,
+                "gap_start": datetime(2024, 1, 1, 0, 1, tzinfo=UTC),
+                "gap_end": datetime(2024, 1, 1, 0, 1, tzinfo=UTC),
+                "missing_intervals": 1,
+            },
+        ]
+
+    def test_rate_gap_private_helpers_and_validation(self) -> None:
+        """Private helpers should preserve schema and reject invalid inputs."""
+        empty = history._empty_rate_gap_report()  # type: ignore[attr-defined]
+        assert list(empty.columns) == [
+            "table",
+            "symbol",
+            "timeframe",
+            "granularity",
+            "granularity_seconds",
+            "gap_start",
+            "gap_end",
+            "missing_intervals",
+        ]
+
+        class _BadInt:
+            def __int__(self) -> int:
+                msg = "bad-int"
+                raise ValueError(msg)
+
+        assert history._coerce_optional_int(None) is None  # type: ignore[attr-defined]
+        false_value: object = False
+        assert history._coerce_optional_int(false_value) is None  # type: ignore[attr-defined]
+        assert history._coerce_optional_int(" +7 ") == 7  # type: ignore[attr-defined]
+        assert history._coerce_optional_int("bad") is None  # type: ignore[attr-defined]
+        assert history._coerce_optional_int(object()) is None  # type: ignore[attr-defined]
+        assert history._coerce_optional_int(_BadInt()) is None  # type: ignore[attr-defined]
+
+        metadata = history._rate_gap_metadata(  # type: ignore[attr-defined]
+            "custom_rates",
+            pd.DataFrame({
+                "symbol": ["EURUSD", "GBPUSD"],
+                "timeframe": [1, 1],
+            }),
+            granularity_seconds=60,
+        )
+        assert metadata["symbol"] is None
+        assert metadata["timeframe"] == 1
+        assert metadata["granularity"] == "M1"
+
+        fallback_metadata = history._rate_gap_metadata(  # type: ignore[attr-defined]
+            "rate_USDJPY__M1_1",
+            pd.DataFrame({"time": []}),
+            granularity_seconds=60,
+        )
+        assert fallback_metadata["symbol"] == "USDJPY"
+        assert fallback_metadata["timeframe"] == 1
+        assert fallback_metadata["granularity"] == "M1"
+
+        unique_symbol_metadata = history._rate_gap_metadata(  # type: ignore[attr-defined]
+            "custom_rates",
+            pd.DataFrame({"symbol": ["EURUSD"], "timeframe": [1]}),
+            granularity_seconds=60,
+        )
+        assert unique_symbol_metadata["symbol"] == "EURUSD"
+
+        multi_timeframe_metadata = history._rate_gap_metadata(  # type: ignore[attr-defined]
+            "custom_rates",
+            pd.DataFrame({"timeframe": [1, 5]}),
+            granularity_seconds=60,
+        )
+        assert multi_timeframe_metadata["timeframe"] is None
+        assert multi_timeframe_metadata["granularity"] is None
+
+    @pytest.mark.parametrize(
+        ("granularity_seconds", "min_gap_intervals", "match"),
+        [
+            pytest.param(
+                0, 1, "granularity_seconds must be positive", id="bad-seconds"
+            ),
+            pytest.param(60, 0, "min_gap_intervals must be positive", id="bad-min-gap"),
+        ],
+    )
+    def test_report_rate_gaps_rejects_invalid_parameters(
+        self,
+        tmp_path: Path,
+        granularity_seconds: int,
+        min_gap_intervals: int,
+        match: str,
+    ) -> None:
+        """Gap reports reject non-positive granularity and min-gap values."""
+        db_path = tmp_path / "invalid-gaps.db"
+        with sqlite3.connect(db_path) as conn:
+            conn.execute("CREATE TABLE custom_rates(time TEXT, close REAL)")
+            conn.execute(
+                "INSERT INTO custom_rates(time, close) VALUES (?, ?)",
+                ("2024-01-01T00:00:00+00:00", 1.0),
+            )
+            with pytest.raises(ValueError, match=match):
+                report_rate_gaps(
+                    conn,
+                    "custom_rates",
+                    granularity_seconds=granularity_seconds,
+                    min_gap_intervals=min_gap_intervals,
+                )
+
+    def test_report_rate_gaps_rejects_unparseable_timestamps(
+        self,
+        tmp_path: Path,
+    ) -> None:
+        """Unparseable table times should fail clearly."""
+        db_path = tmp_path / "bad-times.db"
+        with sqlite3.connect(db_path) as conn:
+            conn.execute("CREATE TABLE custom_rates(time TEXT, close REAL)")
+            conn.execute(
+                "INSERT INTO custom_rates(time, close) VALUES (?, ?)",
+                ("bad-time", 1.0),
+            )
+            with pytest.raises(ValueError, match="contains unparsable time values"):
+                report_rate_gaps(
+                    conn,
+                    "custom_rates",
+                    granularity_seconds=60,
+                )
+
+    def test_report_rate_gaps_empty_and_single_row_sources_return_empty(
+        self,
+        tmp_path: Path,
+    ) -> None:
+        """Empty or single-row sources cannot produce gap rows."""
+        db_path = tmp_path / "too-short.db"
+        with sqlite3.connect(db_path) as conn:
+            conn.execute("CREATE TABLE custom_rates(time TEXT, close REAL)")
+            assert report_rate_gaps(conn, "custom_rates", granularity_seconds=60).empty
+            conn.execute(
+                "INSERT INTO custom_rates(time, close) VALUES (?, ?)",
+                ("2024-01-01T00:00:00+00:00", 1.0),
+            )
+            assert report_rate_gaps(conn, "custom_rates", granularity_seconds=60).empty
+
    def test_load_rate_series_by_granularity_explicit_tables(
        self,
        tmp_path: Path,