From 18df96872b5cb213fadc421a4f85b6d741502742 Mon Sep 17 00:00:00 2001 From: Daichi Narushima <1938249+dceoy@users.noreply.github.com> Date: Thu, 11 Jun 2026 02:30:48 +0900 Subject: [PATCH] Add closed-bar rate helpers (v0.6.0) (#26) * Add closed-bar rate helpers and bump version to 0.6.0. Expose drop_forming_rate_bar and multi-account collectors so downstream apps no longer need count+1 fetches and manual bar trimming. Co-authored-by: Cursor * Bump pygments to 2.20.0 to fix CVE-2026-4539 ReDoS advisory. Co-authored-by: Cursor * Address PR review feedback on closed-bar rate collection. Validate count and start_pos before MT5 fetches, avoid redundant frame copies, clarify empty-series errors, and expand test coverage. Co-authored-by: Cursor * Include symbol and timeframe in empty closed-rate error messages. Co-authored-by: Cursor --------- Co-authored-by: Cursor --- README.md | 14 +++ docs/api/sdk.md | 20 +++++ mt5cli/__init__.py | 6 ++ mt5cli/history.py | 17 ++++ mt5cli/sdk.py | 119 ++++++++++++++++++++++++++ pyproject.toml | 2 +- tests/test_history.py | 41 +++++++++ tests/test_sdk.py | 195 ++++++++++++++++++++++++++++++++++++++++++ uv.lock | 8 +- 9 files changed, 417 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 3bed10c..ba9183a 100644 --- a/README.md +++ b/README.md @@ -137,6 +137,20 @@ update_history_with_config( - **Rate view loading**: use `load_rate_data()` / `load_rate_data_from_connection()` to load a SQLite rate table or view into a `DatetimeIndex` DataFrame. - **Multi-series rate loading**: use `build_rate_targets()` to build neutral `RateTarget(symbol, timeframe)` pairs, `resolve_rate_tables()` to map them to table/view names (pass `require_existing=True` for strict resolution), and `load_rate_series_from_sqlite()` to load them into a mapping keyed by `(symbol, integer timeframe)`. The loader requires existing managed views unless `explicit_tables` is supplied, and rejects duplicate `(symbol, timeframe)` targets. - **Multi-account latest rates**: use `collect_latest_rates_for_accounts()` with `AccountSpec` to read the latest bars for several account groups, merged into a `(symbol, integer timeframe)` mapping. For long-running pollers, `collect_latest_rates_for_accounts_with_retries()` adds bounded exponential backoff that retries only `pdmt5.Mt5TradingError` / `pdmt5.Mt5RuntimeError` and re-raises once `retry_count` is exhausted. +- **Latest closed bars**: use `collect_latest_closed_rates_for_accounts()` when downstream logic must exclude the still-forming current bar. It fetches `count + 1` bars at `start_pos=0`, drops the last row with `drop_forming_rate_bar()`, and validates each series is non-empty. `collect_latest_closed_rates_by_granularity()` returns the same data keyed by `(symbol, granularity_name)` such as `("EURUSD", "M1")`. + +```python +from mt5cli import AccountSpec, collect_latest_closed_rates_by_granularity + +rates = collect_latest_closed_rates_by_granularity( + [AccountSpec(symbols=["EURUSD", "GBPUSD"], login=12345)], + ["M1", "H1"], + count=500, + retry_count=3, +) +eurusd_m1 = rates["EURUSD", "M1"] # closed bars only +``` + - **Credential resolution**: use `resolve_account_spec()` / `resolve_account_specs()` to merge explicit override values over `AccountSpec` fields and expand `${ENV_VAR}` placeholders (via `substitute_env_placeholders()`), raising `ValueError` for missing variables. This keeps secrets out of plan/config files without coupling to any strategy code. - **Throttled history updates**: use `ThrottledHistoryUpdater` to wrap `update_history()` with a minimum `interval_seconds` between successful runs (monotonic clock). Call `should_update()` / `update(client, symbols)` from an application loop; errors propagate by default, or pass `suppress_errors=True` to swallow recoverable `Mt5*Error`/`sqlite3.Error` and let the caller decide logging. - **Granularity-keyed rate loading**: `load_rate_series_by_granularity()` builds targets with `build_rate_targets()`, loads them with `load_rate_series_from_sqlite()`, and returns a mapping keyed by `(symbol | None, granularity_name)` such as `("EURUSD", "M1")` to reduce downstream boilerplate. diff --git a/docs/api/sdk.md b/docs/api/sdk.md index adb6629..7301f3a 100644 --- a/docs/api/sdk.md +++ b/docs/api/sdk.md @@ -28,6 +28,26 @@ rates = collect_latest_rates_for_accounts_with_retries( ) ``` +### Latest closed rate bars + +MetaTrader 5 `start_pos=0` includes the still-forming current bar as the last +row. `collect_latest_closed_rates_for_accounts()` fetches `count + 1` bars, +drops that row with `drop_forming_rate_bar()`, and validates each series is +non-empty. Use `collect_latest_closed_rates_by_granularity()` when callers +prefer keys such as `("EURUSD", "M1")` instead of integer timeframes. + +```python +from mt5cli import AccountSpec, collect_latest_closed_rates_by_granularity + +rates = collect_latest_closed_rates_by_granularity( + [AccountSpec(symbols=["EURUSD"], login=12345)], + ["M1", "H1"], + count=500, + retry_count=3, +) +closed_m1 = rates["EURUSD", "M1"] +``` + ### Resolving credentials and `${ENV_VAR}` placeholders `resolve_account_spec()` / `resolve_account_specs()` merge explicit override diff --git a/mt5cli/__init__.py b/mt5cli/__init__.py index 4eb58fc..94d5a1c 100644 --- a/mt5cli/__init__.py +++ b/mt5cli/__init__.py @@ -6,6 +6,7 @@ from .history import ( RateTarget, build_rate_targets, build_rate_view_name, + drop_forming_rate_bar, load_rate_data, load_rate_data_from_connection, load_rate_series_by_granularity, @@ -24,6 +25,8 @@ from .sdk import ( account_info, build_config, collect_history, + collect_latest_closed_rates_by_granularity, + collect_latest_closed_rates_for_accounts, collect_latest_rates, collect_latest_rates_for_accounts, collect_latest_rates_for_accounts_with_retries, @@ -87,6 +90,8 @@ __all__ = [ "build_rate_targets", "build_rate_view_name", "collect_history", + "collect_latest_closed_rates_by_granularity", + "collect_latest_closed_rates_for_accounts", "collect_latest_rates", "collect_latest_rates_for_accounts", "collect_latest_rates_for_accounts_with_retries", @@ -96,6 +101,7 @@ __all__ = [ "copy_ticks_from", "copy_ticks_range", "detect_format", + "drop_forming_rate_bar", "export_dataframe", "export_dataframe_to_sqlite", "history_deals", diff --git a/mt5cli/history.py b/mt5cli/history.py index 02e9665..23621b5 100644 --- a/mt5cli/history.py +++ b/mt5cli/history.py @@ -106,6 +106,23 @@ def resolve_granularity_name(timeframe: int) -> str: return str(timeframe) +def drop_forming_rate_bar(df_rate: pd.DataFrame) -> pd.DataFrame: + """Return closed bars from chronologically ordered MT5 rate data. + + MetaTrader 5 ``copy_rates_from_pos(start_pos=0)`` includes the still-forming + current bar as the last row. Slice it off so downstream logic only sees + completed bars. Empty frames and single-row frames return empty results. + + Args: + df_rate: Rate data ordered oldest-to-newest with the forming bar last. + + Returns: + A new DataFrame with all rows except the last. Index and columns are + preserved. The input frame is not modified. + """ + return df_rate.iloc[:-1].copy() + + def build_rate_view_name( *, symbol: str, diff --git a/mt5cli/sdk.py b/mt5cli/sdk.py index 2a046d8..b586bb3 100644 --- a/mt5cli/sdk.py +++ b/mt5cli/sdk.py @@ -21,6 +21,8 @@ from .history import ( create_cash_events_view, create_history_indexes, create_positions_reconstructed_view, + drop_forming_rate_bar, + resolve_granularity_name, resolve_history_datasets, resolve_history_tick_flags, resolve_history_timeframes, @@ -49,6 +51,8 @@ __all__ = [ "account_info", "build_config", "collect_history", + "collect_latest_closed_rates_by_granularity", + "collect_latest_closed_rates_for_accounts", "collect_latest_rates", "collect_latest_rates_for_accounts", "collect_latest_rates_for_accounts_with_retries", @@ -129,6 +133,12 @@ def _require_positive(value: float, name: str) -> None: raise ValueError(msg) +def _require_non_negative(value: int, name: str) -> None: + if value < 0: + msg = f"{name} must be non-negative." + raise ValueError(msg) + + def _call_required_client_method(client: Mt5DataClient, name: str) -> object: try: method = getattr(client, name) @@ -1533,6 +1543,115 @@ def collect_latest_rates_for_accounts_with_retries( return _collect() +def collect_latest_closed_rates_for_accounts( + accounts: Sequence[AccountSpec], + timeframes: Sequence[int | str], + count: int, + *, + start_pos: int = 0, + base_config: Mt5Config | None = None, + retry_count: int = 0, + backoff_base: float = 2.0, +) -> dict[tuple[str, int], pd.DataFrame]: + """Collect latest closed rate bars across multiple MT5 account groups. + + When ``start_pos`` is ``0`` (the default), MetaTrader 5 includes the + still-forming current bar as the last row. This helper fetches + ``count + 1`` bars, drops that bar with :func:`drop_forming_rate_bar`, and + validates that each resulting frame is non-empty. When ``start_pos`` is + greater than zero the forming bar is not in range, so only ``count`` bars + are fetched and no row is dropped. + + Wraps :func:`collect_latest_rates_for_accounts_with_retries` for transient + MT5 error handling. + + Args: + accounts: Account groups to read. Each must define at least one symbol. + timeframes: MT5 timeframes as integers or names (for example ``M1``). + count: Number of closed bars to return per symbol/timeframe. + start_pos: Initial bar position offset passed to the underlying collector. + base_config: Optional base configuration whose fields fill any value not + set on an individual account. + retry_count: Maximum number of retries after the first attempt. ``0`` + disables retries. + backoff_base: Base for exponential backoff between retry attempts. + + Returns: + Mapping keyed by ``(symbol, timeframe_int)``. + + Raises: + ValueError: If inputs are invalid, or any series is empty (after + dropping the still-forming bar when ``start_pos`` is ``0``). + """ + _require_positive(count, "count") + _require_non_negative(start_pos, "start_pos") + fetch_count = count + 1 if start_pos == 0 else count + loaded = collect_latest_rates_for_accounts_with_retries( + accounts, + timeframes, + fetch_count, + start_pos=start_pos, + base_config=base_config, + retry_count=retry_count, + backoff_base=backoff_base, + ) + result: dict[tuple[str, int], pd.DataFrame] = {} + for key, df_rate in loaded.items(): + closed = drop_forming_rate_bar(df_rate) if start_pos == 0 else df_rate + if closed.empty: + symbol, timeframe = key + msg = f"Rate data is empty for {symbol!r} at timeframe {timeframe}." + raise ValueError(msg) + result[key] = closed + return result + + +def collect_latest_closed_rates_by_granularity( + accounts: Sequence[AccountSpec], + granularities: Sequence[int | str], + count: int, + *, + start_pos: int = 0, + base_config: Mt5Config | None = None, + retry_count: int = 0, + backoff_base: float = 2.0, +) -> dict[tuple[str, str], pd.DataFrame]: + """Collect latest closed rate bars keyed by symbol and granularity name. + + Thin wrapper around :func:`collect_latest_closed_rates_for_accounts` that + rekeys the result by granularity name (for example ``M1``) instead of the + integer timeframe. + + Args: + accounts: Account groups to read. Each must define at least one symbol. + granularities: MT5 timeframes as integers or names (for example ``M1``). + count: Number of closed bars to return per symbol/timeframe. + start_pos: Initial bar position offset passed to the underlying collector. + base_config: Optional base configuration whose fields fill any value not + set on an individual account. + retry_count: Maximum number of retries after the first attempt. ``0`` + disables retries. + backoff_base: Base for exponential backoff between retry attempts. + + Returns: + Mapping keyed by ``(symbol, granularity_name)``. Propagates + ``ValueError`` from :func:`collect_latest_closed_rates_for_accounts`. + """ + loaded = collect_latest_closed_rates_for_accounts( + accounts, + granularities, + count, + start_pos=start_pos, + base_config=base_config, + retry_count=retry_count, + backoff_base=backoff_base, + ) + return { + (symbol, resolve_granularity_name(timeframe)): frame + for (symbol, timeframe), frame in loaded.items() + } + + def copy_rates_range( symbol: str, timeframe: int | str, diff --git a/pyproject.toml b/pyproject.toml index 788b9c1..7659f43 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "mt5cli" -version = "0.5.3" +version = "0.6.0" description = "Command-line tool for MetaTrader 5" authors = [{name = "dceoy", email = "dceoy@users.noreply.github.com"}] maintainers = [{name = "dceoy", email = "dceoy@users.noreply.github.com"}] diff --git a/tests/test_history.py b/tests/test_history.py index 4e9da81..1ab284e 100644 --- a/tests/test_history.py +++ b/tests/test_history.py @@ -30,6 +30,7 @@ from mt5cli.history import ( create_rate_compatibility_views, deduplicate_history_tables, drop_duplicates_in_table, + drop_forming_rate_bar, filter_incremental_history_deals_frame, filter_trade_history_frame, get_history_deals_account_event_start_datetime, @@ -534,6 +535,46 @@ class TestResolveHistorySettings: assert resolve_granularity_name(1) == "M1" +class TestDropFormingRateBar: + """Tests for drop_forming_rate_bar.""" + + def test_drops_still_forming_last_bar(self) -> None: + """Test the still-forming last bar is removed.""" + df_rate = pd.DataFrame( + {"time": [1, 2, 3], "close": [1.1, 1.2, 1.3]}, + index=pd.Index(["a", "b", "c"], name="idx"), + ) + + result = drop_forming_rate_bar(df_rate) + + pd.testing.assert_frame_equal( + result, + pd.DataFrame( + {"time": [1, 2], "close": [1.1, 1.2]}, + index=pd.Index(["a", "b"], name="idx"), + ), + ) + assert df_rate.shape == (3, 2) + + def test_returns_empty_frame_when_input_empty(self) -> None: + """Test empty frames stay empty.""" + df_rate = pd.DataFrame(columns=["time", "close"]) + + result = drop_forming_rate_bar(df_rate) + + assert result.empty + assert list(result.columns) == ["time", "close"] + + def test_returns_empty_frame_when_only_forming_bar_present(self) -> None: + """Test a single-bar frame becomes empty after dropping the forming bar.""" + df_rate = pd.DataFrame({"time": [1], "close": [1.1]}) + + result = drop_forming_rate_bar(df_rate) + + assert result.empty + assert list(result.columns) == ["time", "close"] + + class TestParseSqliteTimestamp: """Tests for parse_sqlite_timestamp.""" diff --git a/tests/test_sdk.py b/tests/test_sdk.py index b971989..7b976c4 100644 --- a/tests/test_sdk.py +++ b/tests/test_sdk.py @@ -27,6 +27,8 @@ from mt5cli.sdk import ( account_info, build_config, collect_history, + collect_latest_closed_rates_by_granularity, + collect_latest_closed_rates_for_accounts, collect_latest_rates, collect_latest_rates_for_accounts, collect_latest_rates_for_accounts_with_retries, @@ -1523,6 +1525,199 @@ class TestCollectLatestRatesForAccountsWithRetries: sleep.assert_not_called() +class TestCollectLatestClosedRatesForAccounts: + """Tests for collect_latest_closed_rates_for_accounts.""" + + def test_fetches_count_plus_one_and_drops_forming_bar( + self, + mocker: MockerFixture, + ) -> None: + """Test closed-bar collection requests one extra bar at start_pos=0.""" + df_rate = pd.DataFrame({"time": [1, 2, 3], "close": [1.1, 1.2, 1.3]}) + wrapped = mocker.patch( + "mt5cli.sdk.collect_latest_rates_for_accounts_with_retries", + return_value={("EURUSD", 1): df_rate}, + ) + accounts = [AccountSpec(symbols=["EURUSD"])] + + result = collect_latest_closed_rates_for_accounts( + accounts, + ["M1"], + count=2, + retry_count=1, + backoff_base=3, + ) + + wrapped.assert_called_once_with( + accounts, + ["M1"], + 3, + start_pos=0, + base_config=None, + retry_count=1, + backoff_base=3, + ) + pd.testing.assert_frame_equal( + result["EURUSD", 1], + pd.DataFrame({"time": [1, 2], "close": [1.1, 1.2]}), + ) + + def test_rejects_forming_bar_only_frames(self, mocker: MockerFixture) -> None: + """Test empty results after dropping the forming bar raise ValueError.""" + mocker.patch( + "mt5cli.sdk.collect_latest_rates_for_accounts_with_retries", + return_value={("EURUSD", 1): pd.DataFrame({"time": [1], "close": [1.1]})}, + ) + + with pytest.raises(ValueError, match="Rate data is empty"): + collect_latest_closed_rates_for_accounts( + [AccountSpec(symbols=["EURUSD"])], + ["M1"], + count=1, + ) + + def test_skips_extra_fetch_when_start_pos_nonzero( + self, + mocker: MockerFixture, + ) -> None: + """Test start_pos > 0 fetches count bars without dropping the last row.""" + df_rate = pd.DataFrame({"time": [1, 2], "close": [1.1, 1.2]}) + wrapped = mocker.patch( + "mt5cli.sdk.collect_latest_rates_for_accounts_with_retries", + return_value={("EURUSD", 1): df_rate}, + ) + + result = collect_latest_closed_rates_for_accounts( + [AccountSpec(symbols=["EURUSD"])], + ["M1"], + count=2, + start_pos=1, + ) + + wrapped.assert_called_once_with( + [AccountSpec(symbols=["EURUSD"])], + ["M1"], + 2, + start_pos=1, + base_config=None, + retry_count=0, + backoff_base=2.0, + ) + pd.testing.assert_frame_equal(result["EURUSD", 1], df_rate) + + def test_rejects_zero_count_before_fetching(self, mocker: MockerFixture) -> None: + """Test count=0 is rejected before any MT5 collection attempt.""" + wrapped = mocker.patch( + "mt5cli.sdk.collect_latest_rates_for_accounts_with_retries", + ) + + with pytest.raises(ValueError, match="count must be positive"): + collect_latest_closed_rates_for_accounts( + [AccountSpec(symbols=["EURUSD"])], + ["M1"], + count=0, + ) + + wrapped.assert_not_called() + + def test_rejects_negative_start_pos(self, mocker: MockerFixture) -> None: + """Test negative start_pos is rejected before any MT5 collection attempt.""" + wrapped = mocker.patch( + "mt5cli.sdk.collect_latest_rates_for_accounts_with_retries", + ) + + with pytest.raises(ValueError, match="start_pos must be non-negative"): + collect_latest_closed_rates_for_accounts( + [AccountSpec(symbols=["EURUSD"])], + ["M1"], + count=1, + start_pos=-1, + ) + + wrapped.assert_not_called() + + def test_rejects_empty_frames_with_start_pos_nonzero( + self, + mocker: MockerFixture, + ) -> None: + """Test empty upstream frames raise ValueError when start_pos > 0.""" + mocker.patch( + "mt5cli.sdk.collect_latest_rates_for_accounts_with_retries", + return_value={("EURUSD", 1): pd.DataFrame(columns=["time", "close"])}, + ) + + with pytest.raises(ValueError, match="Rate data is empty"): + collect_latest_closed_rates_for_accounts( + [AccountSpec(symbols=["EURUSD"])], + ["M1"], + count=1, + start_pos=1, + ) + + def test_processes_multiple_symbol_timeframe_pairs( + self, + mocker: MockerFixture, + ) -> None: + """Test each returned series is trimmed and validated independently.""" + mocker.patch( + "mt5cli.sdk.collect_latest_rates_for_accounts_with_retries", + return_value={ + ("EURUSD", 1): pd.DataFrame( + {"time": [1, 2, 3], "close": [1.1, 1.2, 1.3]}, + ), + ("GBPUSD", 16385): pd.DataFrame( + {"time": [4, 5, 6], "close": [2.1, 2.2, 2.3]}, + ), + }, + ) + + result = collect_latest_closed_rates_for_accounts( + [AccountSpec(symbols=["EURUSD", "GBPUSD"])], + ["M1", "H1"], + count=2, + ) + + assert set(result) == {("EURUSD", 1), ("GBPUSD", 16385)} + pd.testing.assert_frame_equal( + result["EURUSD", 1], + pd.DataFrame({"time": [1, 2], "close": [1.1, 1.2]}), + ) + pd.testing.assert_frame_equal( + result["GBPUSD", 16385], + pd.DataFrame({"time": [4, 5], "close": [2.1, 2.2]}), + ) + + +class TestCollectLatestClosedRatesByGranularity: + """Tests for collect_latest_closed_rates_by_granularity.""" + + def test_rekeys_by_granularity_name(self, mocker: MockerFixture) -> None: + """Test closed rates are keyed by symbol and granularity name.""" + df_rate = pd.DataFrame({"time": [1, 2], "close": [1.1, 1.2]}) + wrapped = mocker.patch( + "mt5cli.sdk.collect_latest_closed_rates_for_accounts", + return_value={("EURUSD", 1): df_rate}, + ) + + result = collect_latest_closed_rates_by_granularity( + [AccountSpec(symbols=["EURUSD"])], + ["M1"], + count=2, + ) + + wrapped.assert_called_once_with( + [AccountSpec(symbols=["EURUSD"])], + ["M1"], + 2, + start_pos=0, + base_config=None, + retry_count=0, + backoff_base=2.0, + ) + assert ("EURUSD", "M1") in result + pd.testing.assert_frame_equal(result["EURUSD", "M1"], df_rate) + + class TestSubstituteEnvPlaceholders: """Tests for ${ENV_VAR} substitution.""" diff --git a/uv.lock b/uv.lock index b5683a0..c30bb51 100644 --- a/uv.lock +++ b/uv.lock @@ -487,7 +487,7 @@ wheels = [ [[package]] name = "mt5cli" -version = "0.5.3" +version = "0.6.0" source = { editable = "." } dependencies = [ { name = "click" }, @@ -836,11 +836,11 @@ wheels = [ [[package]] name = "pygments" -version = "2.19.2" +version = "2.20.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/b0/77/a5b8c569bf593b0140bde72ea885a803b82086995367bf2037de0159d924/pygments-2.19.2.tar.gz", hash = "sha256:636cb2477cec7f8952536970bc533bc43743542f70392ae026374600add5b887", size = 4968631, upload-time = "2025-06-21T13:39:12.283Z" } +sdist = { url = "https://files.pythonhosted.org/packages/c3/b2/bc9c9196916376152d655522fdcebac55e66de6603a76a02bca1b6414f6c/pygments-2.20.0.tar.gz", hash = "sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f", size = 4955991, upload-time = "2026-03-29T13:29:33.898Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/c7/21/705964c7812476f378728bdf590ca4b771ec72385c533964653c68e86bdc/pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b", size = 1225217, upload-time = "2025-06-21T13:39:07.939Z" }, + { url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151, upload-time = "2026-03-29T13:29:30.038Z" }, ] [[package]]