From a983c8094ee5ce0b8b45deef095bee14073b5547 Mon Sep 17 00:00:00 2001 From: "2569718930@qq.com" <2569718930@qq.com> Date: Tue, 30 Jun 2026 17:06:33 +0800 Subject: [PATCH] Keep scan terminal model forecasts --- tests/test_scan_terminal_modules.py | 148 ++++++++++++++++++++++++++++ web/scan_terminal_city_row.py | 57 ++++++++++- web/scan_terminal_service.py | 15 +++ 3 files changed, 219 insertions(+), 1 deletion(-) diff --git a/tests/test_scan_terminal_modules.py b/tests/test_scan_terminal_modules.py index b5013422..e741b139 100644 --- a/tests/test_scan_terminal_modules.py +++ b/tests/test_scan_terminal_modules.py @@ -446,6 +446,84 @@ def test_scan_city_terminal_rows_refreshes_models_for_wrong_local_date_panel(mon ] +def test_scan_city_terminal_rows_reuses_cached_today_models_when_direct_fetch_disabled(monkeypatch): + today = _local_date_for_offset(3600) + enqueued = [] + payload = { + "display_name": "Paris", + "local_date": "2000-01-01", + "local_time": "12:00", + "utc_offset_seconds": 3600, + "current": {"max_so_far": 20.0}, + "risk": {}, + "deb": {"prediction": 20.0}, + "probabilities": {}, + "multi_model_daily": { + today: { + "deb": {"prediction": 22.4}, + "models": {"ECMWF": 22.0, "GFS": 23.0}, + } + }, + } + + class _Cache: + @staticmethod + def get_city_cache(kind, city): + assert (kind, city) == ("panel", "paris") + return {"payload": payload, "updated_at_ts": 1} + + @staticmethod + def get_canonical_temperature(city): + assert city == "paris" + return None + + @staticmethod + def enqueue_observation_refresh_request(**kwargs): + enqueued.append(kwargs) + return True + + monkeypatch.setattr( + scan_terminal_city_row, + "CITIES", + {"paris": {"lat": 48.85, "lon": 2.35, "tz": 3600}}, + ) + monkeypatch.setattr(scan_terminal_city_row, "_PANEL_CACHE_DB", _Cache()) + monkeypatch.setattr( + scan_terminal_city_row._weather, + "fetch_multi_model", + lambda *_args, **_kwargs: (_ for _ in ()).throw( + AssertionError("direct multi-model fetch should stay disabled") + ), + ) + monkeypatch.setattr( + scan_terminal_city_row, + "calculate_deb_prediction", + lambda city, forecasts, raw_calculator=None: {"prediction": 22.4}, + ) + + result = scan_terminal_city_row._scan_city_terminal_rows( + "paris", + {"market_type": "maxtemp"}, + force_refresh=False, + allow_direct_fetch=False, + ) + + row = result["rows"][0] + assert row["local_date"] == today + assert row["forecast_refreshed"] is True + assert row["forecast_source_local_date"] == today + assert row["deb_prediction"] == 22.4 + assert row["model_cluster_sources"] == {"ECMWF": 22.0, "GFS": 23.0} + assert enqueued == [ + { + "city": "paris", + "kind": "panel", + "priority": "high", + "reason": "scan_terminal_stale_panel", + } + ] + + def test_scan_terminal_fetches_multi_model_daily_in_batches(monkeypatch): today_paris = _local_date_for_offset(3600) today_houston = _local_date_for_offset(-18000) @@ -710,6 +788,76 @@ def test_scan_timeout_prefers_partial_rows_with_models_over_blank_stale_cache(mo assert stale is None +def test_scan_terminal_refresh_without_model_rows_keeps_previous_model_snapshot(monkeypatch): + cached_entry = { + "success_payload": { + "generated_at": "2026-06-01T00:00:00Z", + "snapshot_id": "good-model-snapshot", + "rows": [ + { + "id": "paris:today", + "market_key": "paris:today", + "model_cluster_sources": {"ECMWF": 24.0}, + } + ], + "summary": {"candidate_total": 1}, + "top_signal": None, + }, + "last_failed_at": "2026-06-01T00:01:00Z", + } + failures = [] + + monkeypatch.setattr(scan_terminal_service, "CITIES", {"paris": {"tz": 3600}}) + monkeypatch.setattr(scan_terminal_service, "_fetch_scan_terminal_multi_model_batch", lambda _cities: {}) + monkeypatch.setattr( + scan_terminal_service, + "get_scan_terminal_cache_entry", + lambda _filters: cached_entry, + ) + monkeypatch.setattr( + scan_terminal_service, + "set_scan_terminal_failure_state", + lambda _filters, *, error_message: failures.append(error_message), + ) + monkeypatch.setattr( + scan_terminal_service, + "set_cached_scan_terminal_payload", + lambda *_args, **_kwargs: (_ for _ in ()).throw( + AssertionError("blank model refresh must not overwrite success payload") + ), + ) + + def _scan_city(*_args, **_kwargs): + return { + "city": "paris", + "candidate_total": 1, + "primary_scores": [0.0], + "rows": [ + { + "id": "paris:today", + "market_key": "paris:today", + "final_score": 0.0, + "edge_percent": 0.0, + "volume": 0.0, + "model_cluster_sources": {}, + } + ], + } + + monkeypatch.setattr(scan_terminal_service, "_scan_city_terminal_rows", _scan_city) + + payload = scan_terminal_service._build_scan_terminal_payload_uncached( + {"limit": 1}, + timeout_sec=5, + ) + + assert payload["status"] == "stale" + assert payload["stale"] is True + assert payload["snapshot_id"] == "good-model-snapshot" + assert payload["rows"][0]["model_cluster_sources"] == {"ECMWF": 24.0} + assert failures == ["scan terminal refresh returned rows without model forecasts"] + + def test_scan_city_terminal_rows_uses_canonical_without_analyze(monkeypatch): enqueued = [] diff --git a/web/scan_terminal_city_row.py b/web/scan_terminal_city_row.py index 18c4fc5a..6af494f2 100644 --- a/web/scan_terminal_city_row.py +++ b/web/scan_terminal_city_row.py @@ -93,6 +93,47 @@ def _panel_cache_stale_reason(city: str, cached_entry: Dict[str, Any], payload: return None +def _cached_panel_multi_model_for_local_date( + payload: Dict[str, Any], + local_date: str, + *, + use_fahrenheit: bool, +) -> Optional[Dict[str, Any]]: + daily = payload.get("multi_model_daily") + if isinstance(daily, dict): + daily_entry = daily.get(local_date) + if isinstance(daily_entry, dict): + raw_models = daily_entry.get("models") + if isinstance(raw_models, dict): + forecasts = { + str(model): value + for model, value in raw_models.items() + if _safe_float(value) is not None + } + if forecasts: + return { + "source": "cached_panel_multi_model_daily", + "provider": "panel-cache", + "forecasts": forecasts, + "daily_forecasts": {local_date: forecasts}, + "hourly_times": [], + "hourly_forecasts": {}, + "model_metadata": {}, + "model_keys": list(forecasts.keys()), + "dates": [local_date], + "unit": "fahrenheit" if use_fahrenheit else "celsius", + "scan_terminal_panel_cache": True, + } + + multi_model = payload.get("multi_model") + if isinstance(multi_model, dict) and multi_model_forecasts_for_local_date( + multi_model, + local_date, + ): + return dict(multi_model) + return None + + def _model_spread_sigma(forecasts: Dict[str, float], temp_symbol: str) -> float: values = [ value @@ -413,12 +454,26 @@ def _load_scan_panel_payload( stale_reason = _panel_cache_stale_reason(city, cached_entry, cached_payload) if not force_refresh and stale_reason is None: return cached_payload + effective_multi_model_override = multi_model_override + if not isinstance(effective_multi_model_override, dict): + city_meta = CITIES.get(city) or {} + tz_offset = cached_payload.get("utc_offset_seconds") + if tz_offset is None: + tz_offset = city_meta.get("tz") + local_date = _city_local_date(city, _safe_int(tz_offset, 0)) + cached_panel_multi_model = _cached_panel_multi_model_for_local_date( + cached_payload, + local_date, + use_fahrenheit=bool(city_meta.get("f")), + ) + if cached_panel_multi_model: + effective_multi_model_override = cached_panel_multi_model _enqueue_scan_terminal_refresh(city, reason=stale_reason or "scan_terminal_force_forecast_refresh") refresh_already_queued = True refreshed_payload = _fetch_today_forecast_panel_payload( city, cached_payload, - multi_model_override=multi_model_override, + multi_model_override=effective_multi_model_override, allow_direct_fetch=allow_direct_fetch, ) if refreshed_payload: diff --git a/web/scan_terminal_service.py b/web/scan_terminal_service.py index 3f9bdee2..ee1769b4 100644 --- a/web/scan_terminal_service.py +++ b/web/scan_terminal_service.py @@ -335,6 +335,21 @@ def _build_scan_terminal_payload_uncached( ) if stale_payload is not None: return stale_payload + success_payload = cached_entry.get("success_payload") + if ( + isinstance(success_payload, dict) + and _model_bearing_rows_count(success_payload.get("rows")) > 0 + and _model_bearing_rows_count(ranked_rows) == 0 + ): + error_message = "scan terminal refresh returned rows without model forecasts" + set_scan_terminal_failure_state(filters, error_message=error_message) + failed_entry = get_scan_terminal_cache_entry(filters) or {} + return build_stale_scan_terminal_payload( + filters=filters, + success_payload=success_payload, + error_message=error_message, + failed_at=failed_entry.get("last_failed_at"), + ) payload = { "generated_at": datetime.utcnow().isoformat() + "Z",