gavindiaz/PolyWeather
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Batch scan terminal model forecasts

Browse Source
This commit is contained in:
2569718930@qq.com
2026-06-30 12:40:40 +08:00
parent a5f6bca527
commit 15de44767e
14 changed files with 668 additions and 134 deletions
Download Patch File Download Diff File Expand all files Collapse all files
web/routers/scan.py
+3 -12
View File
@@ -5,7 +5,7 @@ from __future__ import annotations
from fastapi import APIRouter, Request
from fastapi.responses import JSONResponse
from web.services.cache_headers import NO_STORE_CACHE_CONTROL, public_edge_cache_control
from web.services.cache_headers import NO_STORE_CACHE_CONTROL
from web.services.scan_api import (
get_scan_terminal_overview_payload,
get_scan_terminal_payload,
@@ -14,9 +14,6 @@ from web.services.request_timing import attach_server_timing_header
router = APIRouter(tags=["scan"])
SCAN_TERMINAL_CACHE_CONTROL = public_edge_cache_control(300, 900)
@router.get("/api/scan/terminal")
async def scan_terminal(
request: Request,
@@ -53,17 +50,11 @@ async def scan_terminal(
region=region or trading_region or None,
timezone_offset_seconds=timezone_offset_seconds,
)
status = str(payload.get("status") or "").strip().lower()
cache_control = (
SCAN_TERMINAL_CACHE_CONTROL
if not force_refresh and status == "ready" and payload.get("stale") is not True
else NO_STORE_CACHE_CONTROL
)
response = JSONResponse(
content=payload,
headers={
"Cache-Control": cache_control,
"Cloudflare-CDN-Cache-Control": cache_control,
"Cache-Control": NO_STORE_CACHE_CONTROL,
"Cloudflare-CDN-Cache-Control": NO_STORE_CACHE_CONTROL,
},
)
attach_server_timing_header(response, request, "scan_terminal_server_timing")
web/scan_terminal_cache.py
+5 -2
View File
@@ -43,10 +43,13 @@ def _redis_cache_ttl_sec() -> int:
def _redis_cache_prefix() -> str:
return os.getenv(
prefix = os.getenv(
"POLYWEATHER_SCAN_TERMINAL_REDIS_CACHE_PREFIX",
"polyweather:scan_terminal:v1:",
"polyweather:scan_terminal:v2:",
)
if prefix.endswith(":v1:"):
return f"{prefix[:-4]}:v2:"
return prefix
def _redis_entry_key(cache_key: str) -> str:
web/scan_terminal_city_row.py
+269 -14
View File
@@ -8,6 +8,10 @@ from typing import Any, Dict, List, Optional
from src.analysis.deb_algorithm import calculate_deb_prediction, calculate_dynamic_weights
from src.analysis.trend_engine import calculate_prob_distribution
from src.data_collection.nws_open_meteo_sources import (
OPEN_METEO_MULTI_MODEL_ORDER,
_parse_open_meteo_multi_model_daily,
)
from src.data_collection.multi_model_freshness import multi_model_forecasts_for_local_date
from src.database.db_manager import DBManager
from src.utils.refresh_policy import SCAN_ROWS_REFRESH_SEC
@@ -22,6 +26,7 @@ from web.services.city_payloads import aggregate_runway_history
SCAN_ROW_RUNWAY_HISTORY_RESOLUTION = "10m"
SCAN_ROW_MAX_RUNWAY_POINTS = 144
SCAN_TERMINAL_MULTI_MODEL_BATCH_SIZE = 20
_PANEL_CACHE_DB = DBManager()
_analyze = None # compatibility hook for tests that assert scan terminal stays cache-only.
SCAN_PANEL_CACHE_MAX_AGE_SEC = max(300, int(SCAN_ROWS_REFRESH_SEC) * 3)
@@ -101,9 +106,192 @@ def _model_spread_sigma(forecasts: Dict[str, float], temp_symbol: str) -> float:
return max(0.8 * scale, min(4.0 * scale, spread / 2.0))
def _multi_model_cache_key(
city: str,
lat: float,
lon: float,
*,
use_fahrenheit: bool,
) -> str:
version = str(getattr(_weather, "multi_model_cache_version", "v5") or "v5")
return (
f"{round(float(lat), 4)}:{round(float(lon), 4)}:{str(city or '').strip().lower()}:"
f"{'f' if use_fahrenheit else 'c'}:{version}"
)
def _read_cached_multi_model_for_today(
city: str,
*,
lat: float,
lon: float,
use_fahrenheit: bool,
local_date: str,
) -> Optional[Dict[str, Any]]:
maybe_reload = getattr(_weather, "_maybe_reload_open_meteo_disk_cache", None)
if callable(maybe_reload):
try:
maybe_reload()
except Exception:
pass
cache = getattr(_weather, "_multi_model_cache", None)
lock = getattr(_weather, "_multi_model_cache_lock", None)
if not isinstance(cache, dict) or lock is None:
return None
key = _multi_model_cache_key(city, lat, lon, use_fahrenheit=use_fahrenheit)
try:
with lock:
entry = cache.get(key)
data = entry.get("data") if isinstance(entry, dict) else None
except Exception:
return None
if isinstance(data, dict) and multi_model_forecasts_for_local_date(data, local_date):
return dict(data)
return None
def _store_multi_model_cache(
city: str,
payload: Dict[str, Any],
*,
lat: float,
lon: float,
use_fahrenheit: bool,
) -> None:
cache = getattr(_weather, "_multi_model_cache", None)
lock = getattr(_weather, "_multi_model_cache_lock", None)
if not isinstance(cache, dict) or lock is None:
return
key = _multi_model_cache_key(city, lat, lon, use_fahrenheit=use_fahrenheit)
try:
with lock:
cache[key] = {"t": time.time(), "data": dict(payload)}
except Exception:
return
def _fetch_scan_terminal_multi_model_batch(city_names: List[str]) -> Dict[str, Dict[str, Any]]:
"""Fetch daily max multi-model payloads for scan rows in batch.
The scan terminal only needs today's max per model. A batched daily request
avoids 50 per-city calls while still preserving city-local dates.
"""
grouped: Dict[bool, List[Dict[str, Any]]] = {False: [], True: []}
results: Dict[str, Dict[str, Any]] = {}
for city in city_names:
city_meta = CITIES.get(city) or {}
lat = _safe_float(city_meta.get("lat"))
lon = _safe_float(city_meta.get("lon"))
if lat is None or lon is None:
continue
use_fahrenheit = bool(city_meta.get("f"))
local_date = _city_local_date(city, _safe_int(city_meta.get("tz"), 0))
cached = _read_cached_multi_model_for_today(
city,
lat=lat,
lon=lon,
use_fahrenheit=use_fahrenheit,
local_date=local_date,
)
if cached:
results[city] = cached
continue
grouped[use_fahrenheit].append(
{
"city": city,
"lat": lat,
"lon": lon,
"local_date": local_date,
}
)
http_get = getattr(_weather, "_http_get", None)
if not callable(http_get):
return results
stored_any = False
for use_fahrenheit, unit_items in grouped.items():
if not unit_items:
continue
for start in range(0, len(unit_items), SCAN_TERMINAL_MULTI_MODEL_BATCH_SIZE):
items = unit_items[start : start + SCAN_TERMINAL_MULTI_MODEL_BATCH_SIZE]
if not items:
continue
try:
wait_slot = getattr(_weather, "_wait_open_meteo_slot", None)
if callable(wait_slot):
wait_slot("scan-terminal-multi-model-batch")
params: Dict[str, Any] = {
"latitude": ",".join(str(item["lat"]) for item in items),
"longitude": ",".join(str(item["lon"]) for item in items),
"daily": "temperature_2m_max",
"models": ",".join(OPEN_METEO_MULTI_MODEL_ORDER),
"timezone": "auto",
"forecast_days": 3,
}
if use_fahrenheit:
params["temperature_unit"] = "fahrenheit"
response = http_get(
"https://api.open-meteo.com/v1/forecast",
params=params,
timeout=max(10.0, float(getattr(_weather, "open_meteo_timeout_sec", 5.0))),
)
response.raise_for_status()
raw = response.json()
payloads = raw if isinstance(raw, list) else [raw]
for item, location_payload in zip(items, payloads):
daily = location_payload.get("daily", {}) if isinstance(location_payload, dict) else {}
if not isinstance(daily, dict):
continue
dates, daily_forecasts, model_metadata, model_keys = _parse_open_meteo_multi_model_daily(daily)
if not daily_forecasts:
continue
local_date = item["local_date"]
forecasts = daily_forecasts.get(local_date) or {}
if not forecasts:
continue
city = str(item["city"])
result = {
"source": "multi_model",
"provider": "open-meteo",
"forecasts": forecasts,
"daily_forecasts": daily_forecasts,
"hourly_times": [],
"hourly_forecasts": {},
"model_metadata": model_metadata,
"model_keys": model_keys,
"dates": dates,
"unit": "fahrenheit" if use_fahrenheit else "celsius",
"attribution": "Open-Meteo forecast model API; underlying models from ECMWF, DWD, ECCC, NOAA/NCEP, Google and JMA.",
"scan_terminal_batch": True,
}
results[city] = result
_store_multi_model_cache(
city,
result,
lat=float(item["lat"]),
lon=float(item["lon"]),
use_fahrenheit=use_fahrenheit,
)
stored_any = True
except Exception:
continue
if stored_any:
flush = getattr(_weather, "_flush_open_meteo_disk_cache", None)
if callable(flush):
try:
flush()
except Exception:
pass
return results
def _fetch_today_forecast_panel_payload(
city: str,
payload: Dict[str, Any],
*,
multi_model_override: Optional[Dict[str, Any]] = None,
allow_direct_fetch: bool = True,
) -> Optional[Dict[str, Any]]:
city_meta = CITIES.get(city) or {}
lat = _safe_float(city_meta.get("lat"))
@@ -120,15 +308,19 @@ def _fetch_today_forecast_panel_payload(
local_date = _city_local_date(city, tz_offset_int)
local_time = _city_local_time(city, tz_offset_int)
try:
multi_model = _weather.fetch_multi_model(
lat,
lon,
city=city,
use_fahrenheit=use_fahrenheit,
)
except Exception:
multi_model = None
multi_model = multi_model_override
if not isinstance(multi_model, dict):
if not allow_direct_fetch:
return None
try:
multi_model = _weather.fetch_multi_model(
lat,
lon,
city=city,
use_fahrenheit=use_fahrenheit,
)
except Exception:
multi_model = None
forecasts = multi_model_forecasts_for_local_date(multi_model, local_date)
if not forecasts:
return None
@@ -186,7 +378,34 @@ def _fetch_today_forecast_panel_payload(
}
def _load_scan_panel_payload(city: str, *, force_refresh: bool) -> Optional[Dict[str, Any]]:
def _build_forecast_only_panel_payload(
city: str,
multi_model: Dict[str, Any],
) -> Optional[Dict[str, Any]]:
city_meta = CITIES.get(city) or {}
temp_symbol = "°F" if bool(city_meta.get("f")) else "°C"
return _fetch_today_forecast_panel_payload(
city,
{
"display_name": city_meta.get("name") or city_meta.get("display_name") or city,
"current": {},
"risk": {},
"probabilities": {},
"temp_symbol": temp_symbol,
"utc_offset_seconds": _safe_int(city_meta.get("tz"), 0),
},
multi_model_override=multi_model,
allow_direct_fetch=False,
)
def _load_scan_panel_payload(
city: str,
*,
force_refresh: bool,
multi_model_override: Optional[Dict[str, Any]] = None,
allow_direct_fetch: bool = True,
) -> Optional[Dict[str, Any]]:
refresh_already_queued = False
cached_entry = _PANEL_CACHE_DB.get_city_cache("panel", city)
cached_payload = cached_entry.get("payload") if isinstance(cached_entry, dict) else None
@@ -196,7 +415,12 @@ def _load_scan_panel_payload(city: str, *, force_refresh: bool) -> Optional[Dict
return cached_payload
_enqueue_scan_terminal_refresh(city, reason=stale_reason or "scan_terminal_force_forecast_refresh")
refresh_already_queued = True
refreshed_payload = _fetch_today_forecast_panel_payload(city, cached_payload)
refreshed_payload = _fetch_today_forecast_panel_payload(
city,
cached_payload,
multi_model_override=multi_model_override,
allow_direct_fetch=allow_direct_fetch,
)
if refreshed_payload:
return refreshed_payload
@@ -212,7 +436,12 @@ def _load_scan_panel_payload(city: str, *, force_refresh: bool) -> Optional[Dict
city_meta = CITIES.get(city) or {}
payload.setdefault("display_name", city_meta.get("name") or city_meta.get("display_name") or city)
payload.setdefault("temp_symbol", canonical.get("temp_symbol") or "°C")
refreshed_payload = _fetch_today_forecast_panel_payload(city, payload)
refreshed_payload = _fetch_today_forecast_panel_payload(
city,
payload,
multi_model_override=multi_model_override,
allow_direct_fetch=allow_direct_fetch,
)
if refreshed_payload:
payload = refreshed_payload
_enqueue_scan_terminal_refresh(city, reason="scan_terminal_canonical_fallback")
@@ -353,8 +582,16 @@ def _scan_city_terminal_rows(
filters: Dict[str, Any],
*,
force_refresh: bool = False,
multi_model_override: Optional[Dict[str, Any]] = None,
allow_direct_fetch: bool = True,
) -> Dict[str, Any]:
return _scan_city_terminal_rows_quick(city, filters, force_refresh=force_refresh)
return _scan_city_terminal_rows_quick(
city,
filters,
force_refresh=force_refresh,
multi_model_override=multi_model_override,
allow_direct_fetch=allow_direct_fetch,
)
def _scan_city_terminal_rows_quick(
@@ -362,10 +599,28 @@ def _scan_city_terminal_rows_quick(
filters: Dict[str, Any],
*,
force_refresh: bool = False,
multi_model_override: Optional[Dict[str, Any]] = None,
allow_direct_fetch: bool = True,
) -> Dict[str, Any]:
"""Fast path that returns cached analysis rows only — returns a single row per city
with cached analysis data (Obs, DEB, probabilities) but no market prices."""
data = _load_scan_panel_payload(city, force_refresh=force_refresh)
if isinstance(multi_model_override, dict) and multi_model_override:
data = _build_forecast_only_panel_payload(city, multi_model_override)
if data:
row = _build_quick_row(city=city, data=data)
return {
"city": city,
"rows": [row] if row else [],
"candidate_total": 1,
"primary_scores": [float(row.get("final_score") or 0)] if row else [],
}
data = _load_scan_panel_payload(
city,
force_refresh=force_refresh,
multi_model_override=multi_model_override,
allow_direct_fetch=allow_direct_fetch,
)
if not data:
return {
"city": city,
web/scan_terminal_service.py
+93 -46
View File
@@ -29,7 +29,10 @@ from web.scan_terminal_cache import (
set_cached_scan_terminal_payload,
set_scan_terminal_failure_state,
)
from web.scan_terminal_city_row import _scan_city_terminal_rows
from web.scan_terminal_city_row import (
_fetch_scan_terminal_multi_model_batch,
_scan_city_terminal_rows,
)
from web.scan_terminal_filters import (
normalize_scan_terminal_filters as _normalize_scan_terminal_filters,
)
@@ -67,6 +70,19 @@ def _rows_count(payload: Dict[str, Any]) -> int:
return len(rows) if isinstance(rows, list) else 0
def _model_bearing_rows_count(rows: Any) -> int:
if not isinstance(rows, list):
return 0
count = 0
for row in rows:
if not isinstance(row, dict):
continue
sources = row.get("model_cluster_sources")
if isinstance(sources, dict) and sources:
count += 1
return count
def _success_payload_within_stale_window(cached_entry: Dict[str, Any]) -> bool:
timestamp = cached_entry.get("success_t") or cached_entry.get("t")
try:
@@ -116,6 +132,10 @@ def _build_stale_payload_for_timeout_if_better_cached(
success_payload = cached_entry.get("success_payload")
if not isinstance(success_payload, dict) or not success_payload.get("rows"):
return None
if _model_bearing_rows_count(ranked_rows) > _model_bearing_rows_count(
success_payload.get("rows")
):
return None
if _rows_count(success_payload) < len(ranked_rows):
return None
@@ -184,59 +204,86 @@ def _build_scan_terminal_payload_uncached(
return _tz_region(tz)["key"] == region_filter
city_names = [c for c in city_names if _city_in_region(c)]
max_workers = max(1, min(SCAN_TERMINAL_MAX_WORKERS, len(city_names)))
city_results: List[Dict[str, Any]] = []
failed_cities: List[str] = []
failed_reasons: List[str] = []
multi_model_overrides = _fetch_scan_terminal_multi_model_batch(city_names)
scan_city_names = (
[city_name for city_name in city_names if city_name in multi_model_overrides]
if multi_model_overrides
else city_names
)
max_workers = max(1, min(SCAN_TERMINAL_MAX_WORKERS, len(scan_city_names)))
timed_out = False
timeout_message: Optional[str] = None
executor = ThreadPoolExecutor(max_workers=max_workers)
future_map = {
executor.submit(
_scan_city_terminal_rows,
city_name,
filters,
force_refresh=force_refresh,
): city_name
for city_name in city_names
}
try:
try:
completed = as_completed(
future_map,
timeout=build_timeout_sec,
)
for future in completed:
city_name = future_map[future]
try:
city_results.append(future.result())
except Exception as exc:
failed_cities.append(city_name)
failed_reasons.append(str(exc))
logger.warning(
"scan terminal city failed city={}: {}", city_name, exc
if multi_model_overrides:
for city_name in scan_city_names:
try:
city_results.append(
_scan_city_terminal_rows(
city_name,
filters,
force_refresh=force_refresh,
multi_model_override=multi_model_overrides.get(city_name),
allow_direct_fetch=False,
)
except FutureTimeoutError:
timed_out = True
timeout_message = (
f"scan terminal build timed out after {build_timeout_sec:g}s"
)
failed_reasons.append(timeout_message)
for future, city_name in future_map.items():
if not future.done():
future.cancel()
failed_cities.append(city_name)
logger.warning(
"{}; completed={}/{}",
timeout_message,
len(city_results),
len(city_names),
)
finally:
executor.shutdown(wait=False)
)
except Exception as exc:
failed_cities.append(city_name)
failed_reasons.append(str(exc))
logger.warning(
"scan terminal city failed city={}: {}", city_name, exc
)
else:
executor = ThreadPoolExecutor(max_workers=max_workers)
future_map = {
executor.submit(
_scan_city_terminal_rows,
city_name,
filters,
force_refresh=force_refresh,
multi_model_override=multi_model_overrides.get(city_name),
allow_direct_fetch=False,
): city_name
for city_name in scan_city_names
}
try:
try:
completed = as_completed(
future_map,
timeout=build_timeout_sec,
)
for future in completed:
city_name = future_map[future]
try:
city_results.append(future.result())
except Exception as exc:
failed_cities.append(city_name)
failed_reasons.append(str(exc))
logger.warning(
"scan terminal city failed city={}: {}", city_name, exc
)
except FutureTimeoutError:
timed_out = True
timeout_message = (
f"scan terminal build timed out after {build_timeout_sec:g}s"
)
failed_reasons.append(timeout_message)
for future, city_name in future_map.items():
if not future.done():
future.cancel()
failed_cities.append(city_name)
logger.warning(
"{}; completed={}/{}",
timeout_message,
len(city_results),
len(scan_city_names),
)
finally:
executor.shutdown(wait=False)
if city_names and len(failed_cities) >= len(city_names):
if scan_city_names and len(failed_cities) >= len(scan_city_names):
error_message = (
failed_reasons[0] if failed_reasons else "all city market scans failed"
)