gavindiaz/PolyWeather
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add dashboard prewarm worker and cache visibility

Browse Source
This commit is contained in:
2569718930@qq.com
2026-04-08 06:53:49 +08:00
parent 0bb3b573e1
commit c3da29c09c
21 changed files with 1120 additions and 149 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/bot/runtime_coordinator.py
+27
View File
@@ -84,6 +84,7 @@ class StartupCoordinator:
def start_all(self) -> RuntimeStatus:
loops = [
self._start_trade_alert_loop(),
self._start_dashboard_prewarm_loop(),
self._start_polygon_wallet_loop(),
self._start_polymarket_wallet_activity_loop(),
self._start_weekly_reward_loop(),
@@ -179,6 +180,32 @@ class StartupCoordinator:
),
)
def _start_dashboard_prewarm_loop(self) -> LoopStatus:
enabled = _env_bool("POLYWEATHER_DASHBOARD_PREWARM_ENABLED", False)
interval = max(30, _env_int("POLYWEATHER_PREWARM_INTERVAL_SEC", 300))
jitter = max(0, _env_int("POLYWEATHER_PREWARM_JITTER_SEC", 20))
cities_count = _parse_csv_count(os.getenv("POLYWEATHER_PREWARM_CITIES")) or 14
details = {
"interval_sec": interval,
"jitter_sec": jitter,
"cities_count": cities_count,
"include_detail": _env_bool("POLYWEATHER_PREWARM_INCLUDE_DETAIL", True),
"include_market": _env_bool("POLYWEATHER_PREWARM_INCLUDE_MARKET", True),
"force_refresh": _env_bool("POLYWEATHER_PREWARM_FORCE_REFRESH", False),
"base_url": str(os.getenv("POLYWEATHER_BACKEND_URL") or "http://127.0.0.1:8000").strip(),
}
validation_error = None
if not str(os.getenv("POLYWEATHER_BACKEND_ENTITLEMENT_TOKEN") or "").strip():
validation_error = "missing_POLYWEATHER_BACKEND_ENTITLEMENT_TOKEN"
return self._start_with_validation(
key="dashboard_prewarm",
label="站点面板预热",
configured_enabled=enabled,
details=details,
validation_error=validation_error,
starter=lambda: import_module("src.utils.prewarm_dashboard").start_prewarm_worker_thread(),
)
def _start_polygon_wallet_loop(self) -> LoopStatus:
enabled = _env_bool("POLYGON_WALLET_WATCH_ENABLED", False)
chat_ids = get_telegram_chat_ids_from_env()
src/data_collection/metar_sources.py
+3 -3
View File
@@ -5,7 +5,7 @@ import time
from datetime import datetime, timedelta, timezone
from typing import Dict, List, Optional
import requests
import httpx
from loguru import logger
from src.utils.metrics import record_source_call
@@ -209,7 +209,7 @@ class MetarSourceMixin:
record_source_call("metar", "current", "success", (time.perf_counter() - started) * 1000.0)
return result
except requests.exceptions.RequestException as exc:
except httpx.HTTPError as exc:
logger.error(f"METAR 请求失败 ({icao}): {exc}")
with self._metar_cache_lock:
stale = self._metar_cache.get(cache_key)
@@ -269,7 +269,7 @@ class MetarSourceMixin:
self._taf_cache[cache_key] = {"d": result, "t": now_ts}
record_source_call("taf", "current", "success", (time.perf_counter() - started) * 1000.0)
return result
except requests.exceptions.RequestException as exc:
except httpx.HTTPError as exc:
logger.error(f"TAF 请求失败 ({icao}): {exc}")
with self._taf_cache_lock:
stale = self._taf_cache.get(cache_key)
src/data_collection/mgm_sources.py
+9 -5
View File
@@ -32,7 +32,7 @@ class MgmSourceMixin:
# 1. 实时数据 (添加时间戳防止 CDN 缓存)
import time
obs_resp = self.session.get(
obs_resp = self._http_get(
f"{base_url}/sondurumlar?istno={istno}&_={int(time.time() * 1000)}",
headers=headers,
timeout=self.timeout,
@@ -95,7 +95,7 @@ class MgmSourceMixin:
]
for forecast_url in forecast_urls:
try:
daily_resp = self.session.get(
daily_resp = self._http_get(
forecast_url, headers=headers, timeout=self.timeout
)
if daily_resp.status_code == 200:
@@ -132,7 +132,7 @@ class MgmSourceMixin:
# 3. 小时预报
try:
hourly_resp = self.session.get(
hourly_resp = self._http_get(
f"{base_url}/tahminler/saatlik?istno={istno}",
headers=headers,
timeout=self.timeout
@@ -246,7 +246,11 @@ class MgmSourceMixin:
try:
# 1. 加载测站元数据 (缓存到实例中)，用于过滤属于该省份的站点
if not getattr(self, "mgm_stations_meta", None):
meta_resp = self.session.get(f"{base_url}/istasyonlar", headers=headers, timeout=self.timeout)
meta_resp = self._http_get(
f"{base_url}/istasyonlar",
headers=headers,
timeout=self.timeout,
)
if meta_resp.status_code == 200:
meta_json = meta_resp.json()
if isinstance(meta_json, list):
@@ -293,7 +297,7 @@ class MgmSourceMixin:
try:
# sondurumlar?istno={ist_no} 是目前最稳的获取多站数据的办法
url = f"{base_url}/sondurumlar?istno={ist_no}&_={int(time.time() * 1000)}"
resp = self.session.get(url, headers=headers, timeout=5)
resp = self._http_get(url, headers=headers, timeout=5)
if resp.status_code == 200:
obs_list = resp.json()
if obs_list:
src/data_collection/nmc_sources.py
+16 -4
View File
@@ -45,6 +45,20 @@ NMC_CITY_REFERENCES: Dict[str, Dict[str, Any]] = {
class NmcSourceMixin:
def _nmc_http_get(self, url: str):
getter = getattr(self, "_http_get", None)
if callable(getter):
return getter(url)
return self.session.get(url, timeout=self.timeout)
def _nmc_http_get_json(self, url: str):
getter = getattr(self, "_http_get_json", None)
if callable(getter):
return getter(url)
response = self.session.get(url, timeout=self.timeout)
response.raise_for_status()
return response.json()
@staticmethod
def _nmc_optional_text(value: Any) -> Optional[str]:
text = str(value or "").strip()
@@ -73,7 +87,7 @@ class NmcSourceMixin:
return None
try:
resp = self.session.get(page_url, timeout=self.timeout)
resp = self._nmc_http_get(page_url)
resp.raise_for_status()
match = re.search(
r"renderWeatherRealPanel\('([^']+)',\s*'([^']+)'\)",
@@ -116,9 +130,7 @@ class NmcSourceMixin:
try:
url = f"https://www.nmc.cn/rest/real/{station_code}"
resp = self.session.get(url, timeout=self.timeout)
resp.raise_for_status()
payload = resp.json()
payload = self._nmc_http_get_json(url)
if not isinstance(payload, dict) or not isinstance(payload.get("weather"), dict):
record_source_call("nmc", "current", "empty", (time.perf_counter() - started) * 1000.0)
return None
src/data_collection/nws_open_meteo_sources.py
+7 -7
View File
@@ -21,7 +21,7 @@ class NwsOpenMeteoSourceMixin:
points_url = f"https://api.weather.gov/points/{lat},{lon}"
headers = {"User-Agent": "PolyWeather/1.0 (weather-bot)"}
points_resp = self.session.get(
points_resp = self._http_get(
points_url, headers=headers, timeout=self.timeout
)
points_resp.raise_for_status()
@@ -35,7 +35,7 @@ class NwsOpenMeteoSourceMixin:
return None
# 2. 获取预报
forecast_resp = self.session.get(
forecast_resp = self._http_get(
forecast_url, headers=headers, timeout=self.timeout
)
forecast_resp.raise_for_status()
@@ -48,7 +48,7 @@ class NwsOpenMeteoSourceMixin:
hourly_periods = []
if hourly_url:
hourly_resp = self.session.get(
hourly_resp = self._http_get(
hourly_url, headers=headers, timeout=self.timeout
)
hourly_resp.raise_for_status()
@@ -57,7 +57,7 @@ class NwsOpenMeteoSourceMixin:
active_alerts = []
try:
alerts_resp = self.session.get(
alerts_resp = self._http_get(
"https://api.weather.gov/alerts/active",
params={"point": f"{lat},{lon}"},
headers=headers,
@@ -205,7 +205,7 @@ class NwsOpenMeteoSourceMixin:
params["temperature_unit"] = "celsius"
self._wait_open_meteo_slot("forecast")
response = self.session.get(
response = self._http_get(
url,
params=params,
timeout=self.timeout,
@@ -366,7 +366,7 @@ class NwsOpenMeteoSourceMixin:
params["temperature_unit"] = "celsius"
self._wait_open_meteo_slot("ensemble")
response = self.session.get(
response = self._http_get(
url,
params=params,
timeout=self.timeout,
@@ -523,7 +523,7 @@ class NwsOpenMeteoSourceMixin:
params["temperature_unit"] = "fahrenheit"
self._wait_open_meteo_slot("multi-model")
response = self.session.get(
response = self._http_get(
url,
params=params,
timeout=self.timeout,
src/data_collection/polymarket_readonly.py
+5 -2
View File
@@ -19,7 +19,7 @@ import unicodedata
from datetime import datetime, timezone
from typing import Any, Dict, List, Optional, Tuple
import requests
import httpx
from loguru import logger
from src.data_collection.city_registry import ALIASES, CITY_REGISTRY
@@ -383,7 +383,10 @@ class PolymarketReadOnlyLayer:
)
self.edge_threshold = _safe_float(os.getenv("POLYMARKET_SIGNAL_EDGE_PCT")) or 2.0
self._session = requests.Session()
self._session = httpx.Client(
timeout=self.http_timeout,
follow_redirects=True,
)
self._markets_cache: Dict[str, Dict[str, Any]] = {}
self._active_markets_cache: Dict[str, Any] = {"data": [], "t": 0.0}
self._broad_markets_cache: Dict[str, Any] = {"data": [], "t": 0.0}
src/data_collection/settlement_sources.py
+10 -10
View File
@@ -226,13 +226,13 @@ class SettlementSourceMixin:
try:
base = "https://data.weather.gov.hk/weatherAPI/hko_data/regional-weather"
temp_csv = self.session.get(f"{base}/latest_1min_temperature.csv", timeout=self.timeout)
temp_csv = self._http_get(f"{base}/latest_1min_temperature.csv", timeout=self.timeout)
temp_csv.raise_for_status()
maxmin_csv = self.session.get(f"{base}/latest_since_midnight_maxmin.csv", timeout=self.timeout)
maxmin_csv = self._http_get(f"{base}/latest_since_midnight_maxmin.csv", timeout=self.timeout)
maxmin_csv.raise_for_status()
humidity_csv = self.session.get(f"{base}/latest_1min_humidity.csv", timeout=self.timeout)
humidity_csv = self._http_get(f"{base}/latest_1min_humidity.csv", timeout=self.timeout)
humidity_csv.raise_for_status()
wind_csv = self.session.get(f"{base}/latest_10min_wind.csv", timeout=self.timeout)
wind_csv = self._http_get(f"{base}/latest_10min_wind.csv", timeout=self.timeout)
wind_csv.raise_for_status()
temp_rows = self._csv_rows(temp_csv.text)
@@ -302,7 +302,7 @@ class SettlementSourceMixin:
try:
url = "https://opendata.cwa.gov.tw/api/v1/rest/datastore/O-A0003-001"
response = self.session.get(
response = self._http_get(
url,
params={"Authorization": self.cwa_open_data_auth, "format": "JSON", "StationId": "466920"},
timeout=self.timeout,
@@ -361,7 +361,7 @@ class SettlementSourceMixin:
def fetch_hko_forecast(self) -> Optional[float]:
try:
url = "https://data.weather.gov.hk/weatherAPI/opendata/weather.php?dataType=fnd&lang=tc"
res = self.session.get(url, timeout=self.timeout).json()
res = self._http_get_json(url, timeout=self.timeout)
return float(res["weatherForecast"][0]["forecastMaxtemp"]["value"])
except Exception as exc:
logger.warning(f"HKO Forecast request failed: {exc}")
@@ -372,11 +372,11 @@ class SettlementSourceMixin:
if not self.cwa_open_data_auth:
return None
url = "https://opendata.cwa.gov.tw/api/v1/rest/datastore/F-D0047-061"
res = self.session.get(
res = self._http_get_json(
url,
params={"Authorization": self.cwa_open_data_auth, "format": "JSON", "elementName": "MaxT"},
timeout=self.timeout,
).json()
)
locs = res.get("records", {}).get("Locations", [])[0].get("Location", [])
if not locs:
return None
@@ -402,7 +402,7 @@ class SettlementSourceMixin:
return cached
try:
response = self.session.get(
response = self._http_get(
"https://api.synopticdata.com/v2/stations/timeseries",
params={
"STID": normalized_station_code,
@@ -519,7 +519,7 @@ class SettlementSourceMixin:
query = {"token": self.IMGW_METEO_API_TOKEN}
if isinstance(params, dict):
query.update(params)
response = self.session.get(
response = self._http_get(
f"{self.IMGW_METEO_API_BASE}/{path}",
params=query,
timeout=self.timeout,
src/data_collection/weather_sources.py
+82 -9
View File
@@ -1,7 +1,8 @@
import os
import requests
import httpx
import re
import threading
import time
from typing import Optional, Dict, List
from datetime import datetime, timedelta
from loguru import logger
@@ -110,7 +111,17 @@ class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSour
self.config = config
self.timeout = 30 # 增加超时以支持高延迟 VPS
self.session = requests.Session()
self.http_retry_count = max(
0, int(os.getenv("POLYWEATHER_HTTP_RETRY_COUNT", "1"))
)
self.http_retry_backoff_sec = max(
0.0, float(os.getenv("POLYWEATHER_HTTP_RETRY_BACKOFF_SEC", "0.35"))
)
self.session = httpx.Client(
timeout=self.timeout,
follow_redirects=True,
limits=httpx.Limits(max_connections=50, max_keepalive_connections=20),
)
self.open_meteo_cache_ttl_sec = int(
os.getenv("OPEN_METEO_CACHE_TTL_SEC", "900")
)
@@ -186,11 +197,73 @@ class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSour
if proxy:
if not proxy.startswith("http"):
proxy = f"http://{proxy}"
self.session.proxies = {"http": proxy, "https": proxy}
self.session = httpx.Client(
timeout=self.timeout,
follow_redirects=True,
proxy=proxy,
limits=httpx.Limits(max_connections=50, max_keepalive_connections=20),
)
logger.info(f"正在使用天气数据代理: {proxy}")
logger.info("天气数据采集器初始化完成。")
@staticmethod
def _is_retryable_status(status_code: int) -> bool:
return status_code in {408, 500, 502, 503, 504}
def _http_get(self, url: str, **kwargs) -> httpx.Response:
if "timeout" not in kwargs:
kwargs["timeout"] = self.timeout
last_exc: Optional[Exception] = None
last_response: Optional[httpx.Response] = None
attempts = self.http_retry_count + 1
for attempt in range(attempts):
try:
response = self.session.get(url, **kwargs)
last_response = response
if (
attempt < attempts - 1
and self._is_retryable_status(response.status_code)
):
wait_for = self.http_retry_backoff_sec * (attempt + 1)
logger.debug(
"HTTP GET retrying url={} status={} attempt={}/{} wait={}s",
url,
response.status_code,
attempt + 1,
attempts,
round(wait_for, 2),
)
if wait_for > 0:
time.sleep(wait_for)
continue
return response
except (httpx.TimeoutException, httpx.NetworkError) as exc:
last_exc = exc
if attempt >= attempts - 1:
break
wait_for = self.http_retry_backoff_sec * (attempt + 1)
logger.debug(
"HTTP GET retrying url={} error={} attempt={}/{} wait={}s",
url,
type(exc).__name__,
attempt + 1,
attempts,
round(wait_for, 2),
)
if wait_for > 0:
time.sleep(wait_for)
if last_exc is not None:
raise last_exc
if last_response is not None:
return last_response
raise RuntimeError(f"HTTP GET failed without response: {url}")
def _http_get_json(self, url: str, **kwargs):
response = self._http_get(url, **kwargs)
response.raise_for_status()
return response.json()
def fetch_from_openweather(self, city: str, country: str = None) -> Optional[Dict]:
"""
Fetch current weather and forecast from OpenWeatherMap
@@ -210,7 +283,7 @@ class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSour
try:
# Current weather
current_url = "https://api.openweathermap.org/data/2.5/weather"
current_response = self.session.get(
current_response = self._http_get(
current_url,
params={"q": query, "appid": self.openweather_key, "units": "metric"},
timeout=self.timeout,
@@ -220,7 +293,7 @@ class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSour
# 5-day forecast
forecast_url = "https://api.openweathermap.org/data/2.5/forecast"
forecast_response = self.session.get(
forecast_response = self._http_get(
forecast_url,
params={"q": query, "appid": self.openweather_key, "units": "metric"},
timeout=self.timeout,
@@ -245,7 +318,7 @@ class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSour
"forecast": self._parse_openweather_forecast(forecast_data),
}
except requests.exceptions.RequestException as e:
except httpx.HTTPError as e:
logger.error(f"OpenWeatherMap request failed: {e}")
return None
@@ -290,7 +363,7 @@ class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSour
try:
url = f"https://weather.visualcrossing.com/VisualCrossingWebServices/rest/services/timeline/{city}/{start_date}/{end_date}"
response = self.session.get(
response = self._http_get(
url,
params={
"unitGroup": "metric",
@@ -322,7 +395,7 @@ class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSour
],
}
except requests.exceptions.RequestException as e:
except httpx.HTTPError as e:
logger.error(f"Visual Crossing request failed: {e}")
return None
@@ -399,7 +472,7 @@ class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSour
try:
url = "https://geocoding-api.open-meteo.com/v1/search"
response = self.session.get(
response = self._http_get(
url,
params={"name": city, "count": 1, "language": "en", "format": "json"},
timeout=15, # 增加超时时间到 15s
src/utils/prewarm_dashboard.py
+305
View File
@@ -0,0 +1,305 @@
from __future__ import annotations
import json
import os
import random
import threading
import time
from datetime import datetime
from typing import Any, Dict, Optional
import httpx
from loguru import logger
DEFAULT_CITIES = [
"ankara",
"istanbul",
"shanghai",
"beijing",
"shenzhen",
"wuhan",
"chengdu",
"chongqing",
"hong kong",
"taipei",
"london",
"paris",
"new york",
"los angeles",
]
_RUNTIME_LOCK = threading.Lock()
_WORKER_THREAD: Optional[threading.Thread] = None
_RUNTIME_STATE: Dict[str, Any] = {
"cycle_count": 0,
"success_count": 0,
"failure_count": 0,
"last_started_at": None,
"last_finished_at": None,
"last_duration_sec": None,
"last_success": None,
"last_http_status": None,
"last_error": None,
"last_requested_cities": [],
"last_requested_city_count": 0,
"last_include_detail": False,
"last_include_market": False,
"last_force_refresh": False,
"last_warmed_count": 0,
"last_summary_ok": 0,
"last_detail_ok": 0,
"last_market_ok": 0,
"last_failed_count": 0,
}
def _truthy_env(value: str, default: bool = False) -> bool:
raw = str(os.getenv(value) or "").strip().lower()
if not raw:
return default
return raw in {"1", "true", "yes", "on"}
def _parse_cities(value: str) -> list[str]:
items = [item.strip() for item in str(value or "").split(",")]
return [item for item in items if item]
def _update_runtime_state(**kwargs: Any) -> None:
with _RUNTIME_LOCK:
_RUNTIME_STATE.update(kwargs)
def _record_prewarm_result(
*,
ok: bool,
duration_sec: float,
http_status: Optional[int],
error: Optional[str],
warmed_count: int,
summary_ok: int,
detail_ok: int,
market_ok: int,
failed_count: int,
) -> None:
with _RUNTIME_LOCK:
_RUNTIME_STATE["cycle_count"] = int(_RUNTIME_STATE.get("cycle_count") or 0) + 1
if ok:
_RUNTIME_STATE["success_count"] = int(_RUNTIME_STATE.get("success_count") or 0) + 1
else:
_RUNTIME_STATE["failure_count"] = int(_RUNTIME_STATE.get("failure_count") or 0) + 1
_RUNTIME_STATE["last_finished_at"] = datetime.now().isoformat(timespec="seconds")
_RUNTIME_STATE["last_duration_sec"] = round(float(duration_sec), 2)
_RUNTIME_STATE["last_success"] = bool(ok)
_RUNTIME_STATE["last_http_status"] = http_status
_RUNTIME_STATE["last_error"] = error
_RUNTIME_STATE["last_warmed_count"] = int(warmed_count or 0)
_RUNTIME_STATE["last_summary_ok"] = int(summary_ok or 0)
_RUNTIME_STATE["last_detail_ok"] = int(detail_ok or 0)
_RUNTIME_STATE["last_market_ok"] = int(market_ok or 0)
_RUNTIME_STATE["last_failed_count"] = int(failed_count or 0)
def get_prewarm_runtime_summary() -> Dict[str, Any]:
configured_cities = _parse_cities(str(os.getenv("POLYWEATHER_PREWARM_CITIES") or ",".join(DEFAULT_CITIES)))
with _RUNTIME_LOCK:
runtime = dict(_RUNTIME_STATE)
return {
"enabled": _truthy_env("POLYWEATHER_DASHBOARD_PREWARM_ENABLED", False),
"base_url": str(os.getenv("POLYWEATHER_BACKEND_URL") or "http://127.0.0.1:8000").strip(),
"configured_cities": configured_cities,
"configured_city_count": len(configured_cities),
"interval_sec": max(30, int(os.getenv("POLYWEATHER_PREWARM_INTERVAL_SEC", "300"))),
"jitter_sec": max(0, int(os.getenv("POLYWEATHER_PREWARM_JITTER_SEC", "20"))),
"include_detail": _truthy_env("POLYWEATHER_PREWARM_INCLUDE_DETAIL", True),
"include_market": _truthy_env("POLYWEATHER_PREWARM_INCLUDE_MARKET", True),
"force_refresh": _truthy_env("POLYWEATHER_PREWARM_FORCE_REFRESH", False),
"thread_alive": bool(_WORKER_THREAD and _WORKER_THREAD.is_alive()),
"runtime": runtime,
}
def run_prewarm(
*,
base_url: str,
cities: str,
force_refresh: bool,
include_detail: bool,
include_market: bool,
) -> int:
token = str(os.getenv("POLYWEATHER_BACKEND_ENTITLEMENT_TOKEN") or "").strip()
requested_cities = _parse_cities(cities)
started = time.perf_counter()
_update_runtime_state(
last_started_at=datetime.now().isoformat(timespec="seconds"),
last_requested_cities=requested_cities,
last_requested_city_count=len(requested_cities),
last_include_detail=bool(include_detail),
last_include_market=bool(include_market),
last_force_refresh=bool(force_refresh),
last_error=None,
last_http_status=None,
)
if not token:
_record_prewarm_result(
ok=False,
duration_sec=time.perf_counter() - started,
http_status=None,
error="missing_backend_token",
warmed_count=0,
summary_ok=0,
detail_ok=0,
market_ok=0,
failed_count=0,
)
print(
json.dumps(
{
"ok": False,
"reason": "missing_backend_token",
"detail": "POLYWEATHER_BACKEND_ENTITLEMENT_TOKEN is required",
},
ensure_ascii=False,
)
)
return 1
try:
with httpx.Client(timeout=180, follow_redirects=True) as client:
response = client.post(
f"{base_url.rstrip('/')}/api/system/prewarm",
params={
"cities": cities,
"force_refresh": str(bool(force_refresh)).lower(),
"include_detail": str(bool(include_detail)).lower(),
"include_market": str(bool(include_market)).lower(),
},
headers={
"Accept": "application/json",
"x-polyweather-entitlement": token,
},
)
try:
payload = response.json()
except Exception:
payload = {"ok": False, "raw": (response.text or "")[:500]}
warmed_count = int((payload or {}).get("count") or 0) if isinstance(payload, dict) else 0
summary_ok = int((payload or {}).get("summary_ok") or 0) if isinstance(payload, dict) else 0
detail_ok = int((payload or {}).get("detail_ok") or 0) if isinstance(payload, dict) else 0
market_ok = int((payload or {}).get("market_ok") or 0) if isinstance(payload, dict) else 0
failed_count = int((payload or {}).get("failed_count") or 0) if isinstance(payload, dict) else 0
ok = bool(response.is_success and (not isinstance(payload, dict) or payload.get("ok", True)))
_record_prewarm_result(
ok=ok,
duration_sec=time.perf_counter() - started,
http_status=response.status_code,
error=None if ok else str((payload or {}).get("detail") or (payload or {}).get("reason") or response.text[:200]),
warmed_count=warmed_count,
summary_ok=summary_ok,
detail_ok=detail_ok,
market_ok=market_ok,
failed_count=failed_count,
)
print(json.dumps(payload, ensure_ascii=False, indent=2))
return 0 if response.is_success else 1
except Exception as exc:
_record_prewarm_result(
ok=False,
duration_sec=time.perf_counter() - started,
http_status=None,
error=str(exc),
warmed_count=0,
summary_ok=0,
detail_ok=0,
market_ok=0,
failed_count=0,
)
print(json.dumps({"ok": False, "reason": "request_failed", "detail": str(exc)}, ensure_ascii=False, indent=2))
return 1
def run_worker_loop(
*,
base_url: str,
cities: str,
interval_sec: int,
jitter_sec: int,
force_refresh: bool,
include_detail: bool,
include_market: bool,
once: bool = False,
) -> int:
interval_sec = max(30, int(interval_sec))
jitter_sec = max(0, int(jitter_sec))
logger.info(
"dashboard prewarm worker started base_url={} cities={} interval_sec={} jitter_sec={} include_detail={} include_market={} force_refresh={} once={}",
base_url,
cities,
interval_sec,
jitter_sec,
bool(include_detail),
bool(include_market),
bool(force_refresh),
bool(once),
)
while True:
started = time.perf_counter()
exit_code = run_prewarm(
base_url=base_url,
cities=cities,
force_refresh=bool(force_refresh),
include_detail=bool(include_detail),
include_market=bool(include_market),
)
elapsed = time.perf_counter() - started
logger.info(
"dashboard prewarm worker cycle exit_code={} elapsed_sec={} finished_at={}",
exit_code,
round(elapsed, 2),
datetime.now().isoformat(timespec="seconds"),
)
if once:
return exit_code
sleep_sec = max(5.0, interval_sec - elapsed)
if jitter_sec > 0:
sleep_sec += random.randint(0, jitter_sec)
logger.info("dashboard prewarm worker sleeping sleep_sec={}", round(sleep_sec, 2))
time.sleep(sleep_sec)
def start_prewarm_worker_thread() -> Optional[threading.Thread]:
enabled = str(os.getenv("POLYWEATHER_DASHBOARD_PREWARM_ENABLED") or "").strip().lower()
if enabled not in {"1", "true", "yes", "on"}:
return None
base_url = str(os.getenv("POLYWEATHER_BACKEND_URL") or "http://127.0.0.1:8000").strip()
cities = str(os.getenv("POLYWEATHER_PREWARM_CITIES") or ",".join(DEFAULT_CITIES)).strip()
interval_sec = int(os.getenv("POLYWEATHER_PREWARM_INTERVAL_SEC", "300"))
jitter_sec = int(os.getenv("POLYWEATHER_PREWARM_JITTER_SEC", "20"))
force_refresh = str(os.getenv("POLYWEATHER_PREWARM_FORCE_REFRESH") or "").strip().lower() in {"1", "true", "yes", "on"}
include_detail = str(os.getenv("POLYWEATHER_PREWARM_INCLUDE_DETAIL", "true")).strip().lower() in {"1", "true", "yes", "on"}
include_market = str(os.getenv("POLYWEATHER_PREWARM_INCLUDE_MARKET", "true")).strip().lower() in {"1", "true", "yes", "on"}
thread = threading.Thread(
target=run_worker_loop,
kwargs={
"base_url": base_url,
"cities": cities,
"interval_sec": interval_sec,
"jitter_sec": jitter_sec,
"force_refresh": force_refresh,
"include_detail": include_detail,
"include_market": include_market,
"once": False,
},
name="dashboard-prewarm-worker",
daemon=True,
)
thread.start()
global _WORKER_THREAD
_WORKER_THREAD = thread
return thread