gavindiaz/PolyWeather
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add dashboard prewarm worker and cache visibility

Browse Source
This commit is contained in:
2569718930@qq.com
2026-04-08 06:53:49 +08:00
parent 0bb3b573e1
commit c3da29c09c
21 changed files with 1120 additions and 149 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/data_collection/metar_sources.py
+3 -3
View File
@@ -5,7 +5,7 @@ import time
from datetime import datetime, timedelta, timezone
from typing import Dict, List, Optional
import requests
import httpx
from loguru import logger
from src.utils.metrics import record_source_call
@@ -209,7 +209,7 @@ class MetarSourceMixin:
record_source_call("metar", "current", "success", (time.perf_counter() - started) * 1000.0)
return result
except requests.exceptions.RequestException as exc:
except httpx.HTTPError as exc:
logger.error(f"METAR 请求失败 ({icao}): {exc}")
with self._metar_cache_lock:
stale = self._metar_cache.get(cache_key)
@@ -269,7 +269,7 @@ class MetarSourceMixin:
self._taf_cache[cache_key] = {"d": result, "t": now_ts}
record_source_call("taf", "current", "success", (time.perf_counter() - started) * 1000.0)
return result
except requests.exceptions.RequestException as exc:
except httpx.HTTPError as exc:
logger.error(f"TAF 请求失败 ({icao}): {exc}")
with self._taf_cache_lock:
stale = self._taf_cache.get(cache_key)
src/data_collection/mgm_sources.py
+9 -5
View File
@@ -32,7 +32,7 @@ class MgmSourceMixin:
# 1. 实时数据 (添加时间戳防止 CDN 缓存)
import time
obs_resp = self.session.get(
obs_resp = self._http_get(
f"{base_url}/sondurumlar?istno={istno}&_={int(time.time() * 1000)}",
headers=headers,
timeout=self.timeout,
@@ -95,7 +95,7 @@ class MgmSourceMixin:
]
for forecast_url in forecast_urls:
try:
daily_resp = self.session.get(
daily_resp = self._http_get(
forecast_url, headers=headers, timeout=self.timeout
)
if daily_resp.status_code == 200:
@@ -132,7 +132,7 @@ class MgmSourceMixin:
# 3. 小时预报
try:
hourly_resp = self.session.get(
hourly_resp = self._http_get(
f"{base_url}/tahminler/saatlik?istno={istno}",
headers=headers,
timeout=self.timeout
@@ -246,7 +246,11 @@ class MgmSourceMixin:
try:
# 1. 加载测站元数据 (缓存到实例中)，用于过滤属于该省份的站点
if not getattr(self, "mgm_stations_meta", None):
meta_resp = self.session.get(f"{base_url}/istasyonlar", headers=headers, timeout=self.timeout)
meta_resp = self._http_get(
f"{base_url}/istasyonlar",
headers=headers,
timeout=self.timeout,
)
if meta_resp.status_code == 200:
meta_json = meta_resp.json()
if isinstance(meta_json, list):
@@ -293,7 +297,7 @@ class MgmSourceMixin:
try:
# sondurumlar?istno={ist_no} 是目前最稳的获取多站数据的办法
url = f"{base_url}/sondurumlar?istno={ist_no}&_={int(time.time() * 1000)}"
resp = self.session.get(url, headers=headers, timeout=5)
resp = self._http_get(url, headers=headers, timeout=5)
if resp.status_code == 200:
obs_list = resp.json()
if obs_list:
src/data_collection/nmc_sources.py
+16 -4
View File
@@ -45,6 +45,20 @@ NMC_CITY_REFERENCES: Dict[str, Dict[str, Any]] = {
class NmcSourceMixin:
def _nmc_http_get(self, url: str):
getter = getattr(self, "_http_get", None)
if callable(getter):
return getter(url)
return self.session.get(url, timeout=self.timeout)
def _nmc_http_get_json(self, url: str):
getter = getattr(self, "_http_get_json", None)
if callable(getter):
return getter(url)
response = self.session.get(url, timeout=self.timeout)
response.raise_for_status()
return response.json()
@staticmethod
def _nmc_optional_text(value: Any) -> Optional[str]:
text = str(value or "").strip()
@@ -73,7 +87,7 @@ class NmcSourceMixin:
return None
try:
resp = self.session.get(page_url, timeout=self.timeout)
resp = self._nmc_http_get(page_url)
resp.raise_for_status()
match = re.search(
r"renderWeatherRealPanel\('([^']+)',\s*'([^']+)'\)",
@@ -116,9 +130,7 @@ class NmcSourceMixin:
try:
url = f"https://www.nmc.cn/rest/real/{station_code}"
resp = self.session.get(url, timeout=self.timeout)
resp.raise_for_status()
payload = resp.json()
payload = self._nmc_http_get_json(url)
if not isinstance(payload, dict) or not isinstance(payload.get("weather"), dict):
record_source_call("nmc", "current", "empty", (time.perf_counter() - started) * 1000.0)
return None
src/data_collection/nws_open_meteo_sources.py
+7 -7
View File
@@ -21,7 +21,7 @@ class NwsOpenMeteoSourceMixin:
points_url = f"https://api.weather.gov/points/{lat},{lon}"
headers = {"User-Agent": "PolyWeather/1.0 (weather-bot)"}
points_resp = self.session.get(
points_resp = self._http_get(
points_url, headers=headers, timeout=self.timeout
)
points_resp.raise_for_status()
@@ -35,7 +35,7 @@ class NwsOpenMeteoSourceMixin:
return None
# 2. 获取预报
forecast_resp = self.session.get(
forecast_resp = self._http_get(
forecast_url, headers=headers, timeout=self.timeout
)
forecast_resp.raise_for_status()
@@ -48,7 +48,7 @@ class NwsOpenMeteoSourceMixin:
hourly_periods = []
if hourly_url:
hourly_resp = self.session.get(
hourly_resp = self._http_get(
hourly_url, headers=headers, timeout=self.timeout
)
hourly_resp.raise_for_status()
@@ -57,7 +57,7 @@ class NwsOpenMeteoSourceMixin:
active_alerts = []
try:
alerts_resp = self.session.get(
alerts_resp = self._http_get(
"https://api.weather.gov/alerts/active",
params={"point": f"{lat},{lon}"},
headers=headers,
@@ -205,7 +205,7 @@ class NwsOpenMeteoSourceMixin:
params["temperature_unit"] = "celsius"
self._wait_open_meteo_slot("forecast")
response = self.session.get(
response = self._http_get(
url,
params=params,
timeout=self.timeout,
@@ -366,7 +366,7 @@ class NwsOpenMeteoSourceMixin:
params["temperature_unit"] = "celsius"
self._wait_open_meteo_slot("ensemble")
response = self.session.get(
response = self._http_get(
url,
params=params,
timeout=self.timeout,
@@ -523,7 +523,7 @@ class NwsOpenMeteoSourceMixin:
params["temperature_unit"] = "fahrenheit"
self._wait_open_meteo_slot("multi-model")
response = self.session.get(
response = self._http_get(
url,
params=params,
timeout=self.timeout,
src/data_collection/polymarket_readonly.py
+5 -2
View File
@@ -19,7 +19,7 @@ import unicodedata
from datetime import datetime, timezone
from typing import Any, Dict, List, Optional, Tuple
import requests
import httpx
from loguru import logger
from src.data_collection.city_registry import ALIASES, CITY_REGISTRY
@@ -383,7 +383,10 @@ class PolymarketReadOnlyLayer:
)
self.edge_threshold = _safe_float(os.getenv("POLYMARKET_SIGNAL_EDGE_PCT")) or 2.0
self._session = requests.Session()
self._session = httpx.Client(
timeout=self.http_timeout,
follow_redirects=True,
)
self._markets_cache: Dict[str, Dict[str, Any]] = {}
self._active_markets_cache: Dict[str, Any] = {"data": [], "t": 0.0}
self._broad_markets_cache: Dict[str, Any] = {"data": [], "t": 0.0}
src/data_collection/settlement_sources.py
+10 -10
View File
@@ -226,13 +226,13 @@ class SettlementSourceMixin:
try:
base = "https://data.weather.gov.hk/weatherAPI/hko_data/regional-weather"
temp_csv = self.session.get(f"{base}/latest_1min_temperature.csv", timeout=self.timeout)
temp_csv = self._http_get(f"{base}/latest_1min_temperature.csv", timeout=self.timeout)
temp_csv.raise_for_status()
maxmin_csv = self.session.get(f"{base}/latest_since_midnight_maxmin.csv", timeout=self.timeout)
maxmin_csv = self._http_get(f"{base}/latest_since_midnight_maxmin.csv", timeout=self.timeout)
maxmin_csv.raise_for_status()
humidity_csv = self.session.get(f"{base}/latest_1min_humidity.csv", timeout=self.timeout)
humidity_csv = self._http_get(f"{base}/latest_1min_humidity.csv", timeout=self.timeout)
humidity_csv.raise_for_status()
wind_csv = self.session.get(f"{base}/latest_10min_wind.csv", timeout=self.timeout)
wind_csv = self._http_get(f"{base}/latest_10min_wind.csv", timeout=self.timeout)
wind_csv.raise_for_status()
temp_rows = self._csv_rows(temp_csv.text)
@@ -302,7 +302,7 @@ class SettlementSourceMixin:
try:
url = "https://opendata.cwa.gov.tw/api/v1/rest/datastore/O-A0003-001"
response = self.session.get(
response = self._http_get(
url,
params={"Authorization": self.cwa_open_data_auth, "format": "JSON", "StationId": "466920"},
timeout=self.timeout,
@@ -361,7 +361,7 @@ class SettlementSourceMixin:
def fetch_hko_forecast(self) -> Optional[float]:
try:
url = "https://data.weather.gov.hk/weatherAPI/opendata/weather.php?dataType=fnd&lang=tc"
res = self.session.get(url, timeout=self.timeout).json()
res = self._http_get_json(url, timeout=self.timeout)
return float(res["weatherForecast"][0]["forecastMaxtemp"]["value"])
except Exception as exc:
logger.warning(f"HKO Forecast request failed: {exc}")
@@ -372,11 +372,11 @@ class SettlementSourceMixin:
if not self.cwa_open_data_auth:
return None
url = "https://opendata.cwa.gov.tw/api/v1/rest/datastore/F-D0047-061"
res = self.session.get(
res = self._http_get_json(
url,
params={"Authorization": self.cwa_open_data_auth, "format": "JSON", "elementName": "MaxT"},
timeout=self.timeout,
).json()
)
locs = res.get("records", {}).get("Locations", [])[0].get("Location", [])
if not locs:
return None
@@ -402,7 +402,7 @@ class SettlementSourceMixin:
return cached
try:
response = self.session.get(
response = self._http_get(
"https://api.synopticdata.com/v2/stations/timeseries",
params={
"STID": normalized_station_code,
@@ -519,7 +519,7 @@ class SettlementSourceMixin:
query = {"token": self.IMGW_METEO_API_TOKEN}
if isinstance(params, dict):
query.update(params)
response = self.session.get(
response = self._http_get(
f"{self.IMGW_METEO_API_BASE}/{path}",
params=query,
timeout=self.timeout,
src/data_collection/weather_sources.py
+82 -9
View File
@@ -1,7 +1,8 @@
import os
import requests
import httpx
import re
import threading
import time
from typing import Optional, Dict, List
from datetime import datetime, timedelta
from loguru import logger
@@ -110,7 +111,17 @@ class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSour
self.config = config
self.timeout = 30 # 增加超时以支持高延迟 VPS
self.session = requests.Session()
self.http_retry_count = max(
0, int(os.getenv("POLYWEATHER_HTTP_RETRY_COUNT", "1"))
)
self.http_retry_backoff_sec = max(
0.0, float(os.getenv("POLYWEATHER_HTTP_RETRY_BACKOFF_SEC", "0.35"))
)
self.session = httpx.Client(
timeout=self.timeout,
follow_redirects=True,
limits=httpx.Limits(max_connections=50, max_keepalive_connections=20),
)
self.open_meteo_cache_ttl_sec = int(
os.getenv("OPEN_METEO_CACHE_TTL_SEC", "900")
)
@@ -186,11 +197,73 @@ class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSour
if proxy:
if not proxy.startswith("http"):
proxy = f"http://{proxy}"
self.session.proxies = {"http": proxy, "https": proxy}
self.session = httpx.Client(
timeout=self.timeout,
follow_redirects=True,
proxy=proxy,
limits=httpx.Limits(max_connections=50, max_keepalive_connections=20),
)
logger.info(f"正在使用天气数据代理: {proxy}")
logger.info("天气数据采集器初始化完成。")
@staticmethod
def _is_retryable_status(status_code: int) -> bool:
return status_code in {408, 500, 502, 503, 504}
def _http_get(self, url: str, **kwargs) -> httpx.Response:
if "timeout" not in kwargs:
kwargs["timeout"] = self.timeout
last_exc: Optional[Exception] = None
last_response: Optional[httpx.Response] = None
attempts = self.http_retry_count + 1
for attempt in range(attempts):
try:
response = self.session.get(url, **kwargs)
last_response = response
if (
attempt < attempts - 1
and self._is_retryable_status(response.status_code)
):
wait_for = self.http_retry_backoff_sec * (attempt + 1)
logger.debug(
"HTTP GET retrying url={} status={} attempt={}/{} wait={}s",
url,
response.status_code,
attempt + 1,
attempts,
round(wait_for, 2),
)
if wait_for > 0:
time.sleep(wait_for)
continue
return response
except (httpx.TimeoutException, httpx.NetworkError) as exc:
last_exc = exc
if attempt >= attempts - 1:
break
wait_for = self.http_retry_backoff_sec * (attempt + 1)
logger.debug(
"HTTP GET retrying url={} error={} attempt={}/{} wait={}s",
url,
type(exc).__name__,
attempt + 1,
attempts,
round(wait_for, 2),
)
if wait_for > 0:
time.sleep(wait_for)
if last_exc is not None:
raise last_exc
if last_response is not None:
return last_response
raise RuntimeError(f"HTTP GET failed without response: {url}")
def _http_get_json(self, url: str, **kwargs):
response = self._http_get(url, **kwargs)
response.raise_for_status()
return response.json()
def fetch_from_openweather(self, city: str, country: str = None) -> Optional[Dict]:
"""
Fetch current weather and forecast from OpenWeatherMap
@@ -210,7 +283,7 @@ class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSour
try:
# Current weather
current_url = "https://api.openweathermap.org/data/2.5/weather"
current_response = self.session.get(
current_response = self._http_get(
current_url,
params={"q": query, "appid": self.openweather_key, "units": "metric"},
timeout=self.timeout,
@@ -220,7 +293,7 @@ class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSour
# 5-day forecast
forecast_url = "https://api.openweathermap.org/data/2.5/forecast"
forecast_response = self.session.get(
forecast_response = self._http_get(
forecast_url,
params={"q": query, "appid": self.openweather_key, "units": "metric"},
timeout=self.timeout,
@@ -245,7 +318,7 @@ class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSour
"forecast": self._parse_openweather_forecast(forecast_data),
}
except requests.exceptions.RequestException as e:
except httpx.HTTPError as e:
logger.error(f"OpenWeatherMap request failed: {e}")
return None
@@ -290,7 +363,7 @@ class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSour
try:
url = f"https://weather.visualcrossing.com/VisualCrossingWebServices/rest/services/timeline/{city}/{start_date}/{end_date}"
response = self.session.get(
response = self._http_get(
url,
params={
"unitGroup": "metric",
@@ -322,7 +395,7 @@ class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSour
],
}
except requests.exceptions.RequestException as e:
except httpx.HTTPError as e:
logger.error(f"Visual Crossing request failed: {e}")
return None
@@ -399,7 +472,7 @@ class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSour
try:
url = "https://geocoding-api.open-meteo.com/v1/search"
response = self.session.get(
response = self._http_get(
url,
params={"name": city, "count": 1, "language": "en", "format": "json"},
timeout=15, # 增加超时时间到 15s