gavindiaz/PolyWeather
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Remove AI analysis and rank cities by recent DEB performance

Browse Source
This commit is contained in:
2569718930@qq.com
2026-04-05 07:13:00 +08:00
parent 343c5c9c2f
commit a5b5711863
17 changed files with 124 additions and 905 deletions
Download Patch File Download Diff File Expand all files Collapse all files
frontend/components/dashboard/CitySidebar.tsx
+16 -5
View File
@@ -21,6 +21,10 @@ function toRiskGroup(level?: string): RiskGroupKey {
return "other";
}
function toPerformanceGroup(city: CityListItem): RiskGroupKey {
return toRiskGroup(city.deb_recent_tier);
}
function normalizeExpandedGroups(
value: unknown,
): Record<RiskGroupKey, boolean> {
@@ -60,11 +64,17 @@ export function CitySidebar() {
const sortedCities = useMemo(
() =>
[...store.cities].sort((a, b) => {
const aGroup = toRiskGroup(a.risk_level);
const bGroup = toRiskGroup(b.risk_level);
const aGroup = toPerformanceGroup(a);
const bGroup = toPerformanceGroup(b);
const aHitRate = Number(a.deb_recent_hit_rate ?? -1);
const bHitRate = Number(b.deb_recent_hit_rate ?? -1);
const aSamples = Number(a.deb_recent_sample_count ?? 0);
const bSamples = Number(b.deb_recent_sample_count ?? 0);
return (
(riskOrder[aGroup] ?? 3) -
(riskOrder[bGroup] ?? 3) ||
bHitRate - aHitRate ||
bSamples - aSamples ||
a.display_name.localeCompare(b.display_name)
);
}),
@@ -79,7 +89,7 @@ export function CitySidebar() {
other: [],
};
sortedCities.forEach((city) => {
groups[toRiskGroup(city.risk_level)].push(city);
groups[toPerformanceGroup(city)].push(city);
});
return groups;
}, [sortedCities]);
@@ -88,7 +98,7 @@ export function CitySidebar() {
if (!selectedCity) return;
const selected = store.cities.find((city) => city.name === selectedCity);
if (!selected) return;
const groupKey = toRiskGroup(selected.risk_level);
const groupKey = toPerformanceGroup(selected);
setExpandedGroups((current) =>
current[groupKey] ? current : { ...current, [groupKey]: true },
);
@@ -200,6 +210,7 @@ export function CitySidebar() {
const deviationSeverity =
snapshot?.deviation_monitor?.severity || "normal";
const secondaryText = deviationText || peakTempText;
const performanceTier = toPerformanceGroup(city);
return (
<button
@@ -213,7 +224,7 @@ export function CitySidebar() {
}
>
<div className="city-item-main">
<span className={clsx("risk-dot", city.risk_level)} />
<span className={clsx("risk-dot", performanceTier)} />
<span className="city-name-text">{city.display_name}</span>
<span
className={clsx(
frontend/components/dashboard/PanelSections.tsx
-30
View File
@@ -19,7 +19,6 @@ import {
getRiskBadgeLabel,
getTemperatureChartData,
getWeatherSummary,
parseAiAnalysis,
} from "@/lib/dashboard-utils";
function EmptyState({ text }: { text: string }) {
@@ -760,35 +759,6 @@ export function ForecastTable() {
);
}
export function AiAnalysis() {
const { data } = useCityData();
const { t } = useI18n();
if (!data) return null;
const ai = parseAiAnalysis(data.ai_analysis);
return (
<section className="ai-section">
<h3>{t("section.ai")}</h3>
<div className="ai-box">
{!ai.summary && ai.bullets.length === 0 ? (
<span className="ai-placeholder">{t("section.aiEmpty")}</span>
) : (
<>
{ai.summary && <div className="ai-summary">{ai.summary}</div>}
{ai.bullets.length > 0 && (
<ul className="ai-list">
{ai.bullets.map((item) => (
<li key={item}>{item}</li>
))}
</ul>
)}
</>
)}
</div>
</section>
);
}
export function RiskInfo() {
const { data } = useCityData();
const { t } = useI18n();
frontend/components/terminal/AnalyticsPanel.tsx
+1 -21
View File
@@ -4,7 +4,6 @@ import React from "react";
import {
BarChart2,
Target,
ShieldAlert,
Zap,
Info,
Activity,
@@ -23,7 +22,7 @@ export function AnalyticsPanel({
data,
t = {}, // Default empty for now, can be expanded via context or props
}: AnalyticsPanelProps) {
const { overview, market_scan, models, ai_analysis } = data;
const { overview, market_scan, models } = data;
const modelEntries = Object.entries(models)
.filter(([_, v]) => v !== undefined && v !== null)
@@ -222,25 +221,6 @@ export function AnalyticsPanel({
))}
</div>
</section>
{ai_analysis && (
<section className="pt-2">
<div className="flex items-center gap-2 mb-3">
<Activity className="h-3 w-3 text-amber-500" />
<span className="text-[10px] font-black uppercase tracking-[0.15em] text-zinc-400">
AI COGNITIVE ANALYSIS
</span>
</div>
<div className="rounded-lg border border-zinc-800 bg-zinc-900/30 p-4 relative overflow-hidden group">
<div className="absolute top-0 right-0 p-2 opacity-20">
<ShieldAlert className="w-8 h-8 text-amber-500" />
</div>
<p className="text-[11px] leading-relaxed text-zinc-400 relative z-10 font-medium">
{ai_analysis}
</p>
</div>
</section>
)}
</div>
{/* Execute Scan Footer */}
frontend/hooks/useDashboardStore.tsx
+3 -96
View File
@@ -81,96 +81,6 @@ function getInitialProAccessState(): ProAccessState {
};
}
const AI_EMPTY_PATTERNS = [
/暂无\s*AI\s*分析/i,
/当前以结构化气象与模型数据为主/i,
/No\s*AI\s*analysis\s*available/i,
/Structured\s+meteorological\s+and\s+model\s+data/i,
];
function normalizeText(value: unknown) {
return typeof value === "string" ? value.trim() : "";
}
function extractAiPayload(analysis: CityDetail["ai_analysis"]) {
if (!analysis) {
return {
bullets: [] as string[],
summary: "",
};
}
if (typeof analysis === "string") {
return {
bullets: [] as string[],
summary: normalizeText(analysis),
};
}
const summary =
normalizeText(analysis.summary) ||
normalizeText(analysis.text) ||
normalizeText(analysis.message);
const bulletsSource = Array.isArray(analysis.highlights)
? analysis.highlights
: Array.isArray(analysis.points)
? analysis.points
: [];
return {
bullets: bulletsSource.map((item) => normalizeText(item)).filter(Boolean),
summary,
};
}
function hasMeaningfulAiAnalysis(analysis: CityDetail["ai_analysis"]) {
const parsed = extractAiPayload(analysis);
const hasBullets = parsed.bullets.length > 0;
const hasSummary =
Boolean(parsed.summary) &&
!AI_EMPTY_PATTERNS.some((pattern) => pattern.test(parsed.summary));
return hasBullets || hasSummary;
}
function normalizeMetarSignature(detail?: CityDetail) {
if (!detail) return "";
const metar = normalizeText(detail.current?.raw_metar)
.replace(/\s+/g, " ")
.toUpperCase();
const obsTime = normalizeText(detail.current?.obs_time);
return [metar, obsTime].filter(Boolean).join("|");
}
function mergeAiAnalysisIfStable(
previousDetail: CityDetail | undefined,
nextDetail: CityDetail,
) {
if (!previousDetail) return nextDetail;
if (hasMeaningfulAiAnalysis(nextDetail.ai_analysis)) return nextDetail;
if (!hasMeaningfulAiAnalysis(previousDetail.ai_analysis)) return nextDetail;
const prevTemp = Number(previousDetail.current?.temp);
const nextTemp = Number(nextDetail.current?.temp);
const tempUnchanged =
Number.isFinite(prevTemp) &&
Number.isFinite(nextTemp) &&
prevTemp === nextTemp;
const prevMetar = normalizeMetarSignature(previousDetail);
const nextMetar = normalizeMetarSignature(nextDetail);
const metarUnchanged =
Boolean(prevMetar) && Boolean(nextMetar) && prevMetar === nextMetar;
if (!tempUnchanged && !metarUnchanged) {
return nextDetail;
}
return {
...nextDetail,
ai_analysis: previousDetail.ai_analysis,
};
}
function getMarketScanCacheKey(cityName: string, targetDate?: string | null) {
const normalizedDate = String(targetDate || "").trim() || "local";
return `${cityName}::${normalizedDate}`;
@@ -286,7 +196,7 @@ export function DashboardStoreProvider({
const latestDetail = await dashboardClient.getCityDetail(cityName, {
force: false,
});
const detail = mergeAiAnalysisIfStable(cached, latestDetail);
const detail = latestDetail;
setCityDetailsByName((current) => ({
...current,
@@ -337,7 +247,7 @@ export function DashboardStoreProvider({
const latestDetail = await dashboardClient.getCityDetail(cityName, {
force,
});
const detail = mergeAiAnalysisIfStable(cached, latestDetail);
const detail = latestDetail;
setCityDetailsByName((current) => ({
...current,
[cityName]: detail,
@@ -608,10 +518,7 @@ export function DashboardStoreProvider({
const latestDetail = await dashboardClient.getCityDetail(selectedCity, {
force: true,
});
const detail = mergeAiAnalysisIfStable(
previousSelectedDetail,
latestDetail,
);
const detail = latestDetail;
setCityDetailsByName({ [selectedCity]: detail });
setCitySummariesByName((current) => ({
...current,
frontend/lib/dashboard-types.ts
+5
View File
@@ -6,6 +6,11 @@ export interface CityListItem {
lat: number;
lon: number;
risk_level: RiskLevel;
deb_recent_tier?: RiskLevel;
deb_recent_hit_rate?: number | null;
deb_recent_sample_count?: number;
deb_recent_mae?: number | null;
deb_recent_last_date?: string | null;
risk_emoji?: string;
airport: string;
icao: string;
frontend/lib/i18n.ts
+8 -8
View File
@@ -26,10 +26,10 @@ const MESSAGES: Record<Locale, Record<string, string>> = {
"sidebar.currentTemp": "当前 {temp}",
"sidebar.peakTempAt": "峰值 {temp} @ {time}",
"sidebar.peakAt": "峰值 @ {time}",
"sidebar.group.high": "高风险",
"sidebar.group.medium": "中风险",
"sidebar.group.low": "低风险",
"sidebar.group.other": "其他",
"sidebar.group.high": "近期强势",
"sidebar.group.medium": "近期一般",
"sidebar.group.low": "近期偏弱",
"sidebar.group.other": "样本不足",
"dashboard.loading": "正在同步站点观测与结算源，请稍候...",
@@ -189,10 +189,10 @@ const MESSAGES: Record<Locale, Record<string, string>> = {
"sidebar.currentTemp": "Current {temp}",
"sidebar.peakTempAt": "Peak {temp} @ {time}",
"sidebar.peakAt": "Peak @ {time}",
"sidebar.group.high": "High Risk",
"sidebar.group.medium": "Medium Risk",
"sidebar.group.low": "Low Risk",
"sidebar.group.other": "Others",
"sidebar.group.high": "Recent Strong",
"sidebar.group.medium": "Recent Mixed",
"sidebar.group.low": "Recent Weak",
"sidebar.group.other": "Low Sample",
"dashboard.loading":
"Synchronizing station observations and settlement feeds...",
scripts/backfill_recent_daily_actuals_from_metar.py
+1 -1
View File
@@ -25,7 +25,7 @@ def _target_dates(city_info: dict, lookback_days: int) -> list[str]:
def _is_metar_city(city_info: dict) -> bool:
source = str(city_info.get("settlement_source") or "metar").strip().lower()
return source in {"metar", "hko", "noaa", "wunderground"}
return source in {"metar", "hko", "noaa"}
def main() -> None:
src/analysis/city_query_service.py
+1 -1
View File
@@ -335,7 +335,7 @@ def build_city_query_report(
sc_current = {}
city_meta = CITY_REGISTRY.get(city_name.lower(), {})
settlement_source, settlement_source_label = _resolve_settlement_source(city_meta)
use_settlement_current = settlement_source in {"hko", "cwa", "noaa", "wunderground"} and bool(sc_current)
use_settlement_current = settlement_source in {"hko", "cwa", "noaa"} and bool(sc_current)
fallback_utc_offset = int(city_meta.get("tz_offset", 0))
nws_periods = ((weather_data.get("nws") or {}).get("forecast_periods") or [])
if nws_periods:
src/analysis/deb_algorithm.py
+9 -73
View File
@@ -4,7 +4,6 @@ from datetime import datetime, timedelta
from typing import Optional
import requests
from src.analysis.settlement_rounding import apply_city_settlement
from src.data_collection.wunderground_sources import fetch_wunderground_historical_high
from loguru import logger
from src.database.runtime_state import (
DailyRecordRepository,
@@ -630,76 +629,13 @@ def _reconcile_recent_noaa_actual_highs(city_name: str, lookback_days: int = 14)
def _reconcile_recent_wunderground_actual_highs(city_name: str, lookback_days: int = 14):
try:
city_key, city_meta = _resolve_city_history_context(city_name)
if not city_key or not isinstance(city_meta, dict):
return {"ok": False, "reason": "unknown_city", "updated": 0}
settlement_url = str(city_meta.get("settlement_url") or "").strip()
if not settlement_url:
return {"ok": False, "reason": "missing_settlement_url", "updated": 0}
tz_offset = int(city_meta.get("tz_offset") or 0)
history_file = _get_history_file_path()
data = load_history(history_file)
city_data = data.get(city_key) or {}
if not isinstance(city_data, dict) or not city_data:
return {"ok": True, "reason": "no_city_history", "updated": 0}
local_now = datetime.utcnow() + timedelta(seconds=tz_offset)
local_today = local_now.strftime("%Y-%m-%d")
cutoff = (local_now - timedelta(days=max(lookback_days, 1) + 1)).strftime(
"%Y-%m-%d"
)
target_dates = sorted(
d for d in city_data.keys() if isinstance(d, str) and cutoff <= d < local_today
)
if not target_dates:
return {"ok": True, "reason": "no_target_dates", "updated": 0}
updated = 0
scanned_dates = 0
for date_str in target_dates:
result = fetch_wunderground_historical_high(city_key, date_str, url=settlement_url)
if not result.get("ok"):
continue
scanned_dates += 1
corrected = _sf(result.get("actual_high"))
if corrected is None:
continue
rec = city_data.get(date_str) or {}
old = rec.get("actual_high")
try:
old_val = float(old) if old is not None else None
except Exception:
old_val = None
if old_val is None or abs(old_val - corrected) >= 0.1:
rec["actual_high"] = corrected
city_data[date_str] = rec
updated += 1
_persist_truth_record(
city_key,
date_str,
corrected,
city_meta=city_meta,
updated_by="backfill:wunderground_history",
reason="reconcile_recent_actual_highs",
source_payload=result,
)
if updated > 0:
data[city_key] = city_data
save_history(history_file, data)
return {
"ok": True,
"updated": updated,
"scanned_dates": scanned_dates,
"station_code": city_meta.get("settlement_station_code"),
"source": "wunderground",
}
except Exception as e:
return {"ok": False, "reason": str(e), "updated": 0}
return {
"ok": True,
"reason": "wunderground_crawler_removed",
"updated": 0,
"scanned_dates": 0,
"source": "wunderground",
}
def reconcile_recent_actual_highs(city_name: str, lookback_days: int = 7):
@@ -732,14 +668,14 @@ def bootstrap_recent_daily_history_if_missing(city_name: str, lookback_days: int
return {"ok": False, "reason": "unknown_city", "seeded": 0, "updated": 0}
settlement_source = str(city_meta.get("settlement_source") or "metar").strip().lower()
if settlement_source not in {"metar", "hko", "noaa", "wunderground"}:
if settlement_source not in {"metar", "hko", "noaa"}:
return {"ok": True, "reason": "unsupported_settlement_source", "seeded": 0, "updated": 0}
icao = str(city_meta.get("icao") or "").strip().upper()
station_code = str(city_meta.get("settlement_station_code") or "").strip().upper()
if settlement_source == "metar" and not icao:
return {"ok": False, "reason": "missing_icao", "seeded": 0, "updated": 0}
if settlement_source in {"hko", "noaa", "wunderground"} and not station_code:
if settlement_source in {"hko", "noaa"} and not station_code:
return {"ok": False, "reason": "missing_station_code", "seeded": 0, "updated": 0}
tz_offset = int(city_meta.get("tz_offset") or 0)
src/data_collection/settlement_sources.py
+2 -8
View File
@@ -600,14 +600,8 @@ class SettlementSourceMixin:
city_meta = CITY_REGISTRY.get(normalized) or {}
settlement_source = str(city_meta.get("settlement_source") or "").strip().lower()
if settlement_source == "wunderground":
settlement_url = str(city_meta.get("settlement_url") or "").strip()
if settlement_url:
return self.fetch_wunderground_settlement_current(
normalized,
url=settlement_url,
station_label=str(city_meta.get("settlement_station_label") or "").strip() or None,
icao=str(city_meta.get("icao") or "").strip() or None,
)
logger.info("Settlement current skipped city={} source=wunderground reason=crawler_removed", city)
return None
if settlement_source == "hko":
raw_candidates = city_meta.get("settlement_station_candidates") or []
if isinstance(raw_candidates, str):
src/data_collection/weather_sources.py
+1 -4
View File
@@ -10,10 +10,9 @@ from src.data_collection.settlement_sources import SettlementSourceMixin
from src.data_collection.metar_sources import MetarSourceMixin
from src.data_collection.mgm_sources import MgmSourceMixin
from src.data_collection.nws_open_meteo_sources import NwsOpenMeteoSourceMixin
from src.data_collection.wunderground_sources import WundergroundSourceMixin
class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSourceMixin, MgmSourceMixin, NwsOpenMeteoSourceMixin, WundergroundSourceMixin):
class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSourceMixin, MgmSourceMixin, NwsOpenMeteoSourceMixin):
"""
Multi-source weather data collector
@@ -108,8 +107,6 @@ class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSour
def __init__(self, config: dict):
self.config = config
weather_cfg = config.get("weather", {})
self.wunderground_key = weather_cfg.get("wunderground_api_key")
self.timeout = 30 # 增加超时以支持高延迟 VPS
self.session = requests.Session()
src/data_collection/wunderground_sources.py
-595
View File
@@ -1,595 +0,0 @@
from __future__ import annotations
import json
import re
from datetime import datetime, timedelta, timezone
from typing import Any, Dict, Optional
import requests
from loguru import logger
from src.data_collection.city_registry import CITY_REGISTRY
from src.analysis.settlement_rounding import apply_city_settlement
class WundergroundSourceMixin:
_WU_PAGE_TTL_SEC = 180
def _fetch_wunderground_page(self, url: str) -> Optional[str]:
cache_key = f"wu:page:{url}"
cached = self._get_settlement_cache(cache_key)
if isinstance(cached, dict):
html = str(cached.get("html") or "")
if html:
return html
try:
response = self.session.get(
url,
headers={
"User-Agent": "Mozilla/5.0",
"Referer": url,
},
timeout=self.timeout,
)
response.raise_for_status()
html = str(response.text or "")
if not html:
return None
ttl_backup = getattr(self, "settlement_cache_ttl_sec", self._WU_PAGE_TTL_SEC)
try:
self.settlement_cache_ttl_sec = self._WU_PAGE_TTL_SEC
self._set_settlement_cache(cache_key, {"html": html})
finally:
self.settlement_cache_ttl_sec = ttl_backup
return html
except Exception as exc:
logger.warning(f"Wunderground page fetch failed url={url}: {exc}")
return None
def _fetch_wunderground_history_page(self, url: str) -> Optional[str]:
if not url:
return None
cache_key = f"wu:history:{url}"
cached = self._get_settlement_cache(cache_key)
if isinstance(cached, dict):
html = str(cached.get("html") or "")
if html:
return html
try:
response = self.session.get(
url,
headers={
"User-Agent": "Mozilla/5.0",
"Referer": url,
},
timeout=self.timeout,
)
response.raise_for_status()
html = str(response.text or "")
if not html:
return None
ttl_backup = getattr(self, "settlement_cache_ttl_sec", self._WU_PAGE_TTL_SEC)
try:
self.settlement_cache_ttl_sec = self._WU_PAGE_TTL_SEC
self._set_settlement_cache(cache_key, {"html": html})
finally:
self.settlement_cache_ttl_sec = ttl_backup
return html
except Exception as exc:
logger.warning(f"Wunderground history page fetch failed url={url}: {exc}")
return None
@staticmethod
def _wu_extract_app_state(html: str) -> Optional[Dict[str, Any]]:
match = re.search(
r'<script id="app-root-state" type="application/json">(.*?)</script>',
html,
re.IGNORECASE | re.DOTALL,
)
if not match:
return None
try:
payload = json.loads(str(match.group(1) or ""))
return payload if isinstance(payload, dict) else None
except Exception:
return None
@staticmethod
def _wu_extract_station_history_url(html: str) -> Optional[str]:
match = re.search(
r'<div[^>]*class="station-name"[^>]*>\s*<a[^>]+href="([^"]*?/history/daily/[^"]+)"',
html,
re.IGNORECASE | re.DOTALL,
)
if not match:
return None
href = str(match.group(1) or "").strip()
if not href:
return None
if href.startswith("http://") or href.startswith("https://"):
return href
return f"https://www.wunderground.com{href}"
@staticmethod
def _wu_extract_station_id_from_history_url(url: Optional[str]) -> Optional[str]:
text = str(url or "").strip()
if not text:
return None
match = re.search(r"/history/daily/(?:[^/]+/){1,3}([^/]+)/date/", text, re.IGNORECASE)
return str(match.group(1) or "").strip() or None if match else None
@staticmethod
def _wu_extract_station_name_from_history_url(url: Optional[str]) -> Optional[str]:
text = str(url or "").strip()
if not text:
return None
match = re.search(r"/history/daily/(?:[^/]+/){1,3}([^/]+)/date/", text, re.IGNORECASE)
return str(match.group(1) or "").strip() or None if match else None
@staticmethod
def _safe_float(value: Any) -> Optional[float]:
if value is None:
return None
text = str(value).strip()
if not text:
return None
try:
return float(text)
except Exception:
return None
@staticmethod
def _wu_extract_station_name(html: str, fallback_icao: str) -> Optional[str]:
pattern = re.compile(
r'</lib-display-unit>\s*([^<]+?)\s*</a>',
re.IGNORECASE,
)
for match in pattern.finditer(html):
candidate = re.sub(r"\s+", " ", str(match.group(1) or "")).strip()
if fallback_icao.lower() in candidate.lower() or "station" in candidate.lower():
return candidate
return None
@staticmethod
def _wu_extract_station_temperature(
html: str,
*,
station_name: Optional[str],
) -> tuple[Optional[float], Optional[str]]:
station_anchor = station_name or "Station"
station_pos = html.find(station_anchor)
if station_pos < 0:
station_pos = html.lower().find("station-name")
if station_pos < 0:
return None, None
window_start = max(0, station_pos - 1800)
window = html[window_start:station_pos]
temp_match = re.search(
r'wu-value[^>]*>\s*(-?\d+(?:\.\d+)?)\s*</span>.*?<span[^>]*>\s*([CF])\s*</span>',
window,
re.IGNORECASE | re.DOTALL,
)
if not temp_match:
return None, None
try:
value = float(temp_match.group(1))
except Exception:
return None, None
unit = str(temp_match.group(2) or "").upper().strip() or None
return value, unit
@staticmethod
def _wu_to_celsius(value: Optional[float], unit: Optional[str]) -> Optional[float]:
if value is None:
return None
normalized = str(unit or "").upper().strip()
if normalized == "F":
return round((float(value) - 32.0) * 5.0 / 9.0, 1)
return round(float(value), 1)
@staticmethod
def _wu_parse_observation_iso(raw: Optional[str], utc_offset_seconds: int) -> Optional[str]:
text = str(raw or "").strip()
if not text:
return None
try:
parsed = datetime.strptime(text, "%Y-%m-%dT%H:%M:%S%z")
return parsed.astimezone(timezone.utc).isoformat()
except Exception:
pass
try:
parsed = datetime.strptime(text, "%Y-%m-%d %H:%M:%S")
parsed = parsed.replace(tzinfo=timezone(timedelta(seconds=utc_offset_seconds)))
return parsed.astimezone(timezone.utc).isoformat()
except Exception:
return None
@staticmethod
def _wu_parse_history_time(raw: Any, utc_offset_seconds: int) -> Optional[str]:
if raw is None:
return None
if isinstance(raw, (int, float)):
try:
return datetime.fromtimestamp(float(raw), timezone.utc).isoformat()
except Exception:
return None
text = str(raw).strip()
if not text:
return None
for fmt in (
"%Y-%m-%dT%H:%M:%S%z",
"%Y-%m-%dT%H:%M:%S",
"%Y-%m-%d %H:%M:%S",
"%Y-%m-%d %H:%M",
):
try:
parsed = datetime.strptime(text, fmt)
if parsed.tzinfo is None:
parsed = parsed.replace(tzinfo=timezone(timedelta(seconds=utc_offset_seconds)))
return parsed.astimezone(timezone.utc).isoformat()
except Exception:
continue
return None
@classmethod
def _wu_observation_temp_c(cls, obs: Dict[str, Any]) -> Optional[float]:
if not isinstance(obs, dict):
return None
metric = obs.get("metric")
if isinstance(metric, dict):
for key in ("temp", "temperature", "tempAvg", "temperatureAvg"):
value = cls._safe_float(metric.get(key))
if value is not None:
return round(float(value), 1)
imperial = obs.get("imperial")
if isinstance(imperial, dict):
for key in ("temp", "temperature", "tempAvg", "temperatureAvg"):
value = cls._safe_float(imperial.get(key))
if value is not None:
return cls._wu_to_celsius(value, "F")
for key in ("temp", "temperature"):
value = cls._safe_float(obs.get(key))
if value is not None:
return round(float(value), 1)
return None
@classmethod
def _wu_observation_time_iso(cls, obs: Dict[str, Any], utc_offset_seconds: int) -> Optional[str]:
if not isinstance(obs, dict):
return None
for key in ("validTimeLocal", "obsTimeLocal", "observationTime", "timestamp", "validTimeUtc", "epoch"):
value = obs.get(key)
parsed = cls._wu_parse_history_time(value, utc_offset_seconds)
if parsed:
return parsed
return None
@classmethod
def _wu_iter_lists(cls, node: Any):
if isinstance(node, dict):
for value in node.values():
yield from cls._wu_iter_lists(value)
elif isinstance(node, list):
yield node
for item in node:
yield from cls._wu_iter_lists(item)
@classmethod
def _wu_extract_history_observations(
cls,
app_state: Dict[str, Any],
*,
utc_offset_seconds: int,
) -> list[dict[str, Any]]:
best: list[dict[str, Any]] = []
for candidate in cls._wu_iter_lists(app_state):
if not candidate or not all(isinstance(item, dict) for item in candidate):
continue
parsed: list[dict[str, Any]] = []
seen_times: set[str] = set()
for item in candidate:
temp_c = cls._wu_observation_temp_c(item)
time_iso = cls._wu_observation_time_iso(item, utc_offset_seconds)
if temp_c is None or not time_iso:
continue
try:
local_dt = datetime.fromisoformat(time_iso.replace("Z", "+00:00")).astimezone(
timezone(timedelta(seconds=utc_offset_seconds))
)
except Exception:
continue
hhmm = local_dt.strftime("%H:%M")
if hhmm in seen_times:
continue
seen_times.add(hhmm)
parsed.append({"time": hhmm, "temp": round(float(temp_c), 1)})
if len(parsed) > len(best):
best = parsed
return best
@staticmethod
def _wu_find_current_observation_block(
app_state: Dict[str, Any],
*,
icao: Optional[str],
) -> Optional[Dict[str, Any]]:
normalized_icao = str(icao or "").strip().upper()
fallback_block: Optional[Dict[str, Any]] = None
for value in app_state.values():
if not isinstance(value, dict):
continue
url = str(value.get("u") or "")
body = value.get("b")
if not isinstance(body, dict) or "observations/current" not in url:
continue
if normalized_icao and f"icaoCode={normalized_icao}" in url:
return body
if fallback_block is None:
fallback_block = body
return fallback_block
def fetch_wunderground_settlement_current(
self,
city: str,
*,
url: str,
station_label: Optional[str] = None,
icao: Optional[str] = None,
) -> Optional[Dict[str, Any]]:
normalized_city = str(city or "").strip().lower()
cache_key = f"wu:settlement:{normalized_city}"
cached = self._get_settlement_cache(cache_key)
if cached:
return cached
html = self._fetch_wunderground_page(url)
if not html:
return None
city_meta = CITY_REGISTRY.get(normalized_city) or {}
utc_offset_seconds = int(city_meta.get("tz_offset") or 0)
fallback_icao = str(icao or "").strip().upper()
history_url = self._wu_extract_station_history_url(html)
station_id = self._wu_extract_station_id_from_history_url(history_url)
station_name = (
station_label
or self._wu_extract_station_name(html, fallback_icao)
or self._wu_extract_station_name_from_history_url(history_url)
or fallback_icao
or str(city or "").title()
)
app_state = self._wu_extract_app_state(html) or {}
current_block = self._wu_find_current_observation_block(app_state, icao=fallback_icao)
display_temp = None
display_unit = "F"
obs_iso = None
humidity = None
wind_speed_kt = None
wind_dir = None
wx_phrase = None
dewpoint_c = None
official_high_c = None
official_low_c = None
if current_block:
display_temp = self._safe_float(current_block.get("temperature"))
obs_iso = self._wu_parse_observation_iso(
current_block.get("validTimeLocal"),
utc_offset_seconds,
)
humidity = self._safe_float(current_block.get("relativeHumidity"))
wind_speed_kt = self._safe_float(current_block.get("windSpeed"))
wind_dir = str(current_block.get("windDirectionCardinal") or "").strip() or None
wx_phrase = str(current_block.get("wxPhraseLong") or "").strip() or None
dewpoint_c = self._wu_to_celsius(
self._safe_float(current_block.get("temperatureDewPoint")),
"F",
)
official_high_c = self._wu_to_celsius(
self._safe_float(
current_block.get("temperatureMaxSince7Am")
or current_block.get("temperatureMax24Hour")
),
"F",
)
official_low_c = self._wu_to_celsius(
self._safe_float(current_block.get("temperatureMin24Hour")),
"F",
)
if display_temp is None:
display_temp, display_unit = self._wu_extract_station_temperature(
html,
station_name=station_name,
)
temp_c = self._wu_to_celsius(self._safe_float(display_temp), display_unit)
if temp_c is None:
logger.warning(f"Wunderground temperature parse failed city={city} url={url}")
return None
obs_iso = obs_iso or datetime.now(timezone.utc).isoformat()
today_obs: list[dict[str, Any]] = []
history_html = self._fetch_wunderground_history_page(history_url) if history_url else None
history_state = self._wu_extract_app_state(history_html or "") if history_html else None
history_obs = (
self._wu_extract_history_observations(
history_state or {},
utc_offset_seconds=utc_offset_seconds,
)
if isinstance(history_state, dict)
else []
)
if history_obs:
today_obs = history_obs
for point in history_obs:
point_time = str(point.get("time") or "").strip()
point_temp = self._safe_float(point.get("temp"))
if not point_time or point_temp is None:
continue
try:
local_dt = datetime.strptime(point_time, "%H:%M").replace(
year=datetime.now(timezone(timedelta(seconds=utc_offset_seconds))).year,
month=datetime.now(timezone(timedelta(seconds=utc_offset_seconds))).month,
day=datetime.now(timezone(timedelta(seconds=utc_offset_seconds))).day,
tzinfo=timezone(timedelta(seconds=utc_offset_seconds)),
)
self._update_official_today_obs(
source_code="wunderground",
station_code=station_id or fallback_icao or normalized_city,
obs_iso=local_dt.astimezone(timezone.utc).isoformat(),
current_temp=point_temp,
utc_offset_seconds=utc_offset_seconds,
)
except Exception:
continue
else:
today_obs = self._update_official_today_obs(
source_code="wunderground",
station_code=station_id or fallback_icao or normalized_city,
obs_iso=obs_iso,
current_temp=temp_c,
utc_offset_seconds=utc_offset_seconds,
)
sampled_max = None
sampled_max_time = None
sampled_low = None
if today_obs:
hottest = max(today_obs, key=lambda item: float(item.get("temp") or -999))
coldest = min(today_obs, key=lambda item: float(item.get("temp") or 999))
sampled_max = self._wu_to_celsius(float(hottest.get("temp")), "C")
sampled_max_time = str(hottest.get("time") or "").strip() or None
sampled_low = self._wu_to_celsius(float(coldest.get("temp")), "C")
max_so_far = official_high_c if official_high_c is not None else sampled_max
today_low = official_low_c if official_low_c is not None else sampled_low
max_temp_time = sampled_max_time
if max_so_far is not None and abs(float(max_so_far) - float(temp_c)) < 0.05:
try:
local_obs = datetime.fromisoformat(obs_iso.replace("Z", "+00:00")).astimezone(
timezone(timedelta(seconds=utc_offset_seconds))
)
max_temp_time = local_obs.strftime("%H:%M")
except Exception:
pass
payload: Dict[str, Any] = {
"source": "wunderground",
"source_label": "Wunderground",
"station_code": station_id or fallback_icao or None,
"station_name": station_name,
"observation_time": obs_iso,
"source_url": url,
"history_url": history_url,
"current": {
"temp": temp_c,
"display_temp": display_temp,
"display_unit": display_unit,
"max_temp_so_far": max_so_far,
"max_temp_time": max_temp_time,
"today_low": today_low,
"humidity": humidity,
"wind_speed_kt": wind_speed_kt,
"wind_dir": wind_dir,
"cloud_desc": wx_phrase,
"dewpoint": dewpoint_c,
},
"today_obs": today_obs,
"unit": "celsius",
}
self._set_settlement_cache(cache_key, payload)
return payload
def _normalize_wu_history_date_url(url: str, target_date: str) -> str:
normalized = str(url or "").strip().rstrip("/")
normalized = re.sub(r"/date/\d{4}-\d{2}-\d{2}$", "", normalized, flags=re.IGNORECASE)
return f"{normalized}/date/{target_date}"
def fetch_wunderground_historical_high(
city: str,
target_date: str,
*,
url: Optional[str] = None,
timeout: int = 15,
session: Optional[requests.Session] = None,
) -> Dict[str, Any]:
city_key = str(city or "").strip().lower()
city_meta = CITY_REGISTRY.get(city_key) or {}
history_url = _normalize_wu_history_date_url(
url or str(city_meta.get("settlement_url") or "").strip(),
target_date,
)
if not history_url:
return {"ok": False, "reason": "missing_history_url", "city": city_key, "date": target_date}
requester = session or requests.Session()
try:
response = requester.get(
history_url,
headers={
"User-Agent": "Mozilla/5.0",
"Referer": history_url,
},
timeout=timeout,
)
response.raise_for_status()
html = str(response.text or "")
except Exception as exc:
logger.warning(f"Wunderground history fetch failed city={city_key} date={target_date}: {exc}")
return {
"ok": False,
"reason": "fetch_failed",
"city": city_key,
"date": target_date,
"history_url": history_url,
"error": str(exc),
}
app_state = WundergroundSourceMixin._wu_extract_app_state(html)
if not isinstance(app_state, dict):
return {
"ok": False,
"reason": "missing_app_state",
"city": city_key,
"date": target_date,
"history_url": history_url,
}
utc_offset_seconds = int(city_meta.get("tz_offset") or 0)
obs = WundergroundSourceMixin._wu_extract_history_observations(
app_state,
utc_offset_seconds=utc_offset_seconds,
)
if not obs:
return {
"ok": False,
"reason": "missing_observations",
"city": city_key,
"date": target_date,
"history_url": history_url,
}
raw_max_temp_c = max(float(point.get("temp")) for point in obs if point.get("temp") is not None)
settled_actual_high = apply_city_settlement(city_key, raw_max_temp_c)
station_code = str(city_meta.get("settlement_station_code") or "").strip().upper() or None
station_label = str(city_meta.get("settlement_station_label") or "").strip() or None
return {
"ok": True,
"city": city_key,
"date": target_date,
"history_url": history_url,
"raw_max_temp_c": round(raw_max_temp_c, 1),
"actual_high": float(settled_actual_high),
"settlement_source": "wunderground",
"settlement_station_code": station_code,
"settlement_station_label": station_label,
"observation_count": len(obs),
"observations": obs,
}
src/utils/config_loader.py
-1
View File
@@ -22,7 +22,6 @@ def load_config():
config = {
"weather": {
"openweather_api_key": get_env_or_none("OPENWEATHER_API_KEY"),
"wunderground_api_key": get_env_or_none("WUNDERGROUND_API_KEY"),
"visualcrossing_api_key": get_env_or_none("VISUALCROSSING_API_KEY"),
"proxy": os.getenv("HTTPS_PROXY") or os.getenv("HTTP_PROXY"),
},
tests/test_web_observability.py
+2
View File
@@ -52,6 +52,8 @@ def test_cities_endpoint_uses_denver_display_name_for_aurora_market():
payload = response.json()
aurora = next(item for item in payload["cities"] if item["name"] == "aurora")
assert aurora["display_name"] == "Denver"
assert aurora["deb_recent_tier"] in {"high", "medium", "low", "other"}
assert "deb_recent_sample_count" in aurora
def test_cities_endpoint_includes_new_wunderground_cities():
tests/test_wunderground_history.py
-44
View File
@@ -1,44 +0,0 @@
from src.data_collection.wunderground_sources import fetch_wunderground_historical_high
class _FakeResponse:
def __init__(self, text: str):
self.text = text
def raise_for_status(self):
return None
class _FakeSession:
def __init__(self, text: str):
self._text = text
def get(self, url, headers=None, timeout=None):
return _FakeResponse(self._text)
def test_fetch_wunderground_historical_high_parses_daily_max():
html = """
<html><body>
<script id="app-root-state" type="application/json">
{
"x": {
"history": [
{"validTimeLocal": "2026-04-05T09:00:00+0800", "metric": {"temp": 24.4}},
{"validTimeLocal": "2026-04-05T13:00:00+0800", "metric": {"temp": 28.6}},
{"validTimeLocal": "2026-04-05T15:00:00+0800", "metric": {"temp": 27.9}}
]
}
}
</script>
</body></html>
"""
result = fetch_wunderground_historical_high(
"taipei",
"2026-04-05",
session=_FakeSession(html),
)
assert result["ok"] is True
assert result["settlement_source"] == "wunderground"
assert result["settlement_station_code"] == "RCSS"
assert result["actual_high"] == 29.0
web/analysis_service.py
+1 -18
View File
@@ -22,7 +22,6 @@ from web.core import (
)
from src.analysis.deb_algorithm import calculate_dynamic_weights
from src.analysis.settlement_rounding import apply_city_settlement
from src.analysis.metar_narrator import describe_metar_report
from src.data_collection.city_registry import ALIASES
from src.models.lgbm_daily_high import predict_lgbm_daily_high
@@ -1166,7 +1165,6 @@ def _analyze(city: str, force_refresh: bool = False) -> Dict[str, Any]:
probability_raw_sigma = None
probability_calibrated_mu = None
probability_calibrated_sigma = None
ai_text = ""
dynamic_commentary = {"summary": "", "notes": []}
try:
_, _ai_context, sd = _trend_analyze(raw, sym, city)
@@ -1196,21 +1194,6 @@ def _analyze(city: str, force_refresh: bool = False) -> Dict[str, Any]:
except Exception as e:
logger.warning(f"Structured analysis skipped for {city}: {e}")
ai_text = describe_metar_report(
raw_metar=str(primary_current.get("raw_metar") or mc.get("raw_metar") or ""),
temp_symbol=sym,
fallback={
"icao": metar.get("icao"),
"station_name": metar.get("station_name"),
"temp": cur_temp,
"wind_speed_kt": _sf(primary_current.get("wind_speed_kt")),
"wind_dir": _sf(primary_current.get("wind_dir")),
"altimeter": _sf(primary_current.get("altimeter")),
"wx_desc": primary_current.get("wx_desc"),
"clouds": primary_current.get("clouds", []) or mc.get("clouds", []),
},
)
# ── 12. Hourly data (today only, for chart) ──
today_hourly: Dict[str, list] = {"times": [], "temps": [], "radiation": []}
for i, ts in enumerate(h_times):
@@ -1562,7 +1545,7 @@ def _analyze(city: str, force_refresh: bool = False) -> Dict[str, Any]:
"metar_today_obs": metar_today_obs_payload,
"metar_recent_obs": metar_recent_obs_payload,
"settlement_today_obs": settlement_today_obs,
"ai_analysis": ai_text,
"ai_analysis": "",
"updated_at": datetime.now(timezone.utc).isoformat(),
}
web/routes.py
+74
View File
@@ -13,6 +13,7 @@ from src.analysis.deb_algorithm import (
load_history,
)
from src.analysis.probability_snapshot_archive import load_snapshot_rows_for_day
from src.analysis.settlement_rounding import apply_city_settlement
from src.data_collection.city_registry import ALIASES
from src.database.runtime_state import TruthRecordRepository
from src.utils.metrics import export_prometheus_metrics
@@ -53,6 +54,9 @@ from web.core import (
router = APIRouter()
_DEB_RECENT_LOOKBACK = 7
_DEB_RECENT_MIN_SAMPLES = 3
TRACKABLE_ANALYTICS_EVENTS = {
"signup_completed",
"dashboard_active",
@@ -174,6 +178,69 @@ def _normalize_city_or_404(name: str) -> str:
return city
def _history_file_path() -> str:
project_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
return os.path.join(project_root, "data", "daily_records.json")
def _build_recent_deb_performance_index(
history_data: Optional[dict] = None,
*,
lookback: int = _DEB_RECENT_LOOKBACK,
min_samples: int = _DEB_RECENT_MIN_SAMPLES,
) -> dict[str, dict[str, object]]:
data = history_data if isinstance(history_data, dict) else load_history(_history_file_path())
index: dict[str, dict[str, object]] = {}
if not isinstance(data, dict):
return index
today = datetime.utcnow().strftime("%Y-%m-%d")
for city_name, rows in data.items():
if not isinstance(rows, dict):
continue
settled: list[tuple[str, float, float]] = []
for date_key in sorted(rows.keys(), reverse=True):
if date_key >= today:
continue
record = rows.get(date_key) or {}
if not isinstance(record, dict):
continue
actual = _sf(record.get("actual_high"))
deb_prediction = _sf(record.get("deb_prediction"))
if actual is None or deb_prediction is None:
continue
settled.append((date_key, actual, deb_prediction))
if len(settled) >= max(lookback, 1):
break
hit_count = 0
abs_errors: list[float] = []
for _, actual, deb_prediction in settled:
abs_errors.append(abs(deb_prediction - actual))
if apply_city_settlement(city_name, actual) == apply_city_settlement(city_name, deb_prediction):
hit_count += 1
sample_count = len(settled)
hit_rate = (hit_count / sample_count) if sample_count > 0 else None
if sample_count < min_samples:
tier = "other"
elif hit_rate is not None and hit_rate >= 0.67:
tier = "high"
elif hit_rate is not None and hit_rate >= 0.34:
tier = "medium"
else:
tier = "low"
index[str(city_name).strip().lower()] = {
"tier": tier,
"sample_count": sample_count,
"hit_rate": round(hit_rate, 4) if hit_rate is not None else None,
"mae": round(sum(abs_errors) / sample_count, 3) if sample_count > 0 else None,
"last_date": settled[0][0] if settled else None,
}
return index
@router.get("/healthz")
async def healthz():
payload = build_health_payload()
@@ -199,9 +266,11 @@ async def metrics():
async def list_cities(request: Request):
try:
out = []
deb_recent_index = _build_recent_deb_performance_index()
for name, info in CITIES.items():
risk = CITY_RISK_PROFILES.get(name, {})
city_meta = CITY_REGISTRY.get(name, {}) or {}
deb_recent = deb_recent_index.get(name, {})
settlement_source = str(info.get("settlement_source") or "metar").strip().lower() or "metar"
out.append(
{
@@ -220,6 +289,11 @@ async def list_cities(request: Request):
settlement_source,
settlement_source.upper(),
),
"deb_recent_tier": deb_recent.get("tier", "other"),
"deb_recent_hit_rate": deb_recent.get("hit_rate"),
"deb_recent_sample_count": deb_recent.get("sample_count", 0),
"deb_recent_mae": deb_recent.get("mae"),
"deb_recent_last_date": deb_recent.get("last_date"),
}
)
return {"cities": out}