gavindiaz/PolyWeather
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

项目体检修复：删除破损的 LGBM 导入、8个死测试、2个死脚本、移除 pytz/lightgbm 依赖

Browse Source
This commit is contained in:
2569718930@qq.com
2026-05-18 22:43:34 +08:00
parent 2b6b18ec20
commit 0e4e7aebb2
12 changed files with 1 additions and 1134 deletions
Download Patch File Download Diff File Expand all files Collapse all files
requirements.txt
-2
View File
@@ -4,9 +4,7 @@ loguru
pyTelegramBotAPI
python-dotenv
netCDF4
pytz
numpy
lightgbm
web3
fastapi
uvicorn
scripts/backfill_daily_record_model_gaps.py
-145
View File
@@ -1,145 +0,0 @@
from __future__ import annotations
import argparse
import json
import sys
from collections import Counter
from pathlib import Path
from typing import Any, Dict
PROJECT_ROOT = Path(__file__).resolve().parents[1]
if str(PROJECT_ROOT) not in sys.path:
sys.path.insert(0, str(PROJECT_ROOT))
from src.analysis.deb_algorithm import load_history, save_history # noqa: E402
from src.analysis.probability_snapshot_archive import ( # noqa: E402
load_snapshot_rows_for_day,
)
from src.database.runtime_state import STATE_STORAGE_FILE, get_state_storage_mode # noqa: E402
from scripts.fit_probability_calibration import _default_history_arg # noqa: E402
def _load_daily_records(path: Path) -> Dict[str, Dict[str, Dict[str, Any]]]:
data = load_history(str(path))
return data if isinstance(data, dict) else {}
def _pick_model_value_from_snapshots(
city: str,
target_date: str,
model_name: str,
) -> float | None:
rows = load_snapshot_rows_for_day(city, target_date)
values = []
for row in rows:
mm = row.get("multi_model") or {}
if not isinstance(mm, dict):
continue
value = mm.get(model_name)
if value is None:
continue
try:
values.append(float(value))
except Exception:
continue
if not values:
return None
counts = Counter(values)
return counts.most_common(1)[0][0]
def main() -> int:
parser = argparse.ArgumentParser(
description="Backfill missing model forecasts in daily_records from archived probability snapshots."
)
parser.add_argument(
"--history-file",
default=_default_history_arg(),
help="Optional legacy daily_records.json path. In sqlite mode this defaults to the runtime database.",
)
parser.add_argument("--city", help="Optional city filter, e.g. ankara")
parser.add_argument("--date", help="Optional YYYY-MM-DD filter")
parser.add_argument(
"--model",
default="MGM",
help="Model name to backfill from snapshot multi_model payloads",
)
parser.add_argument(
"--write",
action="store_true",
help="Write recovered values back to history file / runtime state",
)
args = parser.parse_args()
history_path = Path(args.history_file) if args.history_file else None
data = _load_daily_records(history_path or Path())
model_name = str(args.model or "").strip()
city_filter = str(args.city or "").strip().lower() or None
date_filter = str(args.date or "").strip() or None
recovered = []
missing = []
changed = False
for city, city_rows in sorted(data.items()):
if city_filter and city != city_filter:
continue
if not isinstance(city_rows, dict):
continue
for target_date, record in sorted(city_rows.items()):
if date_filter and target_date != date_filter:
continue
if not isinstance(record, dict):
continue
forecasts = record.get("forecasts") or {}
if not isinstance(forecasts, dict):
forecasts = {}
if forecasts.get(model_name) is not None:
continue
recovered_value = _pick_model_value_from_snapshots(city, target_date, model_name)
if recovered_value is None:
missing.append((city, target_date))
continue
recovered.append((city, target_date, recovered_value))
if args.write:
next_forecasts = dict(forecasts)
next_forecasts[model_name] = recovered_value
record["forecasts"] = next_forecasts
changed = True
print(
json.dumps(
{
"model": model_name,
"recovered_count": len(recovered),
"missing_count": len(missing),
"recovered": [
{"city": city, "date": date_str, "value": value}
for city, date_str, value in recovered
],
"missing": [
{"city": city, "date": date_str}
for city, date_str in missing
],
"write_requested": bool(args.write),
"storage_mode": get_state_storage_mode(),
},
ensure_ascii=False,
indent=2,
)
)
if changed:
previous_mode = get_state_storage_mode()
# Reuse existing save path semantics. In sqlite-only mode, save_history would skip file write.
save_history(str(history_path or ""), data)
if previous_mode == STATE_STORAGE_FILE and (history_path is None or not history_path.exists()):
raise FileNotFoundError(history_path)
return 0
if __name__ == "__main__":
raise SystemExit(main())
scripts/backfill_probability_shadow_history.py
-153
View File
@@ -1,153 +0,0 @@
import argparse
import json
import os
import sys
PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
if PROJECT_ROOT not in sys.path:
sys.path.insert(0, PROJECT_ROOT)
from src.analysis.deb_algorithm import load_history, save_history # noqa: E402
from src.analysis.probability_calibration import ( # noqa: E402
ENGINE_MODE_EMOS_SHADOW,
apply_probability_calibration,
build_probability_features,
)
from scripts.fit_probability_calibration import _default_history_arg # noqa: E402
def _sample_to_features(sample):
peak_flag = sample.get("peak_flag")
if peak_flag == 1.0:
peak_status = "past"
elif peak_flag == 0.5:
peak_status = "in_window"
else:
peak_status = "before"
return build_probability_features(
city_name=sample.get("city") or "",
raw_mu=sample.get("raw_mu"),
raw_sigma=sample.get("raw_sigma"),
deb_prediction=sample.get("deb_prediction"),
ens_data={
"median": sample.get("ens_median"),
"p10": None,
"p90": None,
},
current_forecasts={},
max_so_far=None,
peak_status=peak_status,
local_hour_frac=None,
)
def main():
parser = argparse.ArgumentParser(description="Backfill shadow probability snapshots into daily records.")
parser.add_argument(
"--history-file",
default=_default_history_arg(),
)
parser.add_argument(
"--training-samples",
default=os.path.join(
PROJECT_ROOT,
"artifacts",
"probability_calibration",
"training_samples.json",
),
)
parser.add_argument(
"--calibration-file",
default=os.path.join(
PROJECT_ROOT,
"artifacts",
"probability_calibration",
"default.json",
),
)
args = parser.parse_args()
history = load_history(args.history_file)
with open(args.training_samples, "r", encoding="utf-8") as fh:
training_payload = json.load(fh)
updated = 0
touched = 0
for sample in training_payload.get("samples") or []:
city = str(sample.get("city") or "").strip().lower()
date_str = str(sample.get("date") or "").strip()
if not city or not date_str:
continue
record = ((history.get(city) or {}).get(date_str) or {})
if not isinstance(record, dict) or not record:
continue
legacy_distribution = [
{"value": row.get("v"), "probability": row.get("p")}
for row in (record.get("prob_snapshot") or [])
if isinstance(row, dict) and row.get("v") is not None
]
if not legacy_distribution:
continue
calibration = apply_probability_calibration(
city_name=city,
temp_symbol="°F" if city in {"atlanta", "chicago", "dallas", "miami", "new york", "seattle"} else "°C",
raw_mu=sample.get("raw_mu"),
raw_sigma=sample.get("raw_sigma"),
max_so_far=None,
legacy_distribution=legacy_distribution,
features=_sample_to_features(sample),
calibration_path=args.calibration_file,
mode=ENGINE_MODE_EMOS_SHADOW,
)
shadow_distribution = calibration.get("shadow_distribution") or []
compact_shadow = [
{
"v": int(row.get("value")),
"p": round(float(row.get("probability") or 0.0), 3),
}
for row in shadow_distribution[:4]
if row.get("value") is not None
]
compact_calibration = {
"mode": calibration.get("mode"),
"engine": calibration.get("engine"),
"version": calibration.get("calibration_version"),
"source": calibration.get("calibration_source"),
"raw_mu": calibration.get("raw_mu"),
"raw_sigma": calibration.get("raw_sigma"),
"calibrated_mu": calibration.get("calibrated_mu"),
"calibrated_sigma": calibration.get("calibrated_sigma"),
}
touched += 1
if (
record.get("shadow_prob_snapshot") == compact_shadow
and record.get("probability_calibration") == compact_calibration
):
continue
record["shadow_prob_snapshot"] = compact_shadow
record["probability_calibration"] = compact_calibration
history[city][date_str] = record
updated += 1
save_history(args.history_file, history)
print(
json.dumps(
{
"samples_seen": len(training_payload.get("samples") or []),
"records_considered": touched,
"records_updated": updated,
},
ensure_ascii=False,
indent=2,
)
)
if __name__ == "__main__":
main()
src/analysis/trend_engine.py
+1 -27
View File
@@ -18,7 +18,6 @@ from src.analysis.deb_algorithm import (
from src.analysis.settlement_rounding import apply_city_settlement, is_exact_settlement_city
from src.data_collection.city_registry import CITY_REGISTRY
from src.data_collection.city_risk_profiles import get_city_risk_profile
from src.models.lgbm_daily_high import predict_lgbm_daily_high
SETTLEMENT_SOURCE_LABELS = {
"metar": "METAR",
@@ -415,32 +414,7 @@ def analyze_weather_trend(
peak_status = "before"
if city_name and current_forecasts and deb_prediction is not None:
lgbm_prediction, _ = predict_lgbm_daily_high(
city_name=city_name,
current_forecasts=current_forecasts,
deb_prediction=deb_prediction,
current_temp=cur_temp,
max_so_far=max_so_far,
humidity=_sf(primary_current.get("humidity")),
wind_speed_kt=_sf(primary_current.get("wind_speed_kt")),
visibility_mi=_sf(primary_current.get("visibility_mi")),
local_hour=local_hour,
local_date=local_date_str,
peak_status=peak_status,
)
if lgbm_prediction is not None:
current_forecasts["LGBM"] = lgbm_prediction
blended_high, weight_info = calculate_dynamic_weights(
city_name, current_forecasts
)
if blended_high is not None:
deb_prediction = blended_high
deb_weights = weight_info
_deb_to_save = blended_high
if insights and "DEB 融合预测" in insights[0]:
insights[0] = (
f"🧬 <b>DEB 融合预测</b>：<b>{blended_high}{temp_symbol}</b> ({weight_info})"
)
# DEB blending uses the already-computed set of model forecasts
if ai_features and "DEB系统已通过历史偏差矫正算出期待点是" in ai_features[0]:
ai_features[0] = (
f"🧬 DEB系统已通过历史偏差矫正算出期待点是: {blended_high}{temp_symbol}。"
tests/test_auto_retrain_probability_calibration.py
-42
View File
@@ -1,42 +0,0 @@
from scripts.auto_retrain_probability_calibration import judge_candidate
def _report(sample_count=80, crps=-0.1, mae=0.0, hit=0.0):
return {
"summary": {
"sample_count": sample_count,
"delta": {
"crps": crps,
"mae": mae,
"bucket_hit_rate": hit,
},
}
}
def test_candidate_gate_promotes_when_metrics_pass():
decision = judge_candidate(
_report(),
min_samples=50,
max_delta_crps=0.0,
max_delta_mae=0.05,
min_delta_bucket_hit_rate=-0.05,
)
assert decision["decision"] == "promote"
assert decision["ready_for_promotion"] is True
assert decision["blocking_reasons"] == []
def test_candidate_gate_holds_when_metrics_regress():
decision = judge_candidate(
_report(sample_count=40, crps=0.1, mae=0.2, hit=-0.2),
min_samples=50,
max_delta_crps=0.0,
max_delta_mae=0.05,
min_delta_bucket_hit_rate=-0.05,
)
assert decision["decision"] == "hold"
assert decision["ready_for_promotion"] is False
assert len(decision["blocking_reasons"]) == 4
tests/test_lgbm_daily_high.py
-93
View File
@@ -1,93 +0,0 @@
import src.models.lgbm_daily_high as runtime
class _FakeBooster:
best_iteration = 7
def predict(self, rows, num_iteration=None):
assert len(rows) == 1
return [14.36]
def test_predict_lgbm_daily_high_skips_when_disabled(monkeypatch):
monkeypatch.setenv("POLYWEATHER_LGBM_ENABLED", "false")
prediction, meta = runtime.predict_lgbm_daily_high(
city_name="ankara",
current_forecasts={"Open-Meteo": 12.4},
deb_prediction=12.3,
current_temp=11.0,
max_so_far=11.4,
humidity=62.0,
wind_speed_kt=8.0,
visibility_mi=6.0,
local_hour=10,
local_date="2026-03-24",
peak_status="before",
history_data={},
)
assert prediction is None
assert meta["reason"] == "disabled"
def test_predict_lgbm_daily_high_returns_prediction(monkeypatch):
monkeypatch.setenv("POLYWEATHER_LGBM_ENABLED", "true")
monkeypatch.setenv("POLYWEATHER_LGBM_MIN_HISTORY_POINTS", "3")
monkeypatch.setattr(runtime, "_load_schema", lambda path: {"feature_names": runtime.FEATURE_NAMES})
monkeypatch.setattr(runtime, "_load_booster", lambda path: _FakeBooster())
history_data = {
"ankara": {
"2026-03-20": {"actual_high": 10.0},
"2026-03-21": {"actual_high": 11.0},
"2026-03-22": {"actual_high": 13.0},
"2026-03-23": {"actual_high": 12.0},
}
}
prediction, meta = runtime.predict_lgbm_daily_high(
city_name="ankara",
current_forecasts={"Open-Meteo": 12.4, "ECMWF": 12.1, "GFS": 11.9},
deb_prediction=12.3,
current_temp=11.0,
max_so_far=11.4,
humidity=62.0,
wind_speed_kt=8.0,
visibility_mi=6.0,
local_hour=10,
local_date="2026-03-24",
peak_status="before",
history_data=history_data,
)
assert prediction == 14.4
assert meta["reason"] == "ok"
assert meta["history_count"] == 4
def test_predict_lgbm_daily_high_requires_min_history(monkeypatch):
monkeypatch.setenv("POLYWEATHER_LGBM_ENABLED", "true")
monkeypatch.setenv("POLYWEATHER_LGBM_MIN_HISTORY_POINTS", "5")
monkeypatch.setattr(runtime, "_load_schema", lambda path: {"feature_names": runtime.FEATURE_NAMES})
monkeypatch.setattr(runtime, "_load_booster", lambda path: _FakeBooster())
history_data = {
"ankara": {
"2026-03-21": {"actual_high": 11.0},
"2026-03-22": {"actual_high": 13.0},
"2026-03-23": {"actual_high": 12.0},
}
}
prediction, meta = runtime.predict_lgbm_daily_high(
city_name="ankara",
current_forecasts={"Open-Meteo": 12.4},
deb_prediction=12.3,
current_temp=11.0,
max_so_far=11.4,
humidity=62.0,
wind_speed_kt=8.0,
visibility_mi=6.0,
local_hour=10,
local_date="2026-03-24",
peak_status="before",
history_data=history_data,
)
assert prediction is None
assert meta["reason"] == "insufficient_history"
tests/test_lgbm_features.py
-93
View File
@@ -1,93 +0,0 @@
from src.models.lgbm_features import build_runtime_feature_map, build_training_samples
def test_build_runtime_feature_map_derives_history_and_model_summary():
history_data = {
"ankara": {
"2026-03-20": {"actual_high": 10.0},
"2026-03-21": {"actual_high": 11.0},
"2026-03-22": {"actual_high": 13.0},
"2026-03-23": {"actual_high": 12.0},
}
}
feature_map, meta = build_runtime_feature_map(
city_name="ankara",
current_forecasts={
"Open-Meteo": 12.4,
"ECMWF": 12.1,
"GFS": 11.9,
"GEM": 12.8,
},
deb_prediction=12.3,
current_temp=11.0,
max_so_far=11.4,
humidity=62.0,
wind_speed_kt=8.0,
visibility_mi=6.0,
local_hour=10,
local_date="2026-03-24",
peak_status="before",
history_data=history_data,
)
assert meta["reason"] == "ok"
assert meta["history_count"] == 4
assert feature_map["actual_high_lag_1"] == 12.0
assert feature_map["actual_high_lag_2"] == 13.0
assert feature_map["actual_high_trend_3"] == 1.0
assert feature_map["model_median"] == 12.4
assert round(feature_map["model_spread"], 3) == 0.9
assert feature_map["peak_status_code"] == 0.0
def test_build_runtime_feature_map_returns_none_without_history():
feature_map, meta = build_runtime_feature_map(
city_name="unknown-city",
current_forecasts={"Open-Meteo": 12.4},
deb_prediction=12.3,
current_temp=11.0,
max_so_far=11.4,
humidity=62.0,
wind_speed_kt=8.0,
visibility_mi=6.0,
local_hour=10,
local_date="2026-03-24",
peak_status="before",
history_data={},
)
assert feature_map is None
assert meta["reason"] == "no_history"
def test_build_training_samples_prefers_truth_history_for_target():
history_data = {
"ankara": {
"2026-03-20": {"actual_high": 10.0},
"2026-03-21": {"actual_high": 11.0},
"2026-03-22": {"actual_high": 13.0},
"2026-03-23": {
"actual_high": 12.0,
"deb_prediction": 12.3,
"forecasts": {"Open-Meteo": 12.4, "ECMWF": 12.1},
},
}
}
snapshot_index = {
("ankara", "2026-03-23"): {
"city": "ankara",
"date": "2026-03-23",
"timestamp": "2026-03-23T10:00:00+03:00",
"raw_mu": 12.2,
"deb_prediction": 12.3,
"max_so_far": 11.8,
"peak_status": "before",
"multi_model": {"Open-Meteo": 12.4, "ECMWF": 12.1},
"observation": {"current_temp": 11.5, "humidity": 60.0, "wind_speed_kt": 8.0, "local_hour": 10},
}
}
samples = build_training_samples(history_data=history_data, snapshot_index=snapshot_index)
assert len(samples) == 1
assert samples[0]["sample_source"] == "snapshot"
tests/test_probability_calibration.py
-209
View File
@@ -1,209 +0,0 @@
import json
from pathlib import Path
from src.analysis.probability_calibration import (
ENGINE_MODE_EMOS_PRIMARY,
ENGINE_MODE_EMOS_SHADOW,
ENGINE_MODE_LEGACY,
apply_probability_calibration,
build_probability_features,
fit_calibration,
resolve_probability_engine_mode,
)
def _write_calibration(tmp_path: Path):
payload = {
"version": "test-emos-v1",
"source": "tmp/test-emos-v1.json",
"global": {
"mu": {
"intercept": 0.0,
"raw_mu_coef": 0.0,
"deb_coef": 1.0,
"ens_median_coef": 0.0,
"max_so_far_gap_coef": 0.0,
},
"sigma": {
"intercept": 0.0,
"raw_sigma_coef": 1.0,
"spread_coef": 0.0,
"peak_flag_coef": 0.0,
"max_so_far_gap_coef": 0.0,
},
},
"sigma_constraints": {
"min_ratio": 0.85,
"max_ratio": 1.2,
"absolute_min": 0.25,
"absolute_max": 2.0,
},
"cities": {
"ankara": {
"mu_bias": 0.5,
"sigma_scale": 2.0,
"confidence": 1.0,
}
},
"metrics": {"sample_count": 10, "mean_crps": 0.4},
}
path = tmp_path / "calibration.json"
path.write_text(json.dumps(payload), encoding="utf-8")
return path
def test_default_probability_engine_is_emos_primary(monkeypatch):
monkeypatch.delenv("POLYWEATHER_PROBABILITY_ENGINE", raising=False)
assert resolve_probability_engine_mode() == ENGINE_MODE_EMOS_PRIMARY
assert resolve_probability_engine_mode("unknown-mode") == ENGINE_MODE_EMOS_PRIMARY
monkeypatch.setenv("POLYWEATHER_PROBABILITY_ENGINE", ENGINE_MODE_EMOS_SHADOW)
assert resolve_probability_engine_mode() == ENGINE_MODE_EMOS_SHADOW
def test_shadow_mode_keeps_legacy_distribution(tmp_path):
calibration_path = _write_calibration(tmp_path)
features = build_probability_features(
city_name="ankara",
raw_mu=9.0,
raw_sigma=1.0,
deb_prediction=10.0,
ens_data={"median": 9.5, "p10": 8.0, "p90": 11.0},
current_forecasts={"Open-Meteo": 9.0, "MGM": 10.0},
max_so_far=8.8,
peak_status="before",
local_hour_frac=11.0,
)
legacy_distribution = [{"value": 9, "range": "[8.5~9.5)", "probability": 0.7}]
result = apply_probability_calibration(
city_name="ankara",
temp_symbol="°C",
raw_mu=9.0,
raw_sigma=1.0,
max_so_far=8.8,
legacy_distribution=legacy_distribution,
features=features,
calibration_path=str(calibration_path),
mode=ENGINE_MODE_EMOS_SHADOW,
)
assert result["mode"] == ENGINE_MODE_EMOS_SHADOW
assert result["engine"] == ENGINE_MODE_LEGACY
assert result["distribution"] == legacy_distribution
assert result["shadow_distribution"]
assert result["calibrated_mu"] == 10.5
assert result["calibrated_sigma"] == 1.2
assert len(result["shadow_distribution_all"]) >= len(result["shadow_distribution"])
def test_primary_mode_switches_to_calibrated_distribution(tmp_path):
calibration_path = _write_calibration(tmp_path)
features = build_probability_features(
city_name="ankara",
raw_mu=9.0,
raw_sigma=1.0,
deb_prediction=10.0,
ens_data={"median": 9.5, "p10": 8.0, "p90": 11.0},
current_forecasts={"Open-Meteo": 9.0, "MGM": 10.0},
max_so_far=8.8,
peak_status="before",
local_hour_frac=11.0,
)
result = apply_probability_calibration(
city_name="ankara",
temp_symbol="°C",
raw_mu=9.0,
raw_sigma=1.0,
max_so_far=8.8,
legacy_distribution=[{"value": 9, "range": "[8.5~9.5)", "probability": 0.7}],
features=features,
calibration_path=str(calibration_path),
mode=ENGINE_MODE_EMOS_PRIMARY,
)
assert result["mode"] == ENGINE_MODE_EMOS_PRIMARY
assert result["engine"] == "emos"
assert result["calibrated_mu"] == 10.5
assert result["calibrated_sigma"] == 1.2
assert result["distribution"]
assert len(result["distribution_all"]) >= len(result["distribution"])
assert result["distribution"][0]["value"] >= 10
def test_primary_mode_respects_observed_max_floor(tmp_path):
calibration_path = _write_calibration(tmp_path)
features = build_probability_features(
city_name="ankara",
raw_mu=32.0,
raw_sigma=1.0,
deb_prediction=32.0,
ens_data={"median": 31.5, "p10": 30.0, "p90": 34.0},
current_forecasts={"Open-Meteo": 32.0, "MGM": 31.8},
max_so_far=33.0,
peak_status="in_window",
local_hour_frac=14.0,
)
result = apply_probability_calibration(
city_name="ankara",
temp_symbol="°C",
raw_mu=32.0,
raw_sigma=1.0,
max_so_far=33.0,
legacy_distribution=[{"value": 33, "range": "[32.5~33.5)", "probability": 0.7}],
features=features,
calibration_path=str(calibration_path),
mode=ENGINE_MODE_EMOS_PRIMARY,
)
assert result["engine"] == "emos"
assert result["calibrated_mu"] >= 33.0
assert all(row["value"] >= 33 for row in result["distribution"])
def test_fit_calibration_returns_metrics():
samples = [
{
"city": "ankara",
"actual_high": 11.0,
"raw_mu": 10.2,
"raw_sigma": 1.0,
"deb_prediction": 10.5,
"ens_median": 10.6,
"ensemble_spread": 0.9,
"max_so_far_gap": 0.5,
"peak_flag": 0.0,
},
{
"city": "ankara",
"actual_high": 12.0,
"raw_mu": 11.1,
"raw_sigma": 1.0,
"deb_prediction": 11.3,
"ens_median": 11.2,
"ensemble_spread": 1.0,
"max_so_far_gap": 0.4,
"peak_flag": 0.5,
},
{
"city": "new york",
"actual_high": 19.0,
"raw_mu": 18.2,
"raw_sigma": 1.4,
"deb_prediction": 18.4,
"ens_median": 18.3,
"ensemble_spread": 1.2,
"max_so_far_gap": 0.6,
"peak_flag": 1.0,
},
]
result = fit_calibration(samples, version="unit-test-v1")
assert result["version"] == "unit-test-v1"
assert result["metrics"]["sample_count"] == 3
assert "mean_crps" in result["metrics"]
tests/test_probability_rollout.py
-65
View File
@@ -1,65 +0,0 @@
from src.analysis.probability_rollout import judge_probability_rollout
def test_judge_probability_rollout_holds_on_shadow_brier_regression():
evaluation_report = {
"summary": {
"sample_count": 105,
"delta": {
"crps": -0.09,
"mae": 0.0,
"bucket_hit_rate": 0.0,
},
}
}
shadow_report = {
"summary": {
"samples": 103,
"delta_mae": 0.01,
"delta_bucket_hit_rate": 0.01,
"delta_bucket_brier": 0.29,
},
"by_city": {
"miami": {
"samples": 4,
"delta_mae": 0.24,
"delta_bucket_hit_rate": -0.5,
"delta_bucket_brier": 0.47,
}
},
}
payload = judge_probability_rollout(evaluation_report, shadow_report)
assert payload["decision"] == "hold"
assert payload["ready_for_primary"] is False
assert payload["blocking_reasons"]
assert payload["worst_shadow_regressions"][0]["city"] == "miami"
def test_judge_probability_rollout_promotes_on_clean_metrics():
evaluation_report = {
"summary": {
"sample_count": 120,
"delta": {
"crps": -0.08,
"mae": 0.0,
"bucket_hit_rate": 0.02,
},
}
}
shadow_report = {
"summary": {
"samples": 110,
"delta_mae": 0.0,
"delta_bucket_hit_rate": 0.01,
"delta_bucket_brier": 0.01,
},
"by_city": {},
}
payload = judge_probability_rollout(evaluation_report, shadow_report)
assert payload["decision"] == "promote"
assert payload["ready_for_primary"] is True
assert payload["blocking_reasons"] == []
tests/test_probability_shadow_report.py
-61
View File
@@ -1,61 +0,0 @@
import json
from pathlib import Path
from scripts.build_probability_shadow_report import main
def test_shadow_report_builds_summary(monkeypatch, tmp_path: Path):
history_file = tmp_path / "daily_records.json"
output_file = tmp_path / "shadow_report.json"
history_file.write_text(
json.dumps(
{
"ankara": {
"2026-03-18": {
"actual_high": 10.0,
"mu": 9.6,
"prob_snapshot": [
{"v": 10, "p": 0.48},
{"v": 9, "p": 0.41},
],
"shadow_prob_snapshot": [
{"v": 10, "p": 0.55},
{"v": 9, "p": 0.28},
],
"probability_calibration": {
"mode": "emos_shadow",
"engine": "legacy",
"version": "emos-test",
"raw_mu": 9.6,
"calibrated_mu": 9.9,
},
}
}
},
ensure_ascii=False,
indent=2,
),
encoding="utf-8",
)
monkeypatch.setattr(
"sys.argv",
[
"build_probability_shadow_report.py",
"--history-file",
str(history_file),
"--output",
str(output_file),
],
)
main()
payload = json.loads(output_file.read_text(encoding="utf-8"))
assert payload["summary"]["samples"] == 1
assert payload["summary"]["legacy_mean_mae"] == 0.4
assert payload["summary"]["shadow_mean_mae"] == 0.1
assert payload["summary"]["delta_mae"] == -0.3
assert payload["summary"]["legacy_bucket_hit_rate"] == 1.0
assert payload["summary"]["shadow_bucket_hit_rate"] == 1.0
assert payload["recent_observations"][0]["calibration_version"] == "emos-test"
tests/test_probability_snapshot_archive.py
-183
View File
@@ -1,183 +0,0 @@
import json
from pathlib import Path
import pytest
import src.analysis.probability_snapshot_archive as snapshot_archive
from src.database.runtime_state import RuntimeStateDB, TrainingFeatureRecordRepository
@pytest.fixture(autouse=True)
def _force_file_mode(monkeypatch):
monkeypatch.setenv("POLYWEATHER_STATE_STORAGE_MODE", "file")
def test_append_probability_snapshot_writes_jsonl(tmp_path: Path, monkeypatch):
archive_path = tmp_path / "probability_training_snapshots.jsonl"
db = RuntimeStateDB(str(tmp_path / "polyweather.db"))
monkeypatch.setattr(
snapshot_archive,
"_training_feature_repo",
TrainingFeatureRecordRepository(db),
)
monkeypatch.setattr(
snapshot_archive,
"_snapshot_repo",
snapshot_archive.ProbabilitySnapshotRepository(db),
)
snapshot_archive.append_probability_snapshot(
city_name="ankara",
local_date="2026-03-20",
observation_time="2026-03-20T12:00:00+03:00",
temp_symbol="°C",
raw_mu=15.2,
raw_sigma=1.2,
deb_prediction=15.4,
ens_data={"p10": 14.8, "median": 15.8, "p90": 17.9},
current_forecasts={"ECMWF": 15.8, "GFS": 14.1},
max_so_far=15.0,
peak_status="before",
probabilities=[{"value": 15, "probability": 0.552}],
shadow_probabilities=[{"value": 15, "probability": 0.324}],
calibration_summary={
"engine": "legacy",
"mode": "emos_shadow",
"calibration_version": "emos-test",
"calibration_source": "artifacts/probability_calibration/default.json",
"calibrated_mu": 15.1,
"calibrated_sigma": 1.25,
},
archive_path=str(archive_path),
)
lines = archive_path.read_text(encoding="utf-8").strip().splitlines()
assert len(lines) == 1
payload = json.loads(lines[0])
assert payload["city"] == "ankara"
assert payload["date"] == "2026-03-20"
assert payload["raw_mu"] == 15.2
assert payload["ensemble"]["median"] == 15.8
assert payload["prob_snapshot"][0]["v"] == 15
assert payload["shadow_prob_snapshot"][0]["v"] == 15
assert payload["calibration_version"] == "emos-test"
def test_append_probability_snapshot_skips_near_duplicate(tmp_path: Path):
archive_path = tmp_path / "probability_training_snapshots.jsonl"
kwargs = dict(
city_name="ankara",
local_date="2026-03-20",
observation_time="2026-03-20T12:00:00+03:00",
temp_symbol="°C",
raw_mu=15.2,
raw_sigma=1.2,
deb_prediction=15.4,
ens_data={"p10": 14.8, "median": 15.8, "p90": 17.9},
current_forecasts={"ECMWF": 15.8, "GFS": 14.1},
max_so_far=15.0,
peak_status="before",
probabilities=[{"value": 15, "probability": 0.552}],
shadow_probabilities=[{"value": 15, "probability": 0.324}],
calibration_summary={
"engine": "legacy",
"mode": "emos_shadow",
"calibration_version": "emos-test",
"calibration_source": "artifacts/probability_calibration/default.json",
"calibrated_mu": 15.1,
"calibrated_sigma": 1.25,
},
archive_path=str(archive_path),
)
snapshot_archive.append_probability_snapshot(**kwargs)
snapshot_archive.append_probability_snapshot(**kwargs)
lines = archive_path.read_text(encoding="utf-8").strip().splitlines()
assert len(lines) == 1
def test_append_probability_snapshot_writes_on_bucket_change(tmp_path: Path):
archive_path = tmp_path / "probability_training_snapshots.jsonl"
base_kwargs = dict(
city_name="ankara",
local_date="2026-03-20",
observation_time="2026-03-20T12:00:00+03:00",
temp_symbol="°C",
raw_mu=15.2,
raw_sigma=1.2,
deb_prediction=15.4,
ens_data={"p10": 14.8, "median": 15.8, "p90": 17.9},
current_forecasts={"ECMWF": 15.8, "GFS": 14.1},
max_so_far=15.0,
peak_status="before",
shadow_probabilities=[{"value": 15, "probability": 0.324}],
calibration_summary={
"engine": "legacy",
"mode": "emos_shadow",
"calibration_version": "emos-test",
"calibration_source": "artifacts/probability_calibration/default.json",
"calibrated_mu": 15.1,
"calibrated_sigma": 1.25,
},
archive_path=str(archive_path),
)
snapshot_archive.append_probability_snapshot(
probabilities=[{"value": 15, "probability": 0.552}],
**base_kwargs,
)
snapshot_archive.append_probability_snapshot(
probabilities=[{"value": 16, "probability": 0.552}],
**base_kwargs,
)
lines = archive_path.read_text(encoding="utf-8").strip().splitlines()
assert len(lines) == 2
def test_append_probability_snapshot_dual_writes_training_feature_store(tmp_path: Path, monkeypatch):
monkeypatch.setenv("POLYWEATHER_STATE_STORAGE_MODE", "sqlite")
monkeypatch.setenv("POLYWEATHER_DB_PATH", str(tmp_path / "polyweather.db"))
db = RuntimeStateDB(str(tmp_path / "polyweather.db"))
monkeypatch.setattr(
snapshot_archive,
"_training_feature_repo",
TrainingFeatureRecordRepository(db),
)
monkeypatch.setattr(
snapshot_archive,
"_snapshot_repo",
snapshot_archive.ProbabilitySnapshotRepository(db),
)
snapshot_archive.append_probability_snapshot(
city_name="ankara",
local_date="2026-03-20",
observation_time="2026-03-20T12:00:00+03:00",
temp_symbol="°C",
raw_mu=15.2,
raw_sigma=1.2,
deb_prediction=15.4,
ens_data={"p10": 14.8, "median": 15.8, "p90": 17.9},
current_forecasts={"ECMWF": 15.8, "GFS": 14.1},
max_so_far=15.0,
peak_status="before",
probabilities=[{"value": 15, "probability": 0.552}],
shadow_probabilities=[{"value": 15, "probability": 0.324}],
calibration_summary={
"engine": "legacy",
"mode": "emos_shadow",
"calibration_version": "emos-test",
"calibration_source": "artifacts/probability_calibration/default.json",
"calibrated_mu": 15.1,
"calibrated_sigma": 1.25,
},
)
payload = TrainingFeatureRecordRepository(db).get_record("ankara", "2026-03-20")
assert payload is not None
assert payload["mu"] == 15.2
assert payload["forecasts"]["ECMWF"] == 15.8
assert payload["probability_features"]["ens_median"] == 15.8
tests/test_probability_training_dataset.py
-61
View File
@@ -1,61 +0,0 @@
from scripts.fit_probability_calibration import _extract_samples
def test_extract_samples_prefers_snapshot_rows_for_same_city_day():
history = {
"ankara": {
"2026-03-19": {
"actual_high": 11.0,
"mu": 10.8,
"deb_prediction": 10.9,
"forecasts": {"ECMWF": 10.5, "GFS": 11.2},
"probability_features": {
"ens_median": 10.7,
"ensemble_spread": 0.8,
"peak_status": "before",
},
}
}
}
snapshot_rows = [
{
"city": "ankara",
"date": "2026-03-19",
"timestamp": "2026-03-19T12:00:00+03:00",
"raw_mu": 11.2,
"raw_sigma": 1.1,
"deb_prediction": 11.0,
"ensemble": {"p10": 10.0, "median": 11.1, "p90": 12.2},
"multi_model": {"ECMWF": 10.5, "GFS": 11.2},
"max_so_far": 10.9,
"peak_status": "in_window",
}
]
truth_history = {
"ankara": {
"2026-03-19": {
"actual_high": 11.0,
"settlement_source": "metar",
"settlement_station_code": "LTAC",
"truth_version": "v1",
"updated_by": "test",
"truth_updated_at": 123.0,
}
}
}
samples, filled = _extract_samples(
history,
truth_history=truth_history,
settlement_history={},
snapshot_rows=snapshot_rows,
)
assert filled == 0
assert len(samples) == 1
assert samples[0]["sample_source"] == "snapshot"
assert samples[0]["raw_mu"] == 11.2
assert samples[0]["peak_flag"] == 0.5
assert samples[0]["settlement_source"] == "metar"
assert samples[0]["settlement_station_code"] == "LTAC"
assert samples[0]["truth_version"] == "v1"