Unify runtime state in SQLite and add rollout observability

src/analysis/deb_algorithm.py

@@ -3,6 +3,13 @@ import json
 from datetime import datetime, timedelta
 import requests
 from src.analysis.settlement_rounding import apply_city_settlement
+from loguru import logger
+from src.database.runtime_state import (
+    DailyRecordRepository,
+    STATE_STORAGE_DUAL,
+    STATE_STORAGE_SQLITE,
+    get_state_storage_mode,
+)
 
 # Cross-platform file locking
 import sys
@@ -36,6 +43,7 @@ else:
 # Simple memory cache to avoid blasting the disk if queried 10 times a minute
 _history_cache = {}
 _history_mtime = 0
+_daily_record_repo = DailyRecordRepository()
 
 
 def _sf(value):
@@ -54,7 +62,24 @@ def _is_excluded_model_name(model_name: str) -> bool:
 
 def load_history(filepath):
     global _history_cache, _history_mtime
+    mode = get_state_storage_mode()
+
+    if mode == STATE_STORAGE_SQLITE:
+        try:
+            data = _daily_record_repo.load_all()
+            _history_cache = data
+            return data
+        except Exception as e:
+            logger.error(f"Error loading daily records from sqlite, fallback to file: {e}")
+
     if not os.path.exists(filepath):
+        if mode == STATE_STORAGE_DUAL:
+            try:
+                data = _daily_record_repo.load_all()
+                _history_cache = data
+                return data
+            except Exception:
+                return {}
         return {}
 
     try:
@@ -80,6 +105,18 @@ def load_history(filepath):
 def save_history(filepath, data):
     global _history_cache, _history_mtime
     _history_cache = data
+    mode = get_state_storage_mode()
+
+    if mode in {STATE_STORAGE_DUAL, STATE_STORAGE_SQLITE}:
+        try:
+            _daily_record_repo.replace_all(data)
+        except Exception as e:
+            logger.error(f"Error saving daily records to sqlite: {e}")
+            if mode == STATE_STORAGE_SQLITE:
+                return
+
+    if mode == STATE_STORAGE_SQLITE:
+        return
     try:
         with open(filepath, "w", encoding="utf-8") as f:
             _lock_ex(f)
@@ -251,6 +288,7 @@ def update_daily_record(
     )
     history_file = os.path.join(project_root, "data", "daily_records.json")
 
+    mode = get_state_storage_mode()
     data = load_history(history_file)
     if city_name not in data:
         data[city_name] = {}
@@ -359,7 +397,18 @@ def update_daily_record(
         for d in old_dates:
             del data[city][d]
 
-    save_history(history_file, data)
+    if mode in {STATE_STORAGE_DUAL, STATE_STORAGE_SQLITE}:
+        try:
+            _daily_record_repo.upsert_record(city_name, date_str, existing)
+            cutoff = (datetime.now() - timedelta(days=14)).strftime("%Y-%m-%d")
+            _daily_record_repo.delete_older_than(cutoff)
+        except Exception as e:
+            logger.error(f"Error upserting daily record to sqlite city={city_name} date={date_str}: {e}")
+            if mode == STATE_STORAGE_SQLITE:
+                raise
+
+    if mode != STATE_STORAGE_SQLITE:
+        save_history(history_file, data)
 
 
 def calculate_dynamic_weights(city_name, current_forecasts, lookback_days=7):

src/analysis/probability_rollout.py

@@ -0,0 +1,217 @@
+from __future__ import annotations
+
+import json
+import os
+from typing import Any, Dict, List, Optional
+
+
+def _load_json_file(path: str) -> Dict[str, Any]:
+    try:
+        with open(path, "r", encoding="utf-8") as fh:
+            data = json.load(fh)
+        return data if isinstance(data, dict) else {}
+    except Exception:
+        return {}
+
+
+def _sf(value: Any) -> Optional[float]:
+    if value is None:
+        return None
+    try:
+        return float(value)
+    except Exception:
+        return None
+
+
+def _append_reason(reasons: List[str], condition: bool, message: str) -> None:
+    if condition:
+        reasons.append(message)
+
+
+def _top_shadow_regressions(by_city: Dict[str, Any], limit: int = 5) -> List[Dict[str, Any]]:
+    rows: List[Dict[str, Any]] = []
+    for city, metrics in (by_city or {}).items():
+        if not isinstance(metrics, dict):
+            continue
+        rows.append(
+            {
+                "city": city,
+                "samples": int(metrics.get("samples") or 0),
+                "delta_mae": _sf(metrics.get("delta_mae")),
+                "delta_bucket_hit_rate": _sf(metrics.get("delta_bucket_hit_rate")),
+                "delta_bucket_brier": _sf(metrics.get("delta_bucket_brier")),
+            }
+        )
+
+    rows.sort(
+        key=lambda row: (
+            -(row["delta_bucket_brier"] or 0.0),
+            row["delta_bucket_hit_rate"] or 0.0,
+            -(row["delta_mae"] or 0.0),
+        )
+    )
+    return rows[:limit]
+
+
+def judge_probability_rollout(
+    evaluation_report: Dict[str, Any],
+    shadow_report: Dict[str, Any],
+) -> Dict[str, Any]:
+    thresholds = {
+        "evaluation_min_samples": 80,
+        "shadow_min_samples": 50,
+        "max_delta_mae": 0.05,
+        "min_delta_crps": -0.02,
+        "min_delta_bucket_hit_rate": 0.0,
+        "max_delta_bucket_brier_promote": 0.02,
+        "max_delta_bucket_brier_observe": 0.15,
+    }
+
+    eval_summary = (evaluation_report or {}).get("summary") or {}
+    eval_delta = eval_summary.get("delta") or {}
+    shadow_summary = (shadow_report or {}).get("summary") or {}
+
+    eval_samples = int(eval_summary.get("sample_count") or 0)
+    shadow_samples = int(shadow_summary.get("samples") or 0)
+    delta_crps = _sf(eval_delta.get("crps"))
+    delta_mae = _sf(eval_delta.get("mae"))
+    delta_hit = _sf(eval_delta.get("bucket_hit_rate"))
+    shadow_delta_mae = _sf(shadow_summary.get("delta_mae"))
+    shadow_delta_hit = _sf(shadow_summary.get("delta_bucket_hit_rate"))
+    shadow_delta_brier = _sf(shadow_summary.get("delta_bucket_brier"))
+
+    promote_reasons: List[str] = []
+    _append_reason(
+        promote_reasons,
+        eval_samples < thresholds["evaluation_min_samples"],
+        f"离线评估样本不足：{eval_samples} < {thresholds['evaluation_min_samples']}",
+    )
+    _append_reason(
+        promote_reasons,
+        shadow_samples < thresholds["shadow_min_samples"],
+        f"shadow 样本不足：{shadow_samples} < {thresholds['shadow_min_samples']}",
+    )
+    _append_reason(
+        promote_reasons,
+        delta_crps is None or delta_crps > thresholds["min_delta_crps"],
+        f"离线 CRPS 改善不足：delta={delta_crps}",
+    )
+    _append_reason(
+        promote_reasons,
+        delta_mae is None or delta_mae > thresholds["max_delta_mae"],
+        f"离线 MAE 退化超限：delta={delta_mae}",
+    )
+    _append_reason(
+        promote_reasons,
+        delta_hit is None or delta_hit < thresholds["min_delta_bucket_hit_rate"],
+        f"离线 bucket 命中率下降：delta={delta_hit}",
+    )
+    _append_reason(
+        promote_reasons,
+        shadow_delta_mae is None or shadow_delta_mae > thresholds["max_delta_mae"],
+        f"shadow MAE 退化超限：delta={shadow_delta_mae}",
+    )
+    _append_reason(
+        promote_reasons,
+        shadow_delta_hit is None or shadow_delta_hit < thresholds["min_delta_bucket_hit_rate"],
+        f"shadow bucket 命中率下降：delta={shadow_delta_hit}",
+    )
+    _append_reason(
+        promote_reasons,
+        shadow_delta_brier is None
+        or shadow_delta_brier > thresholds["max_delta_bucket_brier_promote"],
+        f"shadow bucket brier 退化超限：delta={shadow_delta_brier}",
+    )
+
+    if not promote_reasons:
+        decision = "promote"
+        summary = "离线与 shadow 指标均达标，可以考虑切换 emos_primary。"
+    else:
+        observe_reasons: List[str] = []
+        _append_reason(
+            observe_reasons,
+            eval_samples < thresholds["evaluation_min_samples"],
+            f"离线评估样本不足：{eval_samples}",
+        )
+        _append_reason(
+            observe_reasons,
+            delta_crps is None or delta_crps > thresholds["min_delta_crps"],
+            f"离线 CRPS 改善不足：delta={delta_crps}",
+        )
+        _append_reason(
+            observe_reasons,
+            delta_mae is None or delta_mae > thresholds["max_delta_mae"],
+            f"离线 MAE 退化超限：delta={delta_mae}",
+        )
+        _append_reason(
+            observe_reasons,
+            delta_hit is None or delta_hit < thresholds["min_delta_bucket_hit_rate"],
+            f"离线 bucket 命中率下降：delta={delta_hit}",
+        )
+        _append_reason(
+            observe_reasons,
+            shadow_samples < thresholds["shadow_min_samples"],
+            f"shadow 样本不足：{shadow_samples}",
+        )
+        _append_reason(
+            observe_reasons,
+            shadow_delta_mae is None or shadow_delta_mae > thresholds["max_delta_mae"],
+            f"shadow MAE 退化超限：delta={shadow_delta_mae}",
+        )
+        _append_reason(
+            observe_reasons,
+            shadow_delta_hit is None or shadow_delta_hit < thresholds["min_delta_bucket_hit_rate"],
+            f"shadow bucket 命中率下降：delta={shadow_delta_hit}",
+        )
+        _append_reason(
+            observe_reasons,
+            shadow_delta_brier is None
+            or shadow_delta_brier > thresholds["max_delta_bucket_brier_observe"],
+            f"shadow bucket brier 退化偏大：delta={shadow_delta_brier}",
+        )
+
+        if not observe_reasons:
+            decision = "observe"
+            summary = "离线评估达标，但 shadow 仍需继续观察，暂不切主路径。"
+        else:
+            decision = "hold"
+            summary = "当前指标不足以切换 emos_primary，应继续保持 shadow。"
+
+    return {
+        "decision": decision,
+        "ready_for_primary": decision == "promote",
+        "summary": summary,
+        "thresholds": thresholds,
+        "evaluation": {
+            "sample_count": eval_samples,
+            "delta_crps": delta_crps,
+            "delta_mae": delta_mae,
+            "delta_bucket_hit_rate": delta_hit,
+        },
+        "shadow": {
+            "sample_count": shadow_samples,
+            "delta_mae": shadow_delta_mae,
+            "delta_bucket_hit_rate": shadow_delta_hit,
+            "delta_bucket_brier": shadow_delta_brier,
+        },
+        "blocking_reasons": promote_reasons,
+        "worst_shadow_regressions": _top_shadow_regressions(
+            (shadow_report or {}).get("by_city") or {}
+        ),
+    }
+
+
+def build_rollout_report(
+    evaluation_report_path: str,
+    shadow_report_path: str,
+) -> Dict[str, Any]:
+    evaluation_report = _load_json_file(evaluation_report_path)
+    shadow_report = _load_json_file(shadow_report_path)
+    decision = judge_probability_rollout(evaluation_report, shadow_report)
+    return {
+        "evaluation_report_path": evaluation_report_path,
+        "shadow_report_path": shadow_report_path,
+        "evaluation_report_exists": os.path.exists(evaluation_report_path),
+        "shadow_report_exists": os.path.exists(shadow_report_path),
+        "decision": decision,
+    }

src/analysis/probability_snapshot_archive.py

@@ -5,10 +5,18 @@ import os
 from datetime import datetime
 from typing import Any, Dict, List, Optional
 
+from src.database.runtime_state import (
+    ProbabilitySnapshotRepository,
+    STATE_STORAGE_DUAL,
+    STATE_STORAGE_SQLITE,
+    get_state_storage_mode,
+)
+
 DEDUP_SCAN_LINES = 200
 MU_THRESHOLD = 0.2
 SIGMA_THRESHOLD = 0.15
 MAX_SO_FAR_THRESHOLD = 0.2
+_snapshot_repo = ProbabilitySnapshotRepository()
 
 
 def _sf(value: Any) -> Optional[float]:
@@ -79,7 +87,15 @@ def _load_recent_rows(path: str, max_lines: int = DEDUP_SCAN_LINES) -> List[Dict
 
 
 def _should_skip_append(path: str, payload: Dict[str, Any]) -> bool:
-    recent_rows = _load_recent_rows(path)
+    mode = get_state_storage_mode()
+    if mode == STATE_STORAGE_SQLITE:
+        recent_rows = _snapshot_repo.load_recent_rows(
+            str(payload.get("city") or ""),
+            str(payload.get("date") or ""),
+            DEDUP_SCAN_LINES,
+        )
+    else:
+        recent_rows = _load_recent_rows(path)
     city = payload.get("city")
     date_str = payload.get("date")
     if not city or not date_str:
@@ -183,5 +199,10 @@ def append_probability_snapshot(
     if _should_skip_append(path, payload):
         return
 
-    with open(path, "a", encoding="utf-8") as fh:
-        fh.write(json.dumps(payload, ensure_ascii=False) + "\n")
+    mode = get_state_storage_mode()
+    if mode in {STATE_STORAGE_DUAL, STATE_STORAGE_SQLITE}:
+        _snapshot_repo.append_snapshot(payload)
+
+    if mode != STATE_STORAGE_SQLITE:
+        with open(path, "a", encoding="utf-8") as fh:
+            fh.write(json.dumps(payload, ensure_ascii=False) + "\n")