删除 LGBM 全部代码和模型文件，EMOS 简化为纯 legacy 高斯分桶模式
This commit is contained in:
2569718930@qq.com
@@ -1,359 +0,0 @@
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
if PROJECT_ROOT not in sys.path:
|
||||
sys.path.insert(0, PROJECT_ROOT)
|
||||
|
||||
from src.analysis.probability_calibration import DEFAULT_CALIBRATION_FILE # noqa: E402
|
||||
|
||||
|
||||
ARTIFACT_DIR = os.path.join(PROJECT_ROOT, "artifacts", "probability_calibration")
|
||||
|
||||
|
||||
def _runtime_calibration_dir() -> str:
|
||||
runtime_dir = str(os.getenv("POLYWEATHER_RUNTIME_DATA_DIR") or "").strip()
|
||||
if runtime_dir:
|
||||
return os.path.join(runtime_dir, "probability_calibration")
|
||||
return ARTIFACT_DIR
|
||||
|
||||
|
||||
DEFAULT_CANDIDATE_ROOT = os.path.join(_runtime_calibration_dir(), "candidates")
|
||||
DEFAULT_DECISION_REPORT = os.path.join(
|
||||
_runtime_calibration_dir(),
|
||||
"auto_retrain_report.json",
|
||||
)
|
||||
|
||||
|
||||
def _sf(value: Any) -> Optional[float]:
|
||||
if value is None:
|
||||
return None
|
||||
try:
|
||||
return float(value)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _env_float(name: str, default: float) -> float:
|
||||
value = _sf(os.getenv(name))
|
||||
return value if value is not None else default
|
||||
|
||||
|
||||
def _env_int(name: str, default: int) -> int:
|
||||
value = _sf(os.getenv(name))
|
||||
return int(value) if value is not None else default
|
||||
|
||||
|
||||
def _load_json(path: str) -> Dict[str, Any]:
|
||||
try:
|
||||
with open(path, "r", encoding="utf-8") as fh:
|
||||
data = json.load(fh)
|
||||
return data if isinstance(data, dict) else {}
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
|
||||
def _write_json(path: str, payload: Dict[str, Any]) -> None:
|
||||
output_dir = os.path.dirname(os.path.abspath(path))
|
||||
if output_dir:
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
with open(path, "w", encoding="utf-8") as fh:
|
||||
json.dump(payload, fh, ensure_ascii=False, indent=2)
|
||||
|
||||
|
||||
def _run_python(args: List[str], *, stream: bool = False) -> Dict[str, Any]:
|
||||
command = [sys.executable, *args]
|
||||
if stream:
|
||||
completed = subprocess.run(
|
||||
command,
|
||||
cwd=PROJECT_ROOT,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
return {
|
||||
"command": command,
|
||||
"returncode": completed.returncode,
|
||||
"stdout": "",
|
||||
"stderr": "",
|
||||
}
|
||||
completed = subprocess.run(
|
||||
command,
|
||||
cwd=PROJECT_ROOT,
|
||||
text=True,
|
||||
capture_output=True,
|
||||
check=False,
|
||||
)
|
||||
return {
|
||||
"command": command,
|
||||
"returncode": completed.returncode,
|
||||
"stdout": completed.stdout,
|
||||
"stderr": completed.stderr,
|
||||
}
|
||||
|
||||
|
||||
def _append_blocker(blockers: List[str], condition: bool, message: str) -> None:
|
||||
if condition:
|
||||
blockers.append(message)
|
||||
|
||||
|
||||
def judge_candidate(
|
||||
evaluation_report: Dict[str, Any],
|
||||
*,
|
||||
min_samples: int,
|
||||
max_delta_crps: float,
|
||||
max_delta_mae: float,
|
||||
min_delta_bucket_hit_rate: float,
|
||||
) -> Dict[str, Any]:
|
||||
summary = evaluation_report.get("summary") or {}
|
||||
delta = summary.get("delta") or {}
|
||||
sample_count = int(summary.get("sample_count") or 0)
|
||||
delta_crps = _sf(delta.get("crps"))
|
||||
delta_mae = _sf(delta.get("mae"))
|
||||
delta_hit = _sf(delta.get("bucket_hit_rate"))
|
||||
|
||||
blockers: List[str] = []
|
||||
_append_blocker(
|
||||
blockers,
|
||||
sample_count < min_samples,
|
||||
f"sample_count {sample_count} < {min_samples}",
|
||||
)
|
||||
_append_blocker(
|
||||
blockers,
|
||||
delta_crps is None or delta_crps > max_delta_crps,
|
||||
f"delta_crps {delta_crps} > {max_delta_crps}",
|
||||
)
|
||||
_append_blocker(
|
||||
blockers,
|
||||
delta_mae is None or delta_mae > max_delta_mae,
|
||||
f"delta_mae {delta_mae} > {max_delta_mae}",
|
||||
)
|
||||
_append_blocker(
|
||||
blockers,
|
||||
delta_hit is None or delta_hit < min_delta_bucket_hit_rate,
|
||||
f"delta_bucket_hit_rate {delta_hit} < {min_delta_bucket_hit_rate}",
|
||||
)
|
||||
|
||||
return {
|
||||
"decision": "promote" if not blockers else "hold",
|
||||
"ready_for_promotion": not blockers,
|
||||
"blocking_reasons": blockers,
|
||||
"thresholds": {
|
||||
"min_samples": min_samples,
|
||||
"max_delta_crps": max_delta_crps,
|
||||
"max_delta_mae": max_delta_mae,
|
||||
"min_delta_bucket_hit_rate": min_delta_bucket_hit_rate,
|
||||
},
|
||||
"metrics": {
|
||||
"sample_count": sample_count,
|
||||
"delta_crps": delta_crps,
|
||||
"delta_mae": delta_mae,
|
||||
"delta_bucket_hit_rate": delta_hit,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _promote(candidate_path: str, target_path: str) -> str:
|
||||
target_dir = os.path.dirname(os.path.abspath(target_path))
|
||||
if target_dir:
|
||||
os.makedirs(target_dir, exist_ok=True)
|
||||
backup_path = os.path.join(
|
||||
target_dir,
|
||||
"default.backup-{ts}.json".format(
|
||||
ts=datetime.now(timezone.utc).strftime("%Y%m%d%H%M%S")
|
||||
),
|
||||
)
|
||||
if os.path.exists(target_path):
|
||||
shutil.copy2(target_path, backup_path)
|
||||
shutil.copy2(candidate_path, target_path)
|
||||
return backup_path
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Train an EMOS candidate, evaluate it, and optionally promote it behind gates."
|
||||
)
|
||||
parser.add_argument("--candidate-root", default=DEFAULT_CANDIDATE_ROOT)
|
||||
parser.add_argument("--target", default=DEFAULT_CALIBRATION_FILE)
|
||||
parser.add_argument("--decision-output", default=DEFAULT_DECISION_REPORT)
|
||||
parser.add_argument(
|
||||
"--promote-if-passed",
|
||||
action="store_true",
|
||||
help="Copy the candidate over the active calibration file only if gates pass.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--run-tests",
|
||||
action="store_true",
|
||||
help="Run focused probability tests before promotion.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--verbose",
|
||||
action="store_true",
|
||||
help="Print child script progress while training and evaluating.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--snapshot-limit",
|
||||
type=int,
|
||||
default=_env_int("POLYWEATHER_EMOS_TRAINING_SNAPSHOT_LIMIT", 0),
|
||||
help="Optional max number of recent probability snapshots to load from SQLite.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--min-samples",
|
||||
type=int,
|
||||
default=_env_int("POLYWEATHER_EMOS_AUTO_MIN_SAMPLES", 50),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--max-delta-crps",
|
||||
type=float,
|
||||
default=_env_float("POLYWEATHER_EMOS_AUTO_MAX_DELTA_CRPS", 0.0),
|
||||
help="Candidate EMOS CRPS may not be worse than legacy by more than this.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--max-delta-mae",
|
||||
type=float,
|
||||
default=_env_float("POLYWEATHER_EMOS_AUTO_MAX_DELTA_MAE", 0.05),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--min-delta-bucket-hit-rate",
|
||||
type=float,
|
||||
default=_env_float("POLYWEATHER_EMOS_AUTO_MIN_DELTA_BUCKET_HIT_RATE", -0.05),
|
||||
help="Soft guard only; bucket hit rate is boundary-sensitive.",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
version = "emos-auto-{ts}".format(
|
||||
ts=datetime.now(timezone.utc).strftime("%Y%m%d%H%M%S")
|
||||
)
|
||||
candidate_dir = os.path.join(args.candidate_root, version)
|
||||
os.makedirs(candidate_dir, exist_ok=True)
|
||||
candidate_path = os.path.join(candidate_dir, "default.json")
|
||||
evaluation_path = os.path.join(candidate_dir, "evaluation_report.json")
|
||||
decision_path = os.path.join(candidate_dir, "decision_report.json")
|
||||
|
||||
fit_args = [
|
||||
"scripts/fit_probability_calibration.py",
|
||||
"--output",
|
||||
candidate_path,
|
||||
"--version",
|
||||
version,
|
||||
]
|
||||
if args.verbose:
|
||||
fit_args.append("--verbose")
|
||||
if args.snapshot_limit and args.snapshot_limit > 0:
|
||||
fit_args.extend(["--snapshot-limit", str(args.snapshot_limit)])
|
||||
fit_result = _run_python(
|
||||
fit_args,
|
||||
stream=args.verbose,
|
||||
)
|
||||
if fit_result["returncode"] != 0:
|
||||
payload = {
|
||||
"ok": False,
|
||||
"version": version,
|
||||
"stage": "fit",
|
||||
"fit": fit_result,
|
||||
}
|
||||
_write_json(decision_path, payload)
|
||||
_write_json(args.decision_output, payload)
|
||||
print(json.dumps(payload, ensure_ascii=False, indent=2))
|
||||
return fit_result["returncode"] or 1
|
||||
|
||||
eval_args = [
|
||||
"scripts/evaluate_probability_calibration.py",
|
||||
"--calibration-file",
|
||||
candidate_path,
|
||||
"--output",
|
||||
evaluation_path,
|
||||
]
|
||||
if args.verbose:
|
||||
eval_args.append("--verbose")
|
||||
if args.snapshot_limit and args.snapshot_limit > 0:
|
||||
eval_args.extend(["--snapshot-limit", str(args.snapshot_limit)])
|
||||
eval_result = _run_python(
|
||||
eval_args,
|
||||
stream=args.verbose,
|
||||
)
|
||||
if eval_result["returncode"] != 0:
|
||||
payload = {
|
||||
"ok": False,
|
||||
"version": version,
|
||||
"stage": "evaluate",
|
||||
"candidate_path": candidate_path,
|
||||
"fit": fit_result,
|
||||
"evaluate": eval_result,
|
||||
}
|
||||
_write_json(decision_path, payload)
|
||||
_write_json(args.decision_output, payload)
|
||||
print(json.dumps(payload, ensure_ascii=False, indent=2))
|
||||
return eval_result["returncode"] or 1
|
||||
|
||||
evaluation_report = _load_json(evaluation_path)
|
||||
decision = judge_candidate(
|
||||
evaluation_report,
|
||||
min_samples=args.min_samples,
|
||||
max_delta_crps=args.max_delta_crps,
|
||||
max_delta_mae=args.max_delta_mae,
|
||||
min_delta_bucket_hit_rate=args.min_delta_bucket_hit_rate,
|
||||
)
|
||||
|
||||
test_result = None
|
||||
if args.run_tests and decision["ready_for_promotion"]:
|
||||
test_result = _run_python(
|
||||
[
|
||||
"-m",
|
||||
"pytest",
|
||||
"tests/test_probability_calibration.py",
|
||||
"tests/test_probability_rollout.py",
|
||||
"tests/test_trend_engine.py",
|
||||
]
|
||||
)
|
||||
if test_result["returncode"] != 0:
|
||||
decision["decision"] = "hold"
|
||||
decision["ready_for_promotion"] = False
|
||||
decision.setdefault("blocking_reasons", []).append(
|
||||
"focused tests failed"
|
||||
)
|
||||
|
||||
promoted = False
|
||||
backup_path = None
|
||||
if args.promote_if_passed and decision["ready_for_promotion"]:
|
||||
backup_path = _promote(candidate_path, args.target)
|
||||
promoted = True
|
||||
|
||||
payload = {
|
||||
"ok": True,
|
||||
"version": version,
|
||||
"generated_at": datetime.now(timezone.utc).isoformat(),
|
||||
"candidate_dir": candidate_dir,
|
||||
"candidate_path": candidate_path,
|
||||
"evaluation_path": evaluation_path,
|
||||
"target_path": args.target,
|
||||
"promote_requested": bool(args.promote_if_passed),
|
||||
"promoted": promoted,
|
||||
"backup_path": backup_path,
|
||||
"decision": decision,
|
||||
"fit": fit_result,
|
||||
"evaluate": eval_result,
|
||||
"tests": test_result,
|
||||
}
|
||||
_write_json(decision_path, payload)
|
||||
_write_json(args.decision_output, payload)
|
||||
print(json.dumps(payload["decision"], ensure_ascii=False, indent=2))
|
||||
print(f"candidate: {candidate_path}")
|
||||
print(f"evaluation: {evaluation_path}")
|
||||
print(f"decision: {decision_path}")
|
||||
if promoted:
|
||||
print(f"promoted to {args.target}; backup: {backup_path}")
|
||||
elif args.promote_if_passed:
|
||||
print("not promoted; gates did not pass")
|
||||
else:
|
||||
print("not promoted; run with --promote-if-passed to allow gated promotion")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,222 +0,0 @@
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
from datetime import datetime
|
||||
|
||||
PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
if PROJECT_ROOT not in sys.path:
|
||||
sys.path.insert(0, PROJECT_ROOT)
|
||||
|
||||
from src.analysis.settlement_rounding import apply_city_settlement # noqa: E402
|
||||
from scripts.fit_probability_calibration import ( # noqa: E402
|
||||
_default_history_arg,
|
||||
_load_history_with_fallback,
|
||||
)
|
||||
|
||||
|
||||
def _sf(value):
|
||||
if value is None:
|
||||
return None
|
||||
try:
|
||||
return float(value)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _mean(values):
|
||||
return round(sum(values) / len(values), 6) if values else None
|
||||
|
||||
|
||||
def _top_bucket(snapshot):
|
||||
if not isinstance(snapshot, list):
|
||||
return None
|
||||
best = None
|
||||
best_prob = -1.0
|
||||
for row in snapshot:
|
||||
if not isinstance(row, dict):
|
||||
continue
|
||||
try:
|
||||
prob = float(row.get("p") if "p" in row else row.get("probability"))
|
||||
except Exception:
|
||||
continue
|
||||
value = row.get("v") if "v" in row else row.get("value")
|
||||
if value is None:
|
||||
continue
|
||||
if prob > best_prob:
|
||||
best = value
|
||||
best_prob = prob
|
||||
return best
|
||||
|
||||
|
||||
def _bucket_probability(snapshot, target_bucket):
|
||||
if not isinstance(snapshot, list):
|
||||
return 0.0
|
||||
for row in snapshot:
|
||||
if not isinstance(row, dict):
|
||||
continue
|
||||
value = row.get("v") if "v" in row else row.get("value")
|
||||
if value != target_bucket:
|
||||
continue
|
||||
try:
|
||||
return float(row.get("p") if "p" in row else row.get("probability") or 0.0)
|
||||
except Exception:
|
||||
return 0.0
|
||||
return 0.0
|
||||
|
||||
|
||||
def _brier_from_snapshot(snapshot, target_bucket):
|
||||
hit_prob = _bucket_probability(snapshot, target_bucket)
|
||||
total = (1.0 - hit_prob) ** 2
|
||||
if isinstance(snapshot, list):
|
||||
for row in snapshot:
|
||||
if not isinstance(row, dict):
|
||||
continue
|
||||
value = row.get("v") if "v" in row else row.get("value")
|
||||
if value == target_bucket:
|
||||
continue
|
||||
try:
|
||||
prob = float(row.get("p") if "p" in row else row.get("probability") or 0.0)
|
||||
except Exception:
|
||||
prob = 0.0
|
||||
total += prob * prob
|
||||
return round(total, 6)
|
||||
|
||||
|
||||
def _blank_metrics():
|
||||
return {
|
||||
"samples": 0,
|
||||
"legacy_mae": [],
|
||||
"shadow_mae": [],
|
||||
"legacy_bucket_hit": [],
|
||||
"shadow_bucket_hit": [],
|
||||
"legacy_bucket_brier": [],
|
||||
"shadow_bucket_brier": [],
|
||||
}
|
||||
|
||||
|
||||
def _rollup(metrics):
|
||||
return {
|
||||
"samples": metrics["samples"],
|
||||
"legacy_mean_mae": _mean(metrics["legacy_mae"]),
|
||||
"shadow_mean_mae": _mean(metrics["shadow_mae"]),
|
||||
"legacy_bucket_hit_rate": _mean(metrics["legacy_bucket_hit"]),
|
||||
"shadow_bucket_hit_rate": _mean(metrics["shadow_bucket_hit"]),
|
||||
"legacy_bucket_brier": _mean(metrics["legacy_bucket_brier"]),
|
||||
"shadow_bucket_brier": _mean(metrics["shadow_bucket_brier"]),
|
||||
"delta_mae": round((_mean(metrics["shadow_mae"]) or 0.0) - (_mean(metrics["legacy_mae"]) or 0.0), 6),
|
||||
"delta_bucket_hit_rate": round((_mean(metrics["shadow_bucket_hit"]) or 0.0) - (_mean(metrics["legacy_bucket_hit"]) or 0.0), 6),
|
||||
"delta_bucket_brier": round((_mean(metrics["shadow_bucket_brier"]) or 0.0) - (_mean(metrics["legacy_bucket_brier"]) or 0.0), 6),
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Build live shadow probability report from daily records.")
|
||||
parser.add_argument(
|
||||
"--history-file",
|
||||
default=_default_history_arg(),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output",
|
||||
default=os.path.join(
|
||||
PROJECT_ROOT,
|
||||
"artifacts",
|
||||
"probability_calibration",
|
||||
"shadow_report.json",
|
||||
),
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
history = _load_history_with_fallback(args.history_file)
|
||||
overall = _blank_metrics()
|
||||
by_city = defaultdict(_blank_metrics)
|
||||
by_date = defaultdict(_blank_metrics)
|
||||
latest_observations = []
|
||||
|
||||
for city, city_records in sorted(history.items()):
|
||||
if not isinstance(city_records, dict):
|
||||
continue
|
||||
for date_str, record in sorted(city_records.items()):
|
||||
if not isinstance(record, dict):
|
||||
continue
|
||||
actual_high = _sf(record.get("actual_high"))
|
||||
shadow_snapshot = record.get("shadow_prob_snapshot")
|
||||
calibration = record.get("probability_calibration") or {}
|
||||
if actual_high is None or not shadow_snapshot:
|
||||
continue
|
||||
|
||||
legacy_mu = _sf(calibration.get("raw_mu"))
|
||||
if legacy_mu is None:
|
||||
legacy_mu = _sf(record.get("mu"))
|
||||
shadow_mu = _sf(calibration.get("calibrated_mu"))
|
||||
if shadow_mu is None:
|
||||
continue
|
||||
|
||||
actual_bucket = apply_city_settlement(city, actual_high)
|
||||
legacy_snapshot = record.get("prob_snapshot") or []
|
||||
legacy_bucket = _top_bucket(legacy_snapshot)
|
||||
shadow_bucket = _top_bucket(shadow_snapshot)
|
||||
|
||||
for metrics in (overall, by_city[city], by_date[date_str]):
|
||||
metrics["samples"] += 1
|
||||
metrics["legacy_mae"].append(abs(legacy_mu - actual_high))
|
||||
metrics["shadow_mae"].append(abs(shadow_mu - actual_high))
|
||||
metrics["legacy_bucket_hit"].append(1.0 if legacy_bucket == actual_bucket else 0.0)
|
||||
metrics["shadow_bucket_hit"].append(1.0 if shadow_bucket == actual_bucket else 0.0)
|
||||
metrics["legacy_bucket_brier"].append(_brier_from_snapshot(legacy_snapshot, actual_bucket))
|
||||
metrics["shadow_bucket_brier"].append(_brier_from_snapshot(shadow_snapshot, actual_bucket))
|
||||
|
||||
latest_observations.append(
|
||||
{
|
||||
"city": city,
|
||||
"date": date_str,
|
||||
"actual_high": actual_high,
|
||||
"actual_bucket": actual_bucket,
|
||||
"legacy_mu": round(legacy_mu, 3),
|
||||
"shadow_mu": round(shadow_mu, 3),
|
||||
"legacy_top_bucket": legacy_bucket,
|
||||
"shadow_top_bucket": shadow_bucket,
|
||||
"calibration_version": calibration.get("version"),
|
||||
"calibration_mode": calibration.get("mode"),
|
||||
}
|
||||
)
|
||||
|
||||
by_city_report = {
|
||||
city: _rollup(metrics)
|
||||
for city, metrics in sorted(by_city.items())
|
||||
}
|
||||
by_date_report = {
|
||||
date_str: _rollup(metrics)
|
||||
for date_str, metrics in sorted(
|
||||
by_date.items(),
|
||||
key=lambda item: datetime.strptime(item[0], "%Y-%m-%d"),
|
||||
)
|
||||
}
|
||||
|
||||
latest_observations = sorted(
|
||||
latest_observations,
|
||||
key=lambda row: (row["date"], row["city"]),
|
||||
reverse=True,
|
||||
)[:100]
|
||||
|
||||
payload = {
|
||||
"generated_at": datetime.utcnow().isoformat() + "Z",
|
||||
"summary": _rollup(overall),
|
||||
"by_city": by_city_report,
|
||||
"by_date": by_date_report,
|
||||
"recent_observations": latest_observations,
|
||||
}
|
||||
|
||||
output_dir = os.path.dirname(os.path.abspath(args.output))
|
||||
if output_dir:
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
with open(args.output, "w", encoding="utf-8") as fh:
|
||||
json.dump(payload, fh, ensure_ascii=False, indent=2)
|
||||
|
||||
print(json.dumps(payload["summary"], ensure_ascii=False, indent=2))
|
||||
print(f"saved shadow report to {args.output}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,270 +0,0 @@
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
|
||||
PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
if PROJECT_ROOT not in sys.path:
|
||||
sys.path.insert(0, PROJECT_ROOT)
|
||||
|
||||
from src.analysis.probability_calibration import ( # noqa: E402
|
||||
ENGINE_MODE_EMOS_PRIMARY,
|
||||
_gaussian_crps,
|
||||
apply_probability_calibration,
|
||||
build_probability_features,
|
||||
)
|
||||
from src.analysis.settlement_rounding import apply_city_settlement # noqa: E402
|
||||
from scripts.fit_probability_calibration import ( # noqa: E402
|
||||
_default_history_arg,
|
||||
_extract_samples,
|
||||
_load_history_with_fallback,
|
||||
_load_json_if_exists,
|
||||
_load_legacy_training_samples,
|
||||
_load_snapshot_rows,
|
||||
_load_training_feature_history,
|
||||
_load_truth_history,
|
||||
merge_samples_with_legacy_archive,
|
||||
)
|
||||
|
||||
|
||||
def _env_int(name, default=None):
|
||||
try:
|
||||
value = os.getenv(name)
|
||||
if value is None or str(value).strip() == "":
|
||||
return default
|
||||
return int(value)
|
||||
except Exception:
|
||||
return default
|
||||
|
||||
|
||||
def _log(enabled, message):
|
||||
if enabled:
|
||||
print(f"[evaluate_probability_calibration] {message}", flush=True)
|
||||
|
||||
|
||||
def _mean(values):
|
||||
return (sum(values) / len(values)) if values else None
|
||||
|
||||
|
||||
def _sample_to_features(sample):
|
||||
return build_probability_features(
|
||||
city_name=sample.get("city") or "",
|
||||
raw_mu=sample.get("raw_mu"),
|
||||
raw_sigma=sample.get("raw_sigma"),
|
||||
deb_prediction=sample.get("deb_prediction"),
|
||||
ens_data={
|
||||
"median": sample.get("ens_median"),
|
||||
"p10": None,
|
||||
"p90": None,
|
||||
},
|
||||
current_forecasts={},
|
||||
max_so_far=None,
|
||||
peak_status="in_window" if sample.get("peak_flag") == 0.5 else "past" if sample.get("peak_flag") == 1.0 else "before",
|
||||
local_hour_frac=None,
|
||||
)
|
||||
|
||||
|
||||
def _top_bucket_value(distribution):
|
||||
if not distribution:
|
||||
return None
|
||||
top = max(
|
||||
(row for row in distribution if isinstance(row, dict)),
|
||||
key=lambda row: float(row.get("probability") or 0.0),
|
||||
default=None,
|
||||
)
|
||||
if not top:
|
||||
return None
|
||||
return top.get("value")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Evaluate legacy vs EMOS probability calibration.")
|
||||
parser.add_argument(
|
||||
"--history-file",
|
||||
default=_default_history_arg(),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--settlement-history",
|
||||
default=os.path.join(
|
||||
PROJECT_ROOT,
|
||||
"artifacts",
|
||||
"probability_calibration",
|
||||
"settlement_history.json",
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--calibration-file",
|
||||
default=os.path.join(
|
||||
PROJECT_ROOT,
|
||||
"artifacts",
|
||||
"probability_calibration",
|
||||
"default.json",
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output",
|
||||
default=os.path.join(
|
||||
PROJECT_ROOT,
|
||||
"artifacts",
|
||||
"probability_calibration",
|
||||
"evaluation_report.json",
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--snapshot-file",
|
||||
default=None,
|
||||
help="Optional legacy JSONL snapshot archive path. In sqlite mode this defaults to the runtime database.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--snapshot-limit",
|
||||
type=int,
|
||||
default=_env_int("POLYWEATHER_EMOS_TRAINING_SNAPSHOT_LIMIT"),
|
||||
help="Optional max number of recent probability snapshots to load from SQLite.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--verbose",
|
||||
action="store_true",
|
||||
help="Print data loading and evaluation progress.",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
_log(args.verbose, "loading daily records")
|
||||
history = _load_history_with_fallback(args.history_file)
|
||||
_log(args.verbose, f"loaded daily record cities={len(history or {})}")
|
||||
_log(args.verbose, "loading training feature history")
|
||||
training_feature_history = _load_training_feature_history()
|
||||
_log(args.verbose, f"loaded training feature cities={len(training_feature_history or {})}")
|
||||
_log(args.verbose, "loading truth history")
|
||||
truth_history = _load_truth_history()
|
||||
_log(args.verbose, f"loaded truth cities={len(truth_history or {})}")
|
||||
_log(args.verbose, "loading settlement history")
|
||||
settlement_history = _load_json_if_exists(args.settlement_history)
|
||||
_log(args.verbose, f"loaded settlement history cities={len(settlement_history or {})}")
|
||||
_log(
|
||||
args.verbose,
|
||||
"loading probability snapshots"
|
||||
+ (f" limit={args.snapshot_limit}" if args.snapshot_limit else ""),
|
||||
)
|
||||
snapshot_rows = _load_snapshot_rows(args.snapshot_file, limit=args.snapshot_limit)
|
||||
_log(args.verbose, f"loaded probability snapshots={len(snapshot_rows or [])}")
|
||||
_log(args.verbose, "loading legacy training archive")
|
||||
legacy_training_samples = _load_legacy_training_samples()
|
||||
_log(args.verbose, f"loaded legacy training samples={len(legacy_training_samples or [])}")
|
||||
_log(args.verbose, "extracting evaluation samples")
|
||||
samples, filled_actual_from_history = _extract_samples(
|
||||
history,
|
||||
training_feature_history=training_feature_history,
|
||||
truth_history=truth_history,
|
||||
settlement_history=settlement_history,
|
||||
snapshot_rows=snapshot_rows,
|
||||
)
|
||||
samples = merge_samples_with_legacy_archive(samples, legacy_training_samples)
|
||||
_log(args.verbose, f"evaluating samples={len(samples or [])}")
|
||||
|
||||
legacy_crps = []
|
||||
emos_crps = []
|
||||
legacy_mae = []
|
||||
emos_mae = []
|
||||
legacy_bucket_hits = []
|
||||
emos_bucket_hits = []
|
||||
by_city = defaultdict(lambda: {
|
||||
"samples": 0,
|
||||
"legacy_crps": [],
|
||||
"emos_crps": [],
|
||||
"legacy_mae": [],
|
||||
"emos_mae": [],
|
||||
"legacy_bucket_hits": [],
|
||||
"emos_bucket_hits": [],
|
||||
})
|
||||
|
||||
for sample in samples:
|
||||
city = str(sample.get("city") or "").strip().lower()
|
||||
actual_high = float(sample["actual_high"])
|
||||
raw_mu = float(sample["raw_mu"])
|
||||
raw_sigma = max(0.1, float(sample["raw_sigma"]))
|
||||
legacy_crps.append(_gaussian_crps(actual_high, raw_mu, raw_sigma))
|
||||
legacy_mae.append(abs(raw_mu - actual_high))
|
||||
|
||||
legacy_bucket = apply_city_settlement(city, raw_mu)
|
||||
actual_bucket = apply_city_settlement(city, actual_high)
|
||||
legacy_bucket_hits.append(1.0 if legacy_bucket == actual_bucket else 0.0)
|
||||
|
||||
calibration = apply_probability_calibration(
|
||||
city_name=city,
|
||||
temp_symbol="°F" if city in {"atlanta", "chicago", "dallas", "miami", "new york", "seattle"} else "°C",
|
||||
raw_mu=raw_mu,
|
||||
raw_sigma=raw_sigma,
|
||||
max_so_far=None,
|
||||
legacy_distribution=[],
|
||||
features=_sample_to_features(sample),
|
||||
calibration_path=args.calibration_file,
|
||||
mode=ENGINE_MODE_EMOS_PRIMARY,
|
||||
)
|
||||
emos_mu = float(calibration.get("calibrated_mu") or raw_mu)
|
||||
emos_sigma = max(0.1, float(calibration.get("calibrated_sigma") or raw_sigma))
|
||||
emos_distribution = calibration.get("distribution") or []
|
||||
emos_crps.append(_gaussian_crps(actual_high, emos_mu, emos_sigma))
|
||||
emos_mae.append(abs(emos_mu - actual_high))
|
||||
emos_bucket = _top_bucket_value(emos_distribution)
|
||||
emos_bucket_hits.append(1.0 if emos_bucket == actual_bucket else 0.0)
|
||||
|
||||
row = by_city[city]
|
||||
row["samples"] += 1
|
||||
row["legacy_crps"].append(legacy_crps[-1])
|
||||
row["emos_crps"].append(emos_crps[-1])
|
||||
row["legacy_mae"].append(legacy_mae[-1])
|
||||
row["emos_mae"].append(emos_mae[-1])
|
||||
row["legacy_bucket_hits"].append(legacy_bucket_hits[-1])
|
||||
row["emos_bucket_hits"].append(emos_bucket_hits[-1])
|
||||
|
||||
summary = {
|
||||
"sample_count": len(samples),
|
||||
"filled_actual_from_history": filled_actual_from_history,
|
||||
"legacy": {
|
||||
"mean_crps": round(_mean(legacy_crps), 6) if legacy_crps else None,
|
||||
"mean_mae": round(_mean(legacy_mae), 6) if legacy_mae else None,
|
||||
"bucket_hit_rate": round(_mean(legacy_bucket_hits), 6) if legacy_bucket_hits else None,
|
||||
},
|
||||
"emos": {
|
||||
"mean_crps": round(_mean(emos_crps), 6) if emos_crps else None,
|
||||
"mean_mae": round(_mean(emos_mae), 6) if emos_mae else None,
|
||||
"bucket_hit_rate": round(_mean(emos_bucket_hits), 6) if emos_bucket_hits else None,
|
||||
},
|
||||
"delta": {
|
||||
"crps": round((_mean(emos_crps) or 0.0) - (_mean(legacy_crps) or 0.0), 6),
|
||||
"mae": round((_mean(emos_mae) or 0.0) - (_mean(legacy_mae) or 0.0), 6),
|
||||
"bucket_hit_rate": round((_mean(emos_bucket_hits) or 0.0) - (_mean(legacy_bucket_hits) or 0.0), 6),
|
||||
},
|
||||
}
|
||||
|
||||
city_report = {}
|
||||
for city, metrics in sorted(by_city.items()):
|
||||
city_report[city] = {
|
||||
"samples": metrics["samples"],
|
||||
"legacy_mean_crps": round(_mean(metrics["legacy_crps"]), 6),
|
||||
"emos_mean_crps": round(_mean(metrics["emos_crps"]), 6),
|
||||
"legacy_mean_mae": round(_mean(metrics["legacy_mae"]), 6),
|
||||
"emos_mean_mae": round(_mean(metrics["emos_mae"]), 6),
|
||||
"legacy_bucket_hit_rate": round(_mean(metrics["legacy_bucket_hits"]), 6),
|
||||
"emos_bucket_hit_rate": round(_mean(metrics["emos_bucket_hits"]), 6),
|
||||
}
|
||||
|
||||
payload = {
|
||||
"summary": summary,
|
||||
"by_city": city_report,
|
||||
}
|
||||
|
||||
output_dir = os.path.dirname(os.path.abspath(args.output))
|
||||
if output_dir:
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
with open(args.output, "w", encoding="utf-8") as fh:
|
||||
json.dump(payload, fh, ensure_ascii=False, indent=2)
|
||||
|
||||
print(json.dumps(summary, ensure_ascii=False, indent=2))
|
||||
print(f"saved evaluation report to {args.output}")
|
||||
_log(args.verbose, "done")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,78 +0,0 @@
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
if PROJECT_ROOT not in sys.path:
|
||||
sys.path.insert(0, PROJECT_ROOT)
|
||||
|
||||
from scripts.fit_probability_calibration import ( # noqa: E402
|
||||
_default_history_arg,
|
||||
_default_snapshot_arg,
|
||||
_extract_samples,
|
||||
_load_history_with_fallback,
|
||||
_load_json_if_exists,
|
||||
_load_snapshot_rows,
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Export normalized probability calibration training samples.")
|
||||
parser.add_argument(
|
||||
"--history-file",
|
||||
default=_default_history_arg(),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--settlement-history",
|
||||
default=os.path.join(
|
||||
PROJECT_ROOT,
|
||||
"artifacts",
|
||||
"probability_calibration",
|
||||
"settlement_history.json",
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output",
|
||||
default=os.path.join(
|
||||
PROJECT_ROOT,
|
||||
"artifacts",
|
||||
"probability_calibration",
|
||||
"training_samples.json",
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--snapshot-file",
|
||||
default=_default_snapshot_arg(),
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
history = _load_history_with_fallback(args.history_file)
|
||||
settlement_history = _load_json_if_exists(args.settlement_history)
|
||||
snapshot_rows = _load_snapshot_rows(args.snapshot_file)
|
||||
samples, filled_actual_from_history = _extract_samples(
|
||||
history,
|
||||
settlement_history=settlement_history,
|
||||
snapshot_rows=snapshot_rows,
|
||||
)
|
||||
snapshot_count = sum(1 for sample in samples if sample.get("sample_source") == "snapshot")
|
||||
daily_record_count = sum(1 for sample in samples if sample.get("sample_source") == "daily_record")
|
||||
|
||||
output_dir = os.path.dirname(os.path.abspath(args.output))
|
||||
if output_dir:
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
payload = {
|
||||
"sample_count": len(samples),
|
||||
"snapshot_sample_count": snapshot_count,
|
||||
"daily_record_sample_count": daily_record_count,
|
||||
"filled_actual_from_history": filled_actual_from_history,
|
||||
"samples": samples,
|
||||
}
|
||||
with open(args.output, "w", encoding="utf-8") as fh:
|
||||
json.dump(payload, fh, ensure_ascii=False, indent=2)
|
||||
|
||||
print(f"exported {len(samples)} samples to {args.output}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,523 +0,0 @@
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from datetime import datetime
|
||||
|
||||
PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
if PROJECT_ROOT not in sys.path:
|
||||
sys.path.insert(0, PROJECT_ROOT)
|
||||
|
||||
from src.analysis.probability_calibration import ( # noqa: E402
|
||||
DEFAULT_CALIBRATION_FILE,
|
||||
default_calibration_payload,
|
||||
fit_calibration,
|
||||
)
|
||||
from src.analysis.deb_algorithm import load_history # noqa: E402
|
||||
from src.database.runtime_state import ( # noqa: E402
|
||||
DailyRecordRepository,
|
||||
ProbabilitySnapshotRepository,
|
||||
STATE_STORAGE_FILE,
|
||||
STATE_STORAGE_SQLITE,
|
||||
TrainingFeatureRecordRepository,
|
||||
TruthRecordRepository,
|
||||
get_state_storage_mode,
|
||||
)
|
||||
|
||||
|
||||
def _sf(value):
|
||||
if value is None:
|
||||
return None
|
||||
try:
|
||||
return float(value)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _env_int(name, default=None):
|
||||
try:
|
||||
value = os.getenv(name)
|
||||
if value is None or str(value).strip() == "":
|
||||
return default
|
||||
return int(value)
|
||||
except Exception:
|
||||
return default
|
||||
|
||||
|
||||
def _log(enabled, message):
|
||||
if enabled:
|
||||
print(f"[fit_probability_calibration] {message}", flush=True)
|
||||
|
||||
|
||||
def _load_json_if_exists(path):
|
||||
if not path or not os.path.exists(path):
|
||||
return {}
|
||||
with open(path, "r", encoding="utf-8") as fh:
|
||||
data = json.load(fh)
|
||||
return data if isinstance(data, dict) else {}
|
||||
|
||||
|
||||
def _legacy_training_samples_path():
|
||||
return os.path.join(
|
||||
PROJECT_ROOT,
|
||||
"artifacts",
|
||||
"probability_calibration",
|
||||
"training_samples.json",
|
||||
)
|
||||
|
||||
|
||||
def _legacy_history_path():
|
||||
return os.path.join(PROJECT_ROOT, "data", "daily_records.json")
|
||||
|
||||
|
||||
def _legacy_snapshot_path():
|
||||
return os.path.join(PROJECT_ROOT, "data", "probability_training_snapshots.jsonl")
|
||||
|
||||
|
||||
def _default_history_arg():
|
||||
return _legacy_history_path() if get_state_storage_mode() == STATE_STORAGE_FILE else None
|
||||
|
||||
|
||||
def _default_snapshot_arg():
|
||||
return _legacy_snapshot_path() if get_state_storage_mode() == STATE_STORAGE_FILE else None
|
||||
|
||||
|
||||
def _load_history_with_fallback(path):
|
||||
if not path:
|
||||
if get_state_storage_mode() == STATE_STORAGE_SQLITE:
|
||||
return DailyRecordRepository().load_all()
|
||||
return {}
|
||||
data = _load_json_if_exists(path)
|
||||
if data:
|
||||
return data
|
||||
return load_history(path)
|
||||
|
||||
|
||||
def _load_truth_history():
|
||||
if get_state_storage_mode() != STATE_STORAGE_SQLITE:
|
||||
return {}
|
||||
try:
|
||||
return TruthRecordRepository().load_all()
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
|
||||
def _load_training_feature_history():
|
||||
if get_state_storage_mode() != STATE_STORAGE_SQLITE:
|
||||
return {}
|
||||
try:
|
||||
return TrainingFeatureRecordRepository().load_all()
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
|
||||
def _load_snapshot_rows(path, limit=None):
|
||||
if get_state_storage_mode() == STATE_STORAGE_SQLITE:
|
||||
repo = ProbabilitySnapshotRepository()
|
||||
if limit is not None and int(limit) > 0:
|
||||
with repo.db.connect() as conn:
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT payload_json
|
||||
FROM probability_training_snapshots_store
|
||||
ORDER BY id DESC
|
||||
LIMIT ?
|
||||
""",
|
||||
(int(limit),),
|
||||
).fetchall()
|
||||
out = []
|
||||
for row in reversed(rows):
|
||||
try:
|
||||
out.append(json.loads(row["payload_json"]))
|
||||
except Exception:
|
||||
continue
|
||||
return out
|
||||
return repo.load_all_rows()
|
||||
rows = []
|
||||
if not path or not os.path.exists(path):
|
||||
return rows
|
||||
with open(path, "r", encoding="utf-8") as fh:
|
||||
for line in fh:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
row = json.loads(line)
|
||||
except Exception:
|
||||
continue
|
||||
if isinstance(row, dict):
|
||||
rows.append(row)
|
||||
return rows
|
||||
|
||||
|
||||
def _load_legacy_training_samples(path=None):
|
||||
payload = _load_json_if_exists(path or _legacy_training_samples_path())
|
||||
rows = payload.get("samples") if isinstance(payload, dict) else None
|
||||
if not isinstance(rows, list):
|
||||
return []
|
||||
return [row for row in rows if isinstance(row, dict)]
|
||||
|
||||
|
||||
def _actual_high_for(history, truth_history, settlement_history, city, date_str):
|
||||
city_rows = (history or {}).get(city) or {}
|
||||
record = city_rows.get(date_str) or {}
|
||||
actual_high = _sf(record.get("actual_high")) if isinstance(record, dict) else None
|
||||
truth_record = ((truth_history.get(city) or {}).get(date_str) or {})
|
||||
if actual_high is None and isinstance(truth_record, dict):
|
||||
actual_high = _sf(truth_record.get("actual_high"))
|
||||
filled = False
|
||||
if actual_high is None:
|
||||
actual_high = _sf(((settlement_history.get(city) or {}).get(date_str) or {}).get("max_temp"))
|
||||
filled = actual_high is not None
|
||||
metadata = {
|
||||
"settlement_source": truth_record.get("settlement_source"),
|
||||
"settlement_station_code": truth_record.get("settlement_station_code"),
|
||||
"truth_version": truth_record.get("truth_version"),
|
||||
"truth_updated_by": truth_record.get("updated_by"),
|
||||
"truth_updated_at": truth_record.get("truth_updated_at"),
|
||||
}
|
||||
return actual_high, filled, metadata
|
||||
|
||||
|
||||
def _extract_snapshot_samples(history, truth_history=None, snapshot_rows=None, settlement_history=None):
|
||||
samples = []
|
||||
filled_actual_from_history = 0
|
||||
today = datetime.utcnow().strftime("%Y-%m-%d")
|
||||
settlement_history = settlement_history or {}
|
||||
|
||||
for row in snapshot_rows or []:
|
||||
city = str(row.get("city") or "").strip().lower()
|
||||
date_str = str(row.get("date") or "").strip()
|
||||
if not city or not date_str or date_str == today:
|
||||
continue
|
||||
|
||||
actual_high, filled, truth_meta = _actual_high_for(
|
||||
history,
|
||||
truth_history or {},
|
||||
settlement_history,
|
||||
city,
|
||||
date_str,
|
||||
)
|
||||
if actual_high is None:
|
||||
continue
|
||||
if filled:
|
||||
filled_actual_from_history += 1
|
||||
|
||||
raw_mu = _sf(row.get("raw_mu"))
|
||||
raw_sigma = _sf(row.get("raw_sigma"))
|
||||
deb_prediction = _sf(row.get("deb_prediction"))
|
||||
ensemble = row.get("ensemble") or {}
|
||||
if not isinstance(ensemble, dict):
|
||||
ensemble = {}
|
||||
ens_median = _sf(ensemble.get("median"))
|
||||
ensemble_spread = None
|
||||
ens_p10 = _sf(ensemble.get("p10"))
|
||||
ens_p90 = _sf(ensemble.get("p90"))
|
||||
if ens_p10 is not None and ens_p90 is not None and ens_p90 >= ens_p10:
|
||||
ensemble_spread = max(0.1, (ens_p90 - ens_p10) / 2.56)
|
||||
multi_model = row.get("multi_model") or {}
|
||||
if not isinstance(multi_model, dict):
|
||||
multi_model = {}
|
||||
forecast_values = [val for val in (_sf(v) for v in multi_model.values()) if val is not None]
|
||||
forecast_values.sort()
|
||||
if ensemble_spread is None:
|
||||
if len(forecast_values) >= 2:
|
||||
ensemble_spread = max(0.6, (forecast_values[-1] - forecast_values[0]) / 2.0)
|
||||
elif raw_sigma is not None:
|
||||
ensemble_spread = raw_sigma
|
||||
else:
|
||||
ensemble_spread = 1.0
|
||||
if raw_sigma is None:
|
||||
raw_sigma = ensemble_spread
|
||||
|
||||
peak_status = str(row.get("peak_status") or "before").strip().lower()
|
||||
peak_flag = 0.0
|
||||
if peak_status == "in_window":
|
||||
peak_flag = 0.5
|
||||
elif peak_status == "past":
|
||||
peak_flag = 1.0
|
||||
|
||||
max_so_far = _sf(row.get("max_so_far"))
|
||||
max_so_far_gap = None
|
||||
if deb_prediction is not None and max_so_far is not None:
|
||||
max_so_far_gap = deb_prediction - max_so_far
|
||||
|
||||
if raw_mu is None:
|
||||
continue
|
||||
|
||||
samples.append(
|
||||
{
|
||||
"city": city,
|
||||
"date": date_str,
|
||||
"timestamp": row.get("timestamp"),
|
||||
"actual_high": actual_high,
|
||||
"raw_mu": raw_mu,
|
||||
"raw_sigma": raw_sigma or 1.0,
|
||||
"deb_prediction": deb_prediction,
|
||||
"ens_median": ens_median if ens_median is not None else raw_mu,
|
||||
"ensemble_spread": ensemble_spread,
|
||||
"max_so_far_gap": max_so_far_gap,
|
||||
"peak_flag": peak_flag,
|
||||
"sample_source": "snapshot",
|
||||
**truth_meta,
|
||||
}
|
||||
)
|
||||
|
||||
return samples, filled_actual_from_history
|
||||
|
||||
|
||||
def _extract_daily_record_samples(
|
||||
history,
|
||||
training_feature_history=None,
|
||||
truth_history=None,
|
||||
settlement_history=None,
|
||||
excluded_keys=None,
|
||||
):
|
||||
samples = []
|
||||
filled_actual_from_history = 0
|
||||
today = datetime.utcnow().strftime("%Y-%m-%d")
|
||||
settlement_history = settlement_history or {}
|
||||
excluded_keys = excluded_keys or set()
|
||||
for city, city_rows in (history or {}).items():
|
||||
if not isinstance(city_rows, dict):
|
||||
continue
|
||||
city_settlement = settlement_history.get(city) or {}
|
||||
for date_str, record in city_rows.items():
|
||||
if date_str == today or not isinstance(record, dict):
|
||||
continue
|
||||
if (city, date_str) in excluded_keys:
|
||||
continue
|
||||
actual_high = _sf(record.get("actual_high"))
|
||||
truth_meta = ((truth_history or {}).get(city) or {}).get(date_str) or {}
|
||||
if actual_high is None:
|
||||
actual_high = _sf(truth_meta.get("actual_high"))
|
||||
if actual_high is None:
|
||||
actual_high = _sf((city_settlement.get(date_str) or {}).get("max_temp"))
|
||||
if actual_high is not None:
|
||||
filled_actual_from_history += 1
|
||||
feature_record = ((training_feature_history or {}).get(city) or {}).get(date_str) or {}
|
||||
source_record = feature_record if isinstance(feature_record, dict) and feature_record else record
|
||||
deb_prediction = _sf(source_record.get("deb_prediction"))
|
||||
raw_mu = _sf(source_record.get("mu")) or deb_prediction
|
||||
forecasts = source_record.get("forecasts") or {}
|
||||
if not isinstance(forecasts, dict):
|
||||
forecasts = {}
|
||||
forecast_values = [val for val in (_sf(v) for v in forecasts.values()) if val is not None]
|
||||
forecast_values.sort()
|
||||
forecast_median = (
|
||||
forecast_values[len(forecast_values) // 2] if forecast_values else None
|
||||
)
|
||||
feature_snapshot = source_record.get("probability_features") or {}
|
||||
if not isinstance(feature_snapshot, dict):
|
||||
feature_snapshot = {}
|
||||
|
||||
ens_median = _sf(feature_snapshot.get("ens_median")) or forecast_median or raw_mu
|
||||
ensemble_spread = _sf(feature_snapshot.get("ensemble_spread"))
|
||||
if ensemble_spread is None:
|
||||
if len(forecast_values) >= 2:
|
||||
ensemble_spread = max(0.6, (forecast_values[-1] - forecast_values[0]) / 2.0)
|
||||
else:
|
||||
ensemble_spread = 1.0
|
||||
raw_sigma = _sf(feature_snapshot.get("raw_sigma")) or ensemble_spread or 1.0
|
||||
peak_status = str(feature_snapshot.get("peak_status") or "before").strip().lower()
|
||||
peak_flag = 0.0
|
||||
if peak_status == "in_window":
|
||||
peak_flag = 0.5
|
||||
elif peak_status == "past":
|
||||
peak_flag = 1.0
|
||||
|
||||
if actual_high is None or raw_mu is None:
|
||||
continue
|
||||
|
||||
max_so_far = _sf(feature_snapshot.get("max_so_far"))
|
||||
max_so_far_gap = _sf(feature_snapshot.get("max_so_far_gap"))
|
||||
if max_so_far_gap is None and max_so_far is not None and deb_prediction is not None:
|
||||
max_so_far_gap = deb_prediction - max_so_far
|
||||
|
||||
samples.append(
|
||||
{
|
||||
"city": city,
|
||||
"date": date_str,
|
||||
"actual_high": actual_high,
|
||||
"raw_mu": raw_mu,
|
||||
"raw_sigma": raw_sigma,
|
||||
"deb_prediction": deb_prediction,
|
||||
"ens_median": ens_median,
|
||||
"ensemble_spread": ensemble_spread,
|
||||
"max_so_far_gap": max_so_far_gap,
|
||||
"peak_flag": peak_flag,
|
||||
"sample_source": "daily_record",
|
||||
"settlement_source": truth_meta.get("settlement_source"),
|
||||
"settlement_station_code": truth_meta.get("settlement_station_code"),
|
||||
"truth_version": truth_meta.get("truth_version"),
|
||||
"truth_updated_by": truth_meta.get("updated_by"),
|
||||
"truth_updated_at": truth_meta.get("truth_updated_at"),
|
||||
}
|
||||
)
|
||||
return samples, filled_actual_from_history
|
||||
|
||||
|
||||
def _extract_samples(history, training_feature_history=None, truth_history=None, settlement_history=None, snapshot_rows=None):
|
||||
snapshot_samples, snapshot_filled = _extract_snapshot_samples(
|
||||
history,
|
||||
truth_history=truth_history,
|
||||
snapshot_rows=snapshot_rows or [],
|
||||
settlement_history=settlement_history,
|
||||
)
|
||||
excluded_keys = {
|
||||
(sample["city"], sample["date"])
|
||||
for sample in snapshot_samples
|
||||
}
|
||||
daily_samples, daily_filled = _extract_daily_record_samples(
|
||||
history,
|
||||
training_feature_history=training_feature_history,
|
||||
truth_history=truth_history,
|
||||
settlement_history=settlement_history,
|
||||
excluded_keys=excluded_keys,
|
||||
)
|
||||
return snapshot_samples + daily_samples, snapshot_filled + daily_filled
|
||||
|
||||
|
||||
def merge_samples_with_legacy_archive(samples, legacy_samples=None):
|
||||
merged = []
|
||||
seen = set()
|
||||
for sample in samples or []:
|
||||
if not isinstance(sample, dict):
|
||||
continue
|
||||
key = (
|
||||
str(sample.get("city") or "").strip().lower(),
|
||||
str(sample.get("date") or "").strip(),
|
||||
str(sample.get("sample_source") or "").strip().lower(),
|
||||
)
|
||||
if not key[0] or not key[1]:
|
||||
continue
|
||||
if key in seen:
|
||||
continue
|
||||
merged.append(sample)
|
||||
seen.add(key)
|
||||
for sample in legacy_samples or []:
|
||||
if not isinstance(sample, dict):
|
||||
continue
|
||||
key = (
|
||||
str(sample.get("city") or "").strip().lower(),
|
||||
str(sample.get("date") or "").strip(),
|
||||
str(sample.get("sample_source") or "").strip().lower(),
|
||||
)
|
||||
if not key[0] or not key[1]:
|
||||
continue
|
||||
if key in seen:
|
||||
continue
|
||||
merged.append(sample)
|
||||
seen.add(key)
|
||||
return merged
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Fit PolyWeather probability calibration parameters.")
|
||||
parser.add_argument(
|
||||
"--history-file",
|
||||
default=_default_history_arg(),
|
||||
help="Optional legacy daily_records.json path. In sqlite mode this defaults to the runtime database.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output",
|
||||
default=DEFAULT_CALIBRATION_FILE,
|
||||
help="Output JSON file for fitted calibration parameters.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--settlement-history",
|
||||
default=os.path.join(
|
||||
PROJECT_ROOT,
|
||||
"artifacts",
|
||||
"probability_calibration",
|
||||
"settlement_history.json",
|
||||
),
|
||||
help="Optional daily settlement history JSON built from historical CSV files.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--snapshot-file",
|
||||
default=_default_snapshot_arg(),
|
||||
help="Optional legacy JSONL snapshot archive path. In sqlite mode this defaults to the runtime database.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--version",
|
||||
default=None,
|
||||
help="Optional explicit calibration version.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--snapshot-limit",
|
||||
type=int,
|
||||
default=_env_int("POLYWEATHER_EMOS_TRAINING_SNAPSHOT_LIMIT"),
|
||||
help="Optional max number of recent probability snapshots to load from SQLite.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--verbose",
|
||||
action="store_true",
|
||||
help="Print data loading and fitting progress.",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
_log(args.verbose, "loading daily records")
|
||||
history = _load_history_with_fallback(args.history_file)
|
||||
_log(args.verbose, f"loaded daily record cities={len(history or {})}")
|
||||
_log(args.verbose, "loading training feature history")
|
||||
training_feature_history = _load_training_feature_history()
|
||||
_log(args.verbose, f"loaded training feature cities={len(training_feature_history or {})}")
|
||||
_log(args.verbose, "loading truth history")
|
||||
truth_history = _load_truth_history()
|
||||
_log(args.verbose, f"loaded truth cities={len(truth_history or {})}")
|
||||
_log(args.verbose, "loading settlement history")
|
||||
settlement_history = _load_json_if_exists(args.settlement_history)
|
||||
_log(args.verbose, f"loaded settlement history cities={len(settlement_history or {})}")
|
||||
_log(
|
||||
args.verbose,
|
||||
"loading probability snapshots"
|
||||
+ (f" limit={args.snapshot_limit}" if args.snapshot_limit else ""),
|
||||
)
|
||||
snapshot_rows = _load_snapshot_rows(args.snapshot_file, limit=args.snapshot_limit)
|
||||
_log(args.verbose, f"loaded probability snapshots={len(snapshot_rows or [])}")
|
||||
_log(args.verbose, "loading legacy training archive")
|
||||
legacy_training_samples = _load_legacy_training_samples()
|
||||
_log(args.verbose, f"loaded legacy training samples={len(legacy_training_samples or [])}")
|
||||
_log(args.verbose, "extracting EMOS samples")
|
||||
samples, filled_actual_from_history = _extract_samples(
|
||||
history,
|
||||
training_feature_history=training_feature_history,
|
||||
truth_history=truth_history,
|
||||
settlement_history=settlement_history,
|
||||
snapshot_rows=snapshot_rows,
|
||||
)
|
||||
samples = merge_samples_with_legacy_archive(samples, legacy_training_samples)
|
||||
_log(args.verbose, f"fitting calibration samples={len(samples or [])}")
|
||||
calibration = fit_calibration(samples, version=args.version)
|
||||
if not samples:
|
||||
calibration = default_calibration_payload(
|
||||
version=args.version,
|
||||
reason="no_samples",
|
||||
)
|
||||
calibration.setdefault("metrics", {})
|
||||
calibration["metrics"]["filled_actual_from_history"] = filled_actual_from_history
|
||||
calibration["metrics"]["settlement_history_city_count"] = len(settlement_history)
|
||||
calibration["metrics"]["legacy_archive_samples"] = len(legacy_training_samples)
|
||||
try:
|
||||
calibration["source"] = os.path.relpath(args.output, PROJECT_ROOT)
|
||||
except ValueError:
|
||||
calibration["source"] = os.path.abspath(args.output)
|
||||
|
||||
output_dir = os.path.dirname(os.path.abspath(args.output))
|
||||
if output_dir:
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
with open(args.output, "w", encoding="utf-8") as fh:
|
||||
json.dump(calibration, fh, ensure_ascii=False, indent=2)
|
||||
|
||||
_log(args.verbose, "done")
|
||||
print(
|
||||
"saved calibration to {path} with {count} samples".format(
|
||||
path=args.output,
|
||||
count=calibration.get("metrics", {}).get("sample_count", 0),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,56 +0,0 @@
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
if PROJECT_ROOT not in sys.path:
|
||||
sys.path.insert(0, PROJECT_ROOT)
|
||||
|
||||
from src.analysis.probability_rollout import build_rollout_report # noqa: E402
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Judge whether EMOS is ready for primary rollout.")
|
||||
parser.add_argument(
|
||||
"--evaluation-report",
|
||||
default=os.path.join(
|
||||
PROJECT_ROOT,
|
||||
"artifacts",
|
||||
"probability_calibration",
|
||||
"evaluation_report.json",
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--shadow-report",
|
||||
default=os.path.join(
|
||||
PROJECT_ROOT,
|
||||
"artifacts",
|
||||
"probability_calibration",
|
||||
"shadow_report.json",
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output",
|
||||
default=os.path.join(
|
||||
PROJECT_ROOT,
|
||||
"artifacts",
|
||||
"probability_calibration",
|
||||
"rollout_report.json",
|
||||
),
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
payload = build_rollout_report(args.evaluation_report, args.shadow_report)
|
||||
output_dir = os.path.dirname(os.path.abspath(args.output))
|
||||
if output_dir:
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
with open(args.output, "w", encoding="utf-8") as fh:
|
||||
json.dump(payload, fh, ensure_ascii=False, indent=2)
|
||||
|
||||
print(json.dumps(payload["decision"], ensure_ascii=False, indent=2))
|
||||
print(f"saved rollout report to {args.output}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,78 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from typing import Any, Dict
|
||||
|
||||
|
||||
ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
SCHEMA_PATH = os.path.join(ROOT_DIR, "artifacts", "models", "lgbm_daily_high_schema.json")
|
||||
|
||||
|
||||
def _load_schema(path: str) -> Dict[str, Any]:
|
||||
with open(path, "r", encoding="utf-8") as fh:
|
||||
data = json.load(fh)
|
||||
if not isinstance(data, dict):
|
||||
raise SystemExit(f"Invalid schema payload in {path}")
|
||||
return data
|
||||
|
||||
|
||||
def _fmt_metric(value: Any) -> str:
|
||||
if value is None:
|
||||
return "--"
|
||||
try:
|
||||
return f"{float(value):.3f}"
|
||||
except Exception:
|
||||
return str(value)
|
||||
|
||||
|
||||
def _winner(metrics: Dict[str, Any]) -> str:
|
||||
candidates = {
|
||||
"LGBM": metrics.get("lgbm_mae"),
|
||||
"DEB": metrics.get("deb_mae"),
|
||||
"Best Single": metrics.get("best_single_mae"),
|
||||
"Median": metrics.get("median_mae"),
|
||||
}
|
||||
filtered = {k: float(v) for k, v in candidates.items() if v is not None}
|
||||
if not filtered:
|
||||
return "--"
|
||||
return min(filtered.items(), key=lambda item: item[1])[0]
|
||||
|
||||
|
||||
def _print_block(label: str, metrics: Dict[str, Any]) -> None:
|
||||
print(label)
|
||||
print(f" Samples : {metrics.get('sample_count', 0)}")
|
||||
print(f" LGBM MAE : {_fmt_metric(metrics.get('lgbm_mae'))}")
|
||||
print(f" DEB MAE : {_fmt_metric(metrics.get('deb_mae'))}")
|
||||
print(f" Best Single : {_fmt_metric(metrics.get('best_single_mae'))}")
|
||||
print(f" Model Median : {_fmt_metric(metrics.get('median_mae'))}")
|
||||
print(f" Winner : {_winner(metrics)}")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
path = sys.argv[1] if len(sys.argv) > 1 else SCHEMA_PATH
|
||||
if not os.path.exists(path):
|
||||
raise SystemExit(f"Schema file not found: {path}")
|
||||
|
||||
schema = _load_schema(path)
|
||||
metrics = schema.get("metrics") or {}
|
||||
validation = metrics.get("validation") or {}
|
||||
full_sample = metrics.get("full_sample") or {}
|
||||
|
||||
print("LightGBM Daily High Report")
|
||||
print(f" Target : {schema.get('target', '--')}")
|
||||
print(f" Horizon : {schema.get('horizon', '--')}")
|
||||
print(f" Sample Count : {schema.get('sample_count', 0)}")
|
||||
print(f" Train Count : {schema.get('train_count', 0)}")
|
||||
print(f" Valid Count : {schema.get('validation_count', 0)}")
|
||||
print(f" Trained At : {schema.get('trained_at', '--')}")
|
||||
print("")
|
||||
_print_block("Validation", validation)
|
||||
print("")
|
||||
_print_block("Full Sample", full_sample)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,202 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from datetime import datetime
|
||||
from typing import Any, Dict, List
|
||||
|
||||
ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
|
||||
|
||||
MODEL_PATH = os.path.join(ROOT_DIR, "artifacts", "models", "lgbm_daily_high.txt")
|
||||
SCHEMA_PATH = os.path.join(ROOT_DIR, "artifacts", "models", "lgbm_daily_high_schema.json")
|
||||
|
||||
|
||||
def _mae(pairs: List[tuple[float, float]]) -> float | None:
|
||||
if not pairs:
|
||||
return None
|
||||
return round(sum(abs(pred - actual) for pred, actual in pairs) / len(pairs), 3)
|
||||
|
||||
|
||||
def _best_single_forecast(sample: Dict[str, Any]) -> float | None:
|
||||
target = float(sample["target"])
|
||||
forecasts = sample.get("forecasts") or {}
|
||||
best_value = None
|
||||
best_error = None
|
||||
for value in forecasts.values():
|
||||
try:
|
||||
numeric = float(value)
|
||||
except Exception:
|
||||
continue
|
||||
error = abs(numeric - target)
|
||||
if best_error is None or error < best_error:
|
||||
best_error = error
|
||||
best_value = numeric
|
||||
return best_value
|
||||
|
||||
|
||||
def _chronological_split(samples: List[Dict[str, Any]]) -> tuple[List[Dict[str, Any]], List[Dict[str, Any]]]:
|
||||
if len(samples) < 12:
|
||||
return samples, []
|
||||
ordered = sorted(samples, key=lambda row: (row["date"], row["city"]))
|
||||
validation_count = max(12, int(round(len(ordered) * 0.2)))
|
||||
validation_count = min(validation_count, len(ordered) - 1)
|
||||
if validation_count <= 0:
|
||||
return ordered, []
|
||||
return ordered[:-validation_count], ordered[-validation_count:]
|
||||
|
||||
|
||||
def _dataset_from_samples(samples: List[Dict[str, Any]], np: Any) -> tuple[Any, Any]:
|
||||
features = np.asarray([row["vector"] for row in samples], dtype=np.float32)
|
||||
targets = np.asarray([row["target"] for row in samples], dtype=np.float32)
|
||||
return features, targets
|
||||
|
||||
|
||||
def _train_model(
|
||||
train_samples: List[Dict[str, Any]],
|
||||
valid_samples: List[Dict[str, Any]],
|
||||
lgb: Any,
|
||||
np: Any,
|
||||
feature_names: List[str],
|
||||
) -> Any:
|
||||
train_x, train_y = _dataset_from_samples(train_samples, np)
|
||||
train_data = lgb.Dataset(train_x, label=train_y, feature_name=feature_names, free_raw_data=True)
|
||||
valid_sets = [train_data]
|
||||
valid_names = ["train"]
|
||||
|
||||
params = {
|
||||
"objective": "regression",
|
||||
"metric": "l1",
|
||||
"learning_rate": 0.05,
|
||||
"num_leaves": 15,
|
||||
"feature_fraction": 0.9,
|
||||
"bagging_fraction": 0.9,
|
||||
"bagging_freq": 1,
|
||||
"min_data_in_leaf": 4,
|
||||
"verbosity": -1,
|
||||
"seed": 42,
|
||||
}
|
||||
|
||||
callbacks = []
|
||||
if valid_samples:
|
||||
valid_x, valid_y = _dataset_from_samples(valid_samples, np)
|
||||
valid_data = lgb.Dataset(valid_x, label=valid_y, feature_name=feature_names, reference=train_data)
|
||||
valid_sets.append(valid_data)
|
||||
valid_names.append("valid")
|
||||
callbacks.append(lgb.early_stopping(stopping_rounds=15, verbose=False))
|
||||
|
||||
return lgb.train(
|
||||
params=params,
|
||||
train_set=train_data,
|
||||
num_boost_round=120,
|
||||
valid_sets=valid_sets,
|
||||
valid_names=valid_names,
|
||||
callbacks=callbacks,
|
||||
)
|
||||
|
||||
|
||||
def _evaluate(booster: Any, samples: List[Dict[str, Any]], np: Any) -> Dict[str, Any]:
|
||||
if not samples:
|
||||
return {
|
||||
"sample_count": 0,
|
||||
"lgbm_mae": None,
|
||||
"deb_mae": None,
|
||||
"best_single_mae": None,
|
||||
"median_mae": None,
|
||||
}
|
||||
|
||||
features, _ = _dataset_from_samples(samples, np)
|
||||
preds = booster.predict(features, num_iteration=booster.best_iteration)
|
||||
lgbm_pairs: List[tuple[float, float]] = []
|
||||
deb_pairs: List[tuple[float, float]] = []
|
||||
best_single_pairs: List[tuple[float, float]] = []
|
||||
median_pairs: List[tuple[float, float]] = []
|
||||
|
||||
for sample, pred in zip(samples, preds):
|
||||
actual = float(sample["target"])
|
||||
lgbm_pairs.append((float(pred), actual))
|
||||
|
||||
deb_prediction = sample.get("deb_prediction")
|
||||
if deb_prediction is not None:
|
||||
deb_pairs.append((float(deb_prediction), actual))
|
||||
|
||||
best_single = _best_single_forecast(sample)
|
||||
if best_single is not None:
|
||||
best_single_pairs.append((best_single, actual))
|
||||
|
||||
median_prediction = (sample.get("features") or {}).get("model_median")
|
||||
if median_prediction is not None:
|
||||
median_pairs.append((float(median_prediction), actual))
|
||||
|
||||
return {
|
||||
"sample_count": len(samples),
|
||||
"lgbm_mae": _mae(lgbm_pairs),
|
||||
"deb_mae": _mae(deb_pairs),
|
||||
"best_single_mae": _mae(best_single_pairs),
|
||||
"median_mae": _mae(median_pairs),
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
if ROOT_DIR not in sys.path:
|
||||
sys.path.insert(0, ROOT_DIR)
|
||||
|
||||
import lightgbm as lgb
|
||||
import numpy as np
|
||||
from src.models.lgbm_features import (
|
||||
FEATURE_NAMES,
|
||||
build_training_samples,
|
||||
schema_payload,
|
||||
)
|
||||
|
||||
samples = build_training_samples()
|
||||
if len(samples) < 16:
|
||||
raise SystemExit(f"Not enough supervised samples for LightGBM training: {len(samples)}")
|
||||
|
||||
train_samples, valid_samples = _chronological_split(samples)
|
||||
booster = _train_model(train_samples, valid_samples, lgb, np, FEATURE_NAMES)
|
||||
|
||||
all_features, all_targets = _dataset_from_samples(samples, np)
|
||||
final_train = lgb.Dataset(all_features, label=all_targets, feature_name=FEATURE_NAMES, free_raw_data=True)
|
||||
final_booster = lgb.train(
|
||||
params={
|
||||
"objective": "regression",
|
||||
"metric": "l1",
|
||||
"learning_rate": 0.05,
|
||||
"num_leaves": 15,
|
||||
"feature_fraction": 0.9,
|
||||
"bagging_fraction": 0.9,
|
||||
"bagging_freq": 1,
|
||||
"min_data_in_leaf": 4,
|
||||
"verbosity": -1,
|
||||
"seed": 42,
|
||||
},
|
||||
train_set=final_train,
|
||||
num_boost_round=max(int(booster.best_iteration or 60), 20),
|
||||
)
|
||||
|
||||
os.makedirs(os.path.dirname(MODEL_PATH), exist_ok=True)
|
||||
final_booster.save_model(MODEL_PATH)
|
||||
|
||||
metrics = {
|
||||
"validation": _evaluate(booster, valid_samples, np),
|
||||
"full_sample": _evaluate(final_booster, samples, np),
|
||||
}
|
||||
schema = schema_payload(
|
||||
model_path=os.path.relpath(MODEL_PATH, ROOT_DIR),
|
||||
sample_count=len(samples),
|
||||
train_count=len(train_samples),
|
||||
validation_count=len(valid_samples),
|
||||
metrics=metrics,
|
||||
)
|
||||
schema["trained_at"] = datetime.utcnow().isoformat() + "Z"
|
||||
with open(SCHEMA_PATH, "w", encoding="utf-8") as fh:
|
||||
json.dump(schema, fh, ensure_ascii=False, indent=2)
|
||||
|
||||
print(json.dumps({"model_path": MODEL_PATH, "schema_path": SCHEMA_PATH, "metrics": metrics}, ensure_ascii=False))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user