From dca4f2d618bee5849981482b15b71a7a79b679d1 Mon Sep 17 00:00:00 2001 From: "2569718930@qq.com" <2569718930@qq.com> Date: Tue, 16 Jun 2026 02:00:22 +0800 Subject: [PATCH] =?UTF-8?q?=E6=9E=B6=E6=9E=84=E9=87=8D=E6=9E=84=EF=BC=9A?= =?UTF-8?q?=E6=8B=86=E5=88=86=20core/ops/DBManager=EF=BC=8C=E7=BB=9F?= =?UTF-8?q?=E4=B8=80=20SQLite=20=E9=94=81=EF=BC=8CDEB=20=E6=94=B9=E8=BF=9B?= =?UTF-8?q?=EF=BC=8C=E6=96=B0=E5=A2=9E=E6=B3=A8=E6=84=8F=E5=8A=9B=E6=A8=A1?= =?UTF-8?q?=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - web/core.py 858→236行，拆出 schemas/middleware/auth/diagnostics - ops_api.py 2876→4个 domain 模块 (users/payments/health/config) - DBManager Supabase HTTP 调用提取到 SupabaseAdminClient - 新增 LockedSQLiteConnection 统一多进程读写锁 - 新增 WeatherCacheManager 替代 12 个独立缓存字典 - METAR 缓存迁移至统一缓存管理器 - analysis_service 提取 _build_intraday_meteorology 到独立模块 - DEB 改进：偏差惩罚、分歧回退、自适应 lookback (MAE ↓12.6%) - 新增 PyTorch 注意力模型 deb_attention.py (数据积累后启用) - 新增 torch 到 requirements.lock --- CLAUDE.md | 2 +- requirements.lock | 4 + src/analysis/deb_algorithm.py | 53 +- src/analysis/deb_attention.py | 276 +++ src/auth/supabase_admin_client.py | 149 ++ src/data_collection/metar_sources.py | 25 +- src/data_collection/weather_cache.py | 120 + src/data_collection/weather_sources.py | 22 +- src/database/db_manager.py | 82 +- tests/test_country_networks.py | 2 + tests/test_realtime_event_store.py | 3 +- web/analysis_service.py | 290 +-- web/auth/guards.py | 269 +++ web/core.py | 815 +------ web/diagnostics/health.py | 371 +++ web/middleware/http.py | 78 + web/realtime_event_store.py | 6 +- web/schemas/auth.py | 48 + web/schemas/payments.py | 47 + web/services/intraday_meteorology.py | 311 +++ web/services/ops/__init__.py | 1 + web/services/ops/config.py | 724 ++++++ web/services/ops/health.py | 1104 +++++++++ web/services/ops/payments.py | 1025 +++++++++ web/services/ops/users.py | 213 ++ web/services/ops_api.py | 2944 +----------------------- 26 files changed, 5010 insertions(+), 3974 deletions(-) create mode 100644 src/analysis/deb_attention.py create mode 100644 src/auth/supabase_admin_client.py create mode 100644 src/data_collection/weather_cache.py create mode 100644 web/auth/guards.py create mode 100644 web/diagnostics/health.py create mode 100644 web/middleware/http.py create mode 100644 web/schemas/auth.py create mode 100644 web/schemas/payments.py create mode 100644 web/services/intraday_meteorology.py create mode 100644 web/services/ops/__init__.py create mode 100644 web/services/ops/config.py create mode 100644 web/services/ops/health.py create mode 100644 web/services/ops/payments.py create mode 100644 web/services/ops/users.py diff --git a/CLAUDE.md b/CLAUDE.md index 0df354cf..4e6a5887 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -6,7 +6,7 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co PolyWeather Pro — a paid institutional weather-intelligence terminal. 50 monitored cities with real-time METAR/AMOS/MADIS observations, DEB multi-model temperature blending, Mu probability calibration, and intraday bias correction. Pure meteorological decision workspace; no market/price layer. Next.js 15 + React 19 frontend (Docker / VPS, behind Cloudflare + Nginx), FastAPI backend (VPS), Telegram bot. -**Business model**: Paid-only, $10/month, no free tier, no trial. Landing page is public; `/terminal` requires login + active subscription. +**Business model**: Paid-only, 29.9 USDC/month or 79.9 USDC/quarter, referral first month 20 USDC. New users get a one-time 3-day trial. Landing page is public; `/terminal` requires login + active subscription. ## Environment & Preferences diff --git a/requirements.lock b/requirements.lock index ea53ce45..571ec7f2 100644 --- a/requirements.lock +++ b/requirements.lock @@ -109,6 +109,10 @@ multidict==6.7.1 netcdf4==1.7.4 # via -r requirements.txt numpy==2.4.6 + +torch @ https://download.pytorch.org/whl/cpu/torch-2.5.1%2Bcpu-cp311-cp311-manylinux_2_28_x86_64.whl ; python_version >= "3.11" +typing_extensions==4.15.0 + # via torch # via # -r requirements.txt # cftime diff --git a/src/analysis/deb_algorithm.py b/src/analysis/deb_algorithm.py index 72accf46..291f20a1 100644 --- a/src/analysis/deb_algorithm.py +++ b/src/analysis/deb_algorithm.py @@ -1159,11 +1159,24 @@ def calculate_dynamic_weight_components( city_data = data[city_name] sorted_dates = sorted(city_data.keys(), reverse=True) + today_str = datetime.now().strftime("%Y-%m-%d") + available_days = sum( + 1 for d in sorted_dates + if d != today_str and city_data[d].get("actual_high") is not None + ) + + # ── 改进3: 自适应 lookback — 数据多的城市用更多历史 ── + adaptive_lookback = min(14, max(7, available_days // 4)) + effective_lookback = max(lookback_days, adaptive_lookback) if lookback_days <= 7 else lookback_days + + # ── 改进1: 偏差惩罚 — per-model signed bias ── + model_biases: dict = {model: 0.0 for model in forecasts.keys()} + bias_samples: dict = {model: 0 for model in forecasts.keys()} errors: dict = {model: [] for model in forecasts.keys()} days_used = 0 for date_str in sorted_dates: - if date_str == datetime.now().strftime("%Y-%m-%d"): + if date_str == today_str: continue record = city_data[date_str] @@ -1174,8 +1187,6 @@ def calculate_dynamic_weight_components( if actual is None: continue - decay_weight = decay_factor ** days_used - for model in forecasts.keys(): if model in past_forecasts and past_forecasts[model] is not None: try: @@ -1183,6 +1194,10 @@ def calculate_dynamic_weight_components( av = float(actual) except (TypeError, ValueError): continue + # Track signed error for bias + model_biases[model] += (pv - av) # positive = model overpredicts + bias_samples[model] += 1 + # Track absolute error for MAE daily_error = abs(pv - av) h_err = ( past_hourly_error.get(model) @@ -1190,15 +1205,17 @@ def calculate_dynamic_weight_components( else None ) blended_error = _blend_mae(daily_error, h_err) + decay_weight = decay_factor ** days_used errors[model].append((blended_error, decay_weight)) days_used += 1 - if days_used >= lookback_days: + if days_used >= effective_lookback: break if days_used < 2: return _equal_weight_result(f"等权平均(由于仅{days_used}天历史)", days_used) + # Compute weighted MAEs maes = {} for model, err_weighted in errors.items(): if err_weighted: @@ -1210,8 +1227,14 @@ def calculate_dynamic_weight_components( else: maes[model] = 2.0 + # Finalize per-model biases — only trust when enough samples + for model in forecasts.keys(): + n = bias_samples.get(model, 0) + model_biases[model] = model_biases[model] / n if n >= 5 else 0.0 + + # ── 改进1: 偏差惩罚进逆误差 ── inverse_errors = { - m: 1.0 / (mae + 0.1) + m: 1.0 / (mae + abs(model_biases[m]) * 0.5 + 0.1) for m, mae in maes.items() if forecasts.get(m) is not None } @@ -1229,12 +1252,30 @@ def calculate_dynamic_weight_components( } weights = {m: inv / total_inv for m, inv in inverse_errors.items()} + + # ── 改进2: 分歧感知的权重回退 ── + forecast_values = [v for v in forecasts.values() if v is not None] + if len(forecast_values) >= 2: + spread = max(forecast_values) - min(forecast_values) + if spread > 3.0: + trust_factor = 3.0 / spread # 分歧>3°F时才回退 + n = len(weights) + weights = { + m: w * trust_factor + (1.0 - trust_factor) / n + for m, w in weights.items() + } + blended_high = sum(forecasts[m] * weights[m] for m in weights) sorted_models = sorted(weights.items(), key=lambda x: x[1], reverse=True) weight_str_parts = [] for m, w in sorted_models[:3]: - weight_str_parts.append(f"{m}({w * 100:.0f}%,MAE:{maes[m]:.1f}°)") + parts = [f"{m}({w * 100:.0f}%"] + if maes.get(m): + parts.append(f"MAE:{maes[m]:.1f}°") + if abs(model_biases.get(m, 0.0)) >= 0.3: + parts.append(f"bias:{model_biases[m]:+.1f}") + weight_str_parts.append(",".join(parts) + ")") if dedup_note: weight_str_parts.append(dedup_note) diff --git a/src/analysis/deb_attention.py b/src/analysis/deb_attention.py new file mode 100644 index 00000000..b043db4f --- /dev/null +++ b/src/analysis/deb_attention.py @@ -0,0 +1,276 @@ +"""PyTorch learnable attention ensemble for DEB. + +Tiny MLP (~5K params) that learns per-city model weighting from +cross-city training data. Trains in seconds on CPU, <1ms inference. + +Usage: + python -m src.analysis.deb_attention --train + python -m src.analysis.deb_attention --eval +""" + +from __future__ import annotations + +import json +import math +import os +import random +from datetime import datetime +from typing import Any, Dict, List, Optional, Tuple + +import torch +import torch.nn as nn +import torch.nn.functional as F + + +def _load_training_data(history, min_models=3): + samples = [] + for city, days in (history or {}).items(): + for date_str, record in days.items(): + forecasts = record.get("forecasts", {}) + actual = record.get("actual_high") + if actual is None: + continue + valid = {m: float(v) for m, v in forecasts.items() if v is not None} + if len(valid) < min_models: + continue + samples.append({"city": city, "forecasts": valid, "actual": float(actual), "date": date_str}) + return samples + + +def _build_vocabs(samples, min_model_freq=2): + from collections import Counter + mf = Counter[str]() + cities = set() + for s in samples: + cities.add(s["city"]) + for m in s["forecasts"]: + mf[m] += 1 + mv = {m: i for i, m in enumerate(sorted([m for m, c in mf.items() if c >= min_model_freq], key=lambda m: (-mf[m], m)))} + cv = {c: i for i, c in enumerate(sorted(cities))} + return mv, cv + + +class DebAttention(nn.Module): + """Learnable attention: per-model scorer + city bias correction. + + Input: per-model forecast values + city id + Output: weighted prediction (learned attention weights) + city bias + """ + + def __init__(self, n_models, n_cities, d_hidden=32, dropout=0.1): + super().__init__() + self.n_models = n_models + self.n_cities = n_cities + # City embedding for context + self.city_emb = nn.Embedding(n_cities + 1, 16, padding_idx=0) + # Per-model scorer: [fc_norm, city_emb] → attention score + self.scorer = nn.Sequential( + nn.Linear(3 + 16, d_hidden), nn.GELU(), nn.Dropout(dropout), + nn.Linear(d_hidden, d_hidden // 2), nn.GELU(), + nn.Linear(d_hidden // 2, 1), + ) + # City-level bias + self.city_bias = nn.Embedding(n_cities + 1, 1, padding_idx=0) + self.register_buffer("fc_mean", torch.tensor(0.0)) + self.register_buffer("fc_std", torch.tensor(10.0)) + + def forward(self, forecasts, city_idx, mask): + """(B, M), (B,), (B, M) → (B,), (B, M)""" + batch, max_m = forecasts.shape + fc_norm = (forecasts - self.fc_mean) / self.fc_std.clamp_min(0.1) + + # Consensus: trimmed mean of valid forecasts + fc_masked = forecasts * mask.float() + n_valid = mask.sum(dim=1, keepdim=True).clamp_min(1) + consensus = fc_masked.sum(dim=1) / n_valid.squeeze() # (B,) + + # Deviation from consensus + dev = fc_norm - consensus.unsqueeze(1) * mask.float() / self.fc_std.clamp_min(0.1) + + # Feature: [fc_norm, dev_from_consensus, is_extreme] + is_extreme = (dev.abs() > 1.5).float() # flag if >1.5 std away + feats = torch.stack([fc_norm, dev, is_extreme], dim=-1) # (B, M, 3) + + city = self.city_emb(city_idx.clamp(0, self.n_cities)) # (B, 16) + city_tiled = city.unsqueeze(1).expand(-1, max_m, -1) # (B, M, 16) + feats = torch.cat([feats, city_tiled], dim=-1) # (B, M, 19) + + scores = self.scorer(feats).squeeze(-1) # (B, M) + scores = scores.masked_fill(~mask, float("-inf")) + attn = F.softmax(scores, dim=-1) + pred = (forecasts * attn).sum(dim=-1) + bias = self.city_bias(city_idx.clamp(0, self.n_cities)).squeeze(-1) + pred = pred + bias * 0.3 + return pred, attn + + +def train_model(history, *, epochs=200, batch_size=128, lr=0.002, patience=30, checkpoint_dir="data", seed=42): + torch.manual_seed(seed) + random.seed(seed) + + samples = _load_training_data(history) + if len(samples) < 50: + raise ValueError(f"Need >=50 samples, got {len(samples)}") + mv, cv = _build_vocabs(samples) + max_models = len(mv) + 4 + + samples.sort(key=lambda s: s["date"]) + split = int(len(samples) * 0.8) + train_s = samples[:split] + val_s = samples[split:] + + all_fc = [v for s in train_s for v in s["forecasts"].values()] + fc_mean = sum(all_fc) / len(all_fc) + fc_std = max(1.0, math.sqrt(sum((v - fc_mean) ** 2 for v in all_fc) / len(all_fc))) + + def precompute(s_list): + X_fc, X_cid, X_mask, Y = [], [], [], [] + for s in s_list: + fc_list = [s["forecasts"][m] for m in sorted(s["forecasts"]) if m in mv] + n = len(fc_list) + X_fc.append(fc_list + [0.0] * (max_models - n)) + X_cid.append(cv.get(s["city"], 0)) + X_mask.append([1.0] * n + [0.0] * (max_models - n)) + Y.append(s["actual"]) + return ( + torch.tensor(X_fc, dtype=torch.float32), + torch.tensor(X_cid, dtype=torch.long), + torch.tensor(X_mask, dtype=torch.bool), + torch.tensor(Y, dtype=torch.float32), + ) + + train_X = precompute(train_s) + val_X = precompute(val_s) + train_loader = torch.utils.data.DataLoader(torch.utils.data.TensorDataset(*train_X), batch_size=batch_size, shuffle=True) + val_loader = torch.utils.data.DataLoader(torch.utils.data.TensorDataset(*val_X), batch_size=batch_size * 2, shuffle=False) + + model = DebAttention(max_models, len(cv) + 1) + model.fc_mean.fill_(fc_mean) + model.fc_std.fill_(fc_std) + + opt = torch.optim.AdamW(model.parameters(), lr=lr, weight_decay=1e-5) + sched = torch.optim.lr_scheduler.ReduceLROnPlateau(opt, mode="min", factor=0.5, patience=10) + + best_val = float("inf") + best_state = None + no_imp = 0 + + print(f"Training: {len(train_s)} train, {len(val_s)} val, models={len(mv)}, cities={len(cv)}") + + for epoch in range(epochs): + model.train() + tr_loss = 0.0 + for fc, cid, mask, y in train_loader: + pred, _ = model(fc, cid, mask) + loss = F.mse_loss(pred, y) + opt.zero_grad() + loss.backward() + opt.step() + tr_loss += loss.item() * fc.size(0) + tr_loss /= len(train_X[0]) + + model.eval() + val_loss = 0.0 + with torch.no_grad(): + for fc, cid, mask, y in val_loader: + pred, _ = model(fc, cid, mask) + loss = F.mse_loss(pred, y) + val_loss += loss.item() * fc.size(0) + val_loss /= len(val_X[0]) + + sched.step(val_loss) + if val_loss < best_val: + best_val = val_loss + best_state = {k: v.cpu().clone() for k, v in model.state_dict().items()} + no_imp = 0 + else: + no_imp += 1 + + if epoch % 20 == 0 or epoch == epochs - 1: + print(f" epoch {epoch:3d} tr={math.sqrt(tr_loss):.3f} val={math.sqrt(val_loss):.3f} lr={opt.param_groups[0]['lr']:.1e}") + + if no_imp >= patience: + print(f" early stop {epoch}") + break + + if best_state: + model.load_state_dict(best_state) + + os.makedirs(checkpoint_dir, exist_ok=True) + ckpt = os.path.join(checkpoint_dir, "deb_attention.pt") + torch.save({"state": model.state_dict(), "mv": mv, "cv": cv, "fc_mean": fc_mean, "fc_std": fc_std, "max_m": max_models}, ckpt) + print(f" saved {ckpt}") + return model, mv, cv + + +def load_model(path): + ckpt = torch.load(path, map_location="cpu", weights_only=False) + model = DebAttention(ckpt["max_m"], len(ckpt["cv"]) + 1) + model.load_state_dict(ckpt["state"]) + model.fc_mean.fill_(ckpt["fc_mean"]) + model.fc_std.fill_(ckpt["fc_std"]) + model.eval() + return model, ckpt["mv"], ckpt["cv"] + + +def predict(model, forecasts, city, mv, cv): + model.eval() + fc_list = [forecasts[m] for m in sorted(forecasts) if m in mv] + if not fc_list: + return None, {} + n = len(fc_list) + max_m = model.n_models + fc_t = torch.zeros(max_m) + mask_t = torch.zeros(max_m, dtype=torch.bool) + fc_t[:n] = torch.tensor(fc_list) + mask_t[:n] = True + cid = cv.get(city, 0) + with torch.no_grad(): + pred, attn = model(fc_t.unsqueeze(0), torch.tensor([cid]), mask_t.unsqueeze(0)) + pred = round(pred.item(), 1) + models = [m for m in sorted(forecasts) if m in mv] + weights = {models[i]: attn[0, i].item() for i in range(n)} + total = sum(weights.values()) + if total > 0: + weights = {m: w / total for m, w in weights.items()} + return pred, weights + + +if __name__ == "__main__": + import sys, statistics + from src.analysis.deb_algorithm import load_history as lh, get_deb_accuracy, calculate_dynamic_weights + from src.data_collection.city_registry import CITY_REGISTRY + + root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + hp = os.path.join(root, "data", "daily_records.json") + history = lh(hp) + + if "--train" in sys.argv: + m, mv, cv = train_model(history) + print("Done.") + + elif "--eval" in sys.argv: + ckpt = os.path.join(root, "data", "deb_attention.pt") + if not os.path.exists(ckpt): + print("No checkpoint. Run --train first.") + sys.exit(1) + m, mv, cv = load_model(ckpt) + base, attn_errs = [], [] + for city in [c for c in CITY_REGISTRY if c in history]: + cd = history.get(city, {}) + for d in sorted(cd.keys(), reverse=True)[:5]: + rec = cd[d]; fc = rec.get("forecasts", {}); actual = rec.get("actual_high") + if not fc or actual is None: continue + try: + bp, _ = calculate_dynamic_weights(city, fc) + if bp: base.append(abs(bp - float(actual))) + except: pass + try: + ap, _ = predict(m, fc, city, mv, cv) + if ap: attn_errs.append(abs(ap - float(actual))) + except: pass + if base and attn_errs: + print(f"Baseline MAE: {statistics.mean(base):.3f}") + print(f"Attention MAE: {statistics.mean(attn_errs):.3f}") + else: + print("python -m src.analysis.deb_attention --train | --eval") diff --git a/src/auth/supabase_admin_client.py b/src/auth/supabase_admin_client.py new file mode 100644 index 00000000..c4672182 --- /dev/null +++ b/src/auth/supabase_admin_client.py @@ -0,0 +1,149 @@ +"""Thin Supabase Admin HTTP client — extracted from DBManager. + +Handles URL construction, auth headers, and the actual HTTP PATCH calls +to Supabase REST / Admin APIs. Cache-cooldown and local SQLite lookups +remain in DBManager. +""" + +from __future__ import annotations + +import os +from typing import Any, Dict, Optional + +import requests +from loguru import logger + + +class SupabaseAdminClient: + """Stateless HTTP client for Supabase Management API calls.""" + + def __init__(self) -> None: + self._supabase_url = str(os.getenv("SUPABASE_URL") or "").strip().rstrip("/") + self._service_role_key = str( + os.getenv("SUPABASE_SERVICE_ROLE_KEY") or "" + ).strip() + + @property + def configured(self) -> bool: + return bool(self._supabase_url and self._service_role_key) + + # ------------------------------------------------------------------ + # URL builders + # ------------------------------------------------------------------ + + def profiles_endpoint(self) -> str: + if not self._supabase_url: + return "" + return f"{self._supabase_url}/rest/v1/profiles" + + def admin_users_endpoint(self) -> str: + if not self._supabase_url: + return "" + return f"{self._supabase_url}/auth/v1/admin/users" + + # ------------------------------------------------------------------ + # Headers + # ------------------------------------------------------------------ + + def _service_headers(self, *, prefer: str = "return=minimal") -> Dict[str, str]: + return { + "apikey": self._service_role_key, + "Authorization": f"Bearer {self._service_role_key}", + "Accept": "application/json", + "Content-Type": "application/json", + "Prefer": prefer, + } + + # ------------------------------------------------------------------ + # Operations + # ------------------------------------------------------------------ + + def patch_user_metadata( + self, + supabase_user_id: str, + metadata: Dict[str, Any], + *, + timeout: float = 8.0, + ) -> bool: + """PATCH /auth/v1/admin/users/{id} with user_metadata.""" + if not self.configured: + return False + endpoint = self.admin_users_endpoint() + if not endpoint: + return False + + try: + resp = requests.patch( + f"{endpoint}/{supabase_user_id}", + json={"user_metadata": metadata}, + headers=self._service_headers(), + timeout=timeout, + ) + ok = resp.status_code in (200, 204) + if not ok: + logger.warning( + "supabase admin patch user_metadata failed " + "user_id={} status={} body={}", + supabase_user_id, + resp.status_code, + (resp.text or "")[:200], + ) + return ok + except Exception as exc: + logger.warning( + "supabase admin patch user_metadata error user_id={}: {}", + supabase_user_id, + exc, + ) + return False + + def patch_profile( + self, + supabase_user_id: str, + fields: Dict[str, Any], + *, + timeout: float = 8.0, + ) -> bool: + """PATCH /rest/v1/profiles?id=eq.{user_id} with telegram fields.""" + if not self.configured: + return False + endpoint = self.profiles_endpoint() + if not endpoint: + return False + + try: + resp = requests.patch( + endpoint, + params={"id": f"eq.{supabase_user_id}"}, + json=fields, + headers=self._service_headers(), + timeout=timeout, + ) + ok = resp.status_code in (200, 204) + if not ok: + logger.warning( + "supabase profiles patch failed " + "user_id={} status={} body={}", + supabase_user_id, + resp.status_code, + (resp.text or "")[:240], + ) + return ok + except Exception as exc: + logger.warning( + "supabase profiles patch error user_id={}: {}", + supabase_user_id, + exc, + ) + return False + + +# Module-level singleton — mirrors the pattern used by DBManager. +_supabase_admin_client: Optional[SupabaseAdminClient] = None + + +def get_supabase_admin_client() -> SupabaseAdminClient: + global _supabase_admin_client + if _supabase_admin_client is None: + _supabase_admin_client = SupabaseAdminClient() + return _supabase_admin_client diff --git a/src/data_collection/metar_sources.py b/src/data_collection/metar_sources.py index f611f266..13b7ced6 100644 --- a/src/data_collection/metar_sources.py +++ b/src/data_collection/metar_sources.py @@ -97,12 +97,11 @@ class MetarSourceMixin: cache_key = f"{icao}:{utc_offset}:{use_fahrenheit}" now_ts = time.time() cache_ttl_sec = self._metar_cache_ttl_for_city(city, icao) - with self._metar_cache_lock: - cached = self._metar_cache.get(cache_key) - if cached and now_ts - cached["t"] < cache_ttl_sec: - logger.debug(f"METAR cache hit {icao} age={int(now_ts - cached['t'])}s") - record_source_call("metar", "current", "cache_hit", (time.perf_counter() - started) * 1000.0) - return cached["d"] + cached = self.cache.get_ttl("metar", cache_key, cache_ttl_sec) + if cached is not None: + logger.debug(f"METAR cache hit {icao} age=...") + record_source_call("metar", "current", "cache_hit", (time.perf_counter() - started) * 1000.0) + return cached try: url = "https://aviationweather.gov/api/data/metar" @@ -331,19 +330,17 @@ class MetarSourceMixin: ) else: logger.info(f"✈️ METAR {icao}: temperature unavailable (obs: {obs_time})") - with self._metar_cache_lock: - self._metar_cache[cache_key] = {"d": result, "t": now_ts} + self.cache.set_ttl("metar", cache_key, result) record_source_call("metar", "current", "success", (time.perf_counter() - started) * 1000.0) return result except httpx.HTTPError as exc: logger.error(f"METAR 请求失败 ({icao}): {exc}") - with self._metar_cache_lock: - stale = self._metar_cache.get(cache_key) - if stale: - logger.warning(f"METAR {icao} 请求失败，使用缓存回退") - record_source_call("metar", "current", "stale_cache", (time.perf_counter() - started) * 1000.0) - return stale["d"] + stale = self.cache.get("metar", cache_key) + if stale: + logger.warning(f"METAR {icao} 请求失败，使用缓存回退") + record_source_call("metar", "current", "stale_cache", (time.perf_counter() - started) * 1000.0) + return stale.get("d") if isinstance(stale, dict) else None record_source_call("metar", "current", "error", (time.perf_counter() - started) * 1000.0) return None except (KeyError, IndexError, TypeError) as exc: diff --git a/src/data_collection/weather_cache.py b/src/data_collection/weather_cache.py new file mode 100644 index 00000000..d159022a --- /dev/null +++ b/src/data_collection/weather_cache.py @@ -0,0 +1,120 @@ +"""Thread-safe TTL cache manager for weather data collector sources. + +Replaces the per-source cache dictionaries and locks scattered across +WeatherDataCollector Mixins with a single unified store. +""" + +from __future__ import annotations + +import threading +import time +from typing import Any, Callable, Dict, Optional + + +class WeatherCacheManager: + """Key-value store with per-entry TTL and automatic trimming.""" + + def __init__(self, trim_interval_writes: int = 200) -> None: + self._lock = threading.Lock() + self._stores: Dict[str, Dict[str, Dict[str, Any]]] = {} + self._write_count = 0 + self._trim_interval = max(1, int(trim_interval_writes)) + + # ------------------------------------------------------------------ + # Public API + # ------------------------------------------------------------------ + + def get(self, store: str, key: str) -> Optional[Dict[str, Any]]: + with self._lock: + entries = self._stores.get(store) + if entries is None: + return None + return entries.get(key) + + def set(self, store: str, key: str, value: Dict[str, Any]) -> None: + with self._lock: + if store not in self._stores: + self._stores[store] = {} + self._stores[store][key] = value + self._write_count += 1 + self._maybe_trim() + + def get_ttl( + self, + store: str, + key: str, + ttl_sec: float, + ) -> Optional[Dict[str, Any]]: + """Return the cached value if it exists and is younger than *ttl_sec*.""" + now = time.time() + with self._lock: + entries = self._stores.get(store) + if entries is None: + return None + entry = entries.get(key) + if entry is None: + return None + cached_ts = float(entry.get("_ts") or 0) + if now - cached_ts > ttl_sec: + return None + return entry.get("d") + + def set_ttl(self, store: str, key: str, value: Dict[str, Any]) -> None: + """Store a value with an automatic timestamp for TTL lookups.""" + self.set(store, key, {"_ts": time.time(), "d": value}) + + def store_size(self, store: str) -> int: + with self._lock: + entries = self._stores.get(store) + return len(entries) if entries else 0 + + # ------------------------------------------------------------------ + # Trimming + # ------------------------------------------------------------------ + + def _maybe_trim(self) -> None: + if self._write_count % self._trim_interval != 0: + return + self.trim_stale() + + def trim_stale( + self, + ttl_map: Optional[Dict[str, float]] = None, + ) -> None: + """Remove entries older than their store's TTL. + + *ttl_map* maps store name → ttl_sec. Stores not in the map are skipped. + """ + if not ttl_map: + return + now = time.time() + with self._lock: + for store_name, ttl_sec in ttl_map.items(): + entries = self._stores.get(store_name) + if not entries: + continue + stale = [ + key + for key, entry in entries.items() + if now - float(entry.get("_ts") or 0) > ttl_sec + ] + for key in stale: + del entries[key] + + def clear_store(self, store: str) -> None: + with self._lock: + self._stores.pop(store, None) + + +# Module-level singleton for the default cache used by WeatherDataCollector. +_weather_cache: Optional[WeatherCacheManager] = None +_weather_cache_lock = threading.Lock() + + +def get_weather_cache() -> WeatherCacheManager: + global _weather_cache + if _weather_cache is None: + with _weather_cache_lock: + if _weather_cache is None: + _weather_cache = WeatherCacheManager() + return _weather_cache diff --git a/src/data_collection/weather_sources.py b/src/data_collection/weather_sources.py index a1a2f9b7..f57ff3f4 100644 --- a/src/data_collection/weather_sources.py +++ b/src/data_collection/weather_sources.py @@ -212,8 +212,7 @@ class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSour os.getenv("METAR_FAST_CACHE_TTL_SEC", str(OBSERVATION_REFRESH_SEC)) ) self.metar_fast_cache_ttl_sec = min(self.metar_fast_cache_ttl_sec, OBSERVATION_REFRESH_SEC) - self._metar_cache: Dict[str, Dict] = {} - self._metar_cache_lock = threading.Lock() + # METAR cache migrated to self.cache (WeatherCacheManager) self.taf_cache_ttl_sec = int( os.getenv("TAF_CACHE_TTL_SEC", "900") ) @@ -300,6 +299,11 @@ class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSour self._CACHE_TRIM_EVERY_N_WRITES = int( os.getenv("POLYWEATHER_CACHE_TRIM_INTERVAL", "200") ) + # Unified cache manager — new sources should use this instead of per-source dicts. + from src.data_collection.weather_cache import WeatherCacheManager + self.cache = WeatherCacheManager( + trim_interval_writes=self._CACHE_TRIM_EVERY_N_WRITES, + ) # 设置代理 proxy = config.get("proxy") @@ -344,7 +348,7 @@ class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSour (self._open_meteo_cache, self._open_meteo_cache_lock, 3600.0), (self._ensemble_cache, self._ensemble_cache_lock, 7200.0), (self._multi_model_cache, self._multi_model_cache_lock, 7200.0), - (self._metar_cache, self._metar_cache_lock, float(self.metar_cache_ttl_sec * 2)), + # metar cache migrated to self.cache (self._taf_cache, self._taf_cache_lock, float(self.taf_cache_ttl_sec * 2)), (self._jma_cache, self._jma_cache_lock, float(self.jma_cache_ttl_sec * 2)), (self._settlement_cache, self._settlement_cache_lock, float(self.settlement_cache_ttl_sec * 2)), @@ -367,6 +371,8 @@ class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSour with lock: for key in stale: cache.pop(key, None) + # Unified cache stores + self.cache.trim_stale({"metar": float(self.metar_cache_ttl_sec * 2)}) def _post_temperature_patch_payload( self, @@ -960,10 +966,12 @@ class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSour icao = self.get_icao_code(city) if icao: prefix = f"{icao}:" - with self._metar_cache_lock: - for key in list(self._metar_cache.keys()): - if key.startswith(prefix): - self._metar_cache.pop(key, None) + # Clear matching entries from unified cache + with self.cache._lock: + entries = self.cache._stores.get("metar", {}) + stale = [k for k in entries if k.startswith(prefix)] + for key in stale: + del entries[key] normalized = str(city or "").strip().lower() with self._jma_cache_lock: self._jma_cache.pop(f"{normalized}:{use_fahrenheit}", None) diff --git a/src/database/db_manager.py b/src/database/db_manager.py index f34a4f80..2455ca9b 100644 --- a/src/database/db_manager.py +++ b/src/database/db_manager.py @@ -14,6 +14,7 @@ import requests from loguru import logger from src.database.sqlite_connection import connect_sqlite +from src.auth.supabase_admin_client import get_supabase_admin_client class DBManager: @@ -54,28 +55,16 @@ class DBManager: self._initialized_paths.add(cache_key) def _supabase_profiles_endpoint(self) -> str: - supabase_url = str(os.getenv("SUPABASE_URL") or "").strip().rstrip("/") - if not supabase_url: - return "" - return f"{supabase_url}/rest/v1/profiles" + return get_supabase_admin_client().profiles_endpoint() def _supabase_service_headers(self) -> Dict[str, str]: - service_role_key = str(os.getenv("SUPABASE_SERVICE_ROLE_KEY") or "").strip() - if not service_role_key: + client = get_supabase_admin_client() + if not client.configured: return {} - return { - "apikey": service_role_key, - "Authorization": f"Bearer {service_role_key}", - "Accept": "application/json", - "Content-Type": "application/json", - "Prefer": "return=minimal", - } + return client._service_headers() def _supabase_admin_users_endpoint(self) -> str: - supabase_url = str(os.getenv("SUPABASE_URL") or "").strip().rstrip("/") - if not supabase_url: - return "" - return f"{supabase_url}/auth/v1/admin/users" + return get_supabase_admin_client().admin_users_endpoint() def _profile_sync_min_interval_sec(self) -> float: raw = str( @@ -249,35 +238,16 @@ class DBManager: ): return False - try: - resp = requests.patch( - f"{endpoint}/{supabase_user_id}", - json={"user_metadata": {"points": points}}, - headers={**headers, "Prefer": "return=minimal"}, - timeout=8, - ) - if resp.status_code not in (200, 204): - logger.warning( - "supabase points sync failed tg={} suid={} status={} body={}", - telegram_id, - supabase_user_id, - resp.status_code, - (resp.text or "")[:200], - ) - return False + admin = get_supabase_admin_client() + if not admin.configured: + return False + ok = admin.patch_user_metadata(supabase_user_id, {"points": points}) + if ok: self._remember_points_metadata_sync( telegram_id=int(telegram_id), points=points, ) - return True - except Exception as exc: - logger.warning( - "supabase points sync error tg={} suid={}: {}", - telegram_id, - supabase_user_id, - exc, - ) - return False + return ok def _sync_supabase_profile_telegram_fields( self, @@ -305,35 +275,15 @@ class DBManager: "telegram_username": str(telegram_username or "").strip() or None, "updated_at": datetime.now().isoformat(), } - try: - response = requests.patch( - endpoint, - params={"id": f"eq.{normalized_uid}"}, - json=payload, - headers=headers, - timeout=8, - ) - if response.status_code not in {200, 204}: - logger.warning( - "supabase profiles telegram sync failed user_id={} status={} body={}", - normalized_uid, - response.status_code, - (response.text or "")[:240], - ) - return False + admin = get_supabase_admin_client() + ok = admin.patch_profile(normalized_uid, payload) + if ok: self._remember_profile_sync( supabase_user_id=normalized_uid, telegram_id=telegram_id, telegram_username=telegram_username, ) - return True - except Exception as exc: - logger.warning( - "supabase profiles telegram sync error user_id={}: {}", - normalized_uid, - exc, - ) - return False + return ok def _sync_bound_supabase_profiles_for_telegram( self, diff --git a/tests/test_country_networks.py b/tests/test_country_networks.py index 9db5167b..bb07258f 100644 --- a/tests/test_country_networks.py +++ b/tests/test_country_networks.py @@ -28,6 +28,8 @@ class _DummyMetarSource(MetarSourceMixin): def __init__(self): self._metar_cache = {} self._metar_cache_lock = threading.Lock() + from src.data_collection.weather_cache import WeatherCacheManager + self.cache = WeatherCacheManager() self.metar_timeout_sec = 0.0 self.metar_latest_timeout_sec = 0.0 self.timeout = 0.0 diff --git a/tests/test_realtime_event_store.py b/tests/test_realtime_event_store.py index e654221c..8b2e4ab9 100644 --- a/tests/test_realtime_event_store.py +++ b/tests/test_realtime_event_store.py @@ -2,6 +2,7 @@ import sqlite3 from datetime import datetime, timezone from src.database.db_manager import DBManager +from src.database.sqlite_connection import connect_sqlite from web.realtime_event_store import RealtimeEventStore from web.realtime_patch_schema import normalize_observation_patch @@ -75,7 +76,7 @@ def test_event_store_cleanup_uses_short_replay_retention(tmp_path): old_event = store.append_event(_event("taipei", 30.0)) fresh_event = store.append_event(_event("taipei", 31.0)) - with sqlite3.connect(db_path) as conn: + with connect_sqlite(db_path) as conn: conn.execute( "UPDATE observation_patch_events SET created_at = ? WHERE revision = ?", ("2026-05-26T00:00:00+00:00", old_event["revision"]), diff --git a/web/analysis_service.py b/web/analysis_service.py index 14488f71..d4b8ca81 100644 --- a/web/analysis_service.py +++ b/web/analysis_service.py @@ -46,14 +46,7 @@ from web.services.observation_freshness import ( build_observation_freshness as _build_observation_freshness, observation_age_min as _observation_age_min, ) -from web.services.analysis_utils import ( - add_signal as _add_signal, - bucket_label as _bucket_label, - bucket_label_from_value as _bucket_label_from_value, - format_clock_minutes as _format_clock_minutes, - next_observation_clock as _next_observation_clock, - top_probability_bucket as _top_probability_bucket, -) +from web.services.intraday_meteorology import build_intraday_meteorology as _build_intraday_meteorology from web.services.analysis_signals import ( _build_deviation_monitor, _build_taf_signal, @@ -403,287 +396,6 @@ def _set_cached_summary(city: str, payload: Dict[str, Any]) -> None: -def _build_intraday_meteorology(data: Dict[str, Any]) -> Dict[str, Any]: - """Build a paid-product intraday meteorology read from existing layers.""" - current = data.get("current") or {} - probabilities = data.get("probabilities") or {} - distribution = probabilities.get("distribution") or [] - top_bucket = _top_probability_bucket(distribution) - unit = str(data.get("temp_symbol") or "°C") - deb = data.get("deb") or {} - peak = data.get("peak") or {} - deviation = data.get("deviation_monitor") or {} - taf_signal = ( - ((data.get("taf") or {}).get("signal") or {}) - if isinstance(data.get("taf"), dict) - else {} - ) - vertical = data.get("vertical_profile_signal") or {} - - current_temp = _sf(current.get("temp")) - max_so_far = _sf(current.get("max_so_far")) - deb_prediction = _sf(deb.get("prediction")) - base_value = _sf(top_bucket.get("value")) if isinstance(top_bucket, dict) else None - if base_value is None: - base_value = deb_prediction - if base_value is None: - base_value = max_so_far if max_so_far is not None else current_temp - - base_case_bucket = _bucket_label(top_bucket, unit) or _bucket_label_from_value(base_value, unit) - upside_bucket = _bucket_label_from_value(base_value + 1.0, unit) if base_value is not None else None - downside_bucket = _bucket_label_from_value(base_value - 1.0, unit) if base_value is not None else None - - signals: list = [] - support_score = 0 - suppress_score = 0 - available_layers = 0 - - direction = str(deviation.get("direction") or "").lower() - severity = str(deviation.get("severity") or "normal").lower() - delta = _sf(deviation.get("current_delta")) - if direction: - available_layers += 1 - strength = "strong" if severity == "strong" else ("medium" if severity == "light" else "weak") - if direction == "hot": - support_score += 2 if strength == "strong" else 1 - _add_signal( - signals, - label="日内节奏", - label_en="Intraday pace", - direction="support", - strength=strength, - summary=f"实测较预期路径偏高 {abs(delta or 0):.1f}{unit}，峰值仍有上修空间。", - summary_en=f"Observed temperature is running {abs(delta or 0):.1f}{unit} above the expected path; the peak still has upside room.", - ) - elif direction == "cold": - suppress_score += 2 if strength == "strong" else 1 - _add_signal( - signals, - label="日内节奏", - label_en="Intraday pace", - direction="suppress", - strength=strength, - summary=f"实测较预期路径偏低 {abs(delta or 0):.1f}{unit}，追更高温档需要等待后续观测确认。", - summary_en=f"Observed temperature is running {abs(delta or 0):.1f}{unit} below the expected path; higher buckets need confirmation from later observations.", - ) - else: - _add_signal( - signals, - label="日内节奏", - label_en="Intraday pace", - direction="neutral", - strength="weak", - summary="实测大体贴近当前预期路径，下一步主要看峰值窗口内是否继续抬升。", - summary_en="Observed temperature is broadly tracking the expected path; the next question is whether it keeps lifting through the peak window.", - ) - - heating_setup = str(vertical.get("heating_setup") or "").lower() - suppression_risk = str(vertical.get("suppression_risk") or "").lower() - if heating_setup or suppression_risk: - available_layers += 1 - if heating_setup == "supportive": - support_score += 2 - _add_signal( - signals, - label="边界层结构", - label_en="Boundary-layer setup", - direction="support", - strength="strong", - summary=str(vertical.get("summary_zh") or "边界层结构支持白天继续混合升温。"), - summary_en=str(vertical.get("summary_en") or "The boundary-layer setup supports continued daytime mixing and warming."), - ) - elif heating_setup == "suppressed" or suppression_risk == "high": - suppress_score += 2 - _add_signal( - signals, - label="边界层结构", - label_en="Boundary-layer setup", - direction="suppress", - strength="strong", - summary=str(vertical.get("summary_zh") or "边界层或云雨结构对午后峰值形成压制。"), - summary_en=str(vertical.get("summary_en") or "Boundary-layer or cloud/rain structure is capping the afternoon peak."), - ) - else: - _add_signal( - signals, - label="边界层结构", - label_en="Boundary-layer setup", - direction="neutral", - strength="medium", - summary=str(vertical.get("summary_zh") or "边界层结构暂未给出单边信号。"), - summary_en=str(vertical.get("summary_en") or "The boundary-layer setup does not yet provide a one-sided signal."), - ) - - taf_suppression = str(taf_signal.get("suppression_level") or "").lower() - taf_disruption = str(taf_signal.get("disruption_level") or "").lower() - taf_has_cloud_rain_cap = taf_suppression in {"medium", "high"} or taf_disruption in { - "medium", - "high", - } - - structural_cap = False - if taf_signal.get("available") or taf_suppression: - available_layers += 1 - if taf_suppression == "high" or taf_disruption == "high": - suppress_score += 2 - direction_value = "suppress" - strength = "strong" - elif taf_suppression == "medium" or taf_disruption == "medium": - suppress_score += 1 - direction_value = "suppress" - strength = "medium" - else: - support_score += 1 - direction_value = "support" - strength = "weak" - _add_signal( - signals, - label="TAF 云雨扰动", - label_en="TAF cloud/rain disruption", - direction=direction_value, - strength=strength, - summary=str(taf_signal.get("summary_zh") or "TAF 暂未提示强云雨压温信号。"), - summary_en=str(taf_signal.get("summary_en") or "TAF does not yet flag a strong cloud/rain temperature cap."), - ) - - airport_delta = _sf(data.get("airport_vs_network_delta")) - lead_signal = data.get("network_lead_signal") or {} - if airport_delta is not None: - available_layers += 1 - leader = str(lead_signal.get("leader_station_label") or lead_signal.get("leader_station_code") or "").strip() - sync_status = str(lead_signal.get("leader_sync_status") or "").strip().lower() - sync_delta = _sf(lead_signal.get("leader_time_delta_vs_anchor_minutes")) - sync_suffix_zh = "" - sync_suffix_en = "" - if sync_status in {"near_realtime", "lagged"} and sync_delta is not None: - sync_suffix_zh = f"；但与机场锚点约差 {sync_delta:.0f} 分钟，作为降权信号处理" - sync_suffix_en = f"; timing differs from the airport anchor by about {sync_delta:.0f} minutes, so this signal is down-weighted" - elif sync_status == "unknown": - sync_suffix_zh = "；周边站观测时间不可完全校验，作为弱参考" - sync_suffix_en = "; station timing is not fully verified, so this is treated as a weak reference" - if airport_delta <= -0.4: - support_score += 1 - _add_signal( - signals, - label="站网对比", - label_en="Station-network comparison", - direction="support", - strength="weak" if sync_suffix_zh else "medium", - summary=f"周边站网较机场锚点偏热 {abs(airport_delta):.1f}{unit}{f'，领先点位 {leader}' if leader else ''}{sync_suffix_zh}。", - summary_en=f"Nearby stations are {abs(airport_delta):.1f}{unit} warmer than the airport anchor{f'; leading site: {leader}' if leader else ''}{sync_suffix_en}.", - ) - elif airport_delta >= 0.4: - suppress_score += 1 - _add_signal( - signals, - label="站网对比", - label_en="Station-network comparison", - direction="suppress", - strength="weak" if sync_suffix_zh else "medium", - summary=f"机场锚点较周边站网偏热 {abs(airport_delta):.1f}{unit}，继续上修需要机场自身后续报文确认{sync_suffix_zh}。", - summary_en=f"The airport anchor is {abs(airport_delta):.1f}{unit} warmer than nearby stations; further upside needs confirmation from later airport reports{sync_suffix_en}.", - ) - else: - _add_signal( - signals, - label="站网对比", - label_en="Station-network comparison", - direction="neutral", - strength="weak", - summary="机场锚点与周边站网基本同步，暂不构成单独上修或下修理由。", - summary_en="The airport anchor and nearby station network are broadly aligned, so this layer does not independently argue for upside or downside.", - ) - - peak_status = str(peak.get("status") or "").lower() - first_h = _sf(peak.get("first_h")) - last_h = _sf(peak.get("last_h")) - peak_window = ( - f"{int(first_h):02d}:00-{int(last_h):02d}:59" - if first_h is not None and last_h is not None - else "--" - ) - if peak_status == "past": - headline = "峰值窗口已过，后续更偏向确认最终高点而非继续上修。" - headline_en = "The peak window has passed; the read now shifts toward confirming the final high rather than chasing further upside." - confidence = "high" if available_layers >= 2 else "medium" - elif suppress_score >= support_score + 2: - structural_cap = any( - signal.get("direction") == "suppress" - and signal.get("label") in {"边界层结构", "站网对比", "日内节奏"} - for signal in signals - ) - if taf_has_cloud_rain_cap and structural_cap: - headline = "峰值同时存在 TAF 云雨扰动和结构压制，当前更偏防守高温上修。" - headline_en = "Both TAF cloud/rain disruption and structural signals are capping the peak; defend against aggressive high-temperature upside for now." - elif taf_has_cloud_rain_cap: - headline = "TAF 提示峰值窗口有云雨扰动，当前更偏防守高温上修。" - headline_en = "TAF flags cloud/rain disruption near the peak window; defend against aggressive high-temperature upside for now." - else: - headline = "峰值主要受结构信号压制，TAF 云雨层暂未构成主压温理由。" - headline_en = "The peak is mainly capped by structural signals; TAF cloud/rain is not the primary suppression reason for now." - confidence = "high" if available_layers >= 3 else "medium" - elif support_score >= suppress_score + 2: - headline = "峰值仍有上修空间，后续重点看峰值窗口内报文能否继续抬升。" - headline_en = "The peak still has upside room; the next check is whether reports keep lifting through the peak window." - confidence = "high" if available_layers >= 3 else "medium" - elif available_layers == 0: - headline = "关键日内层仍在补齐，先以观测锚点和下一次报文为主。" - headline_en = "Key intraday layers are still filling in; anchor the read on observations and the next report." - confidence = "low" - else: - headline = "当前处于分歧判断区，峰值窗口内的下一组观测将决定方向。" - headline_en = "The setup is in a split-decision zone; the next observations inside the peak window should decide direction." - confidence = "medium" if available_layers >= 2 else "low" - - next_observation = _next_observation_clock(data.get("local_time") or current.get("obs_time")) - threshold = base_value - invalidation_rules = [] - invalidation_rules_en = [] - confirmation_rules = [] - confirmation_rules_en = [] - if peak_status == "past": - invalidation_rules.append("若后续官方结算源补录更高值，以结算源最终高点为准。") - invalidation_rules_en.append("If the official settlement source later backfills a higher reading, defer to the final settlement-source high.") - confirmation_rules.append("若峰值窗口后连续两次观测不再创新高，当前高点基本确认。") - confirmation_rules_en.append("If two consecutive post-peak observations fail to make a new high, the current high is broadly confirmed.") - else: - watch_clock = _format_clock_minutes(int(first_h or 13) * 60 + 30) - if threshold is not None: - invalidation_rules.append(f"{watch_clock} 前若仍未接近 {threshold:.0f}{unit}，上修路径降级。") - invalidation_rules_en.append(f"If observations are still not near {threshold:.0f}{unit} before {watch_clock}, downgrade the upside path.") - confirmation_rules.append(f"峰值窗口内任一结算源观测触达或超过 {threshold:.0f}{unit}，基准路径确认度上升。") - confirmation_rules_en.append(f"If any settlement-source observation reaches or exceeds {threshold:.0f}{unit} inside the peak window, confidence in the base path rises.") - invalidation_rules.append("若 TAF 或实况报文出现阵雨、雷暴或低云/云雨压制，高温上沿需要下调。") - invalidation_rules_en.append("If TAF or live reports show showers, thunderstorms, or low-cloud/cloud-rain suppression, lower the upper temperature bound.") - confirmation_rules.append("若实测继续贴近 DEB 曲线且云雨信号不增强，维持当前主路径。") - confirmation_rules_en.append("If observations keep tracking the DEB curve and cloud/rain signals do not strengthen, maintain the current main path.") - - if not signals: - _add_signal( - signals, - label="数据完整性", - label_en="Data completeness", - direction="neutral", - strength="weak", - summary="当前缺少足够的日内结构层，等待下一次观测刷新后再提高判断权重。", - summary_en="There are not enough intraday structure layers yet; wait for the next observation refresh before raising confidence.", - ) - - return { - "headline": headline, - "headline_en": headline_en, - "confidence": confidence, - "base_case_bucket": base_case_bucket, - "upside_bucket": upside_bucket, - "downside_bucket": downside_bucket, - "next_observation_time": next_observation, - "peak_window": peak_window, - "invalidation_rules": invalidation_rules[:4], - "invalidation_rules_en": invalidation_rules_en[:4], - "confirmation_rules": confirmation_rules[:3], - "confirmation_rules_en": confirmation_rules_en[:3], - "signal_contributions": signals[:5], - } def _archive_intraday_path_snapshot(city: str, result: Dict[str, Any]) -> None: diff --git a/web/auth/guards.py b/web/auth/guards.py new file mode 100644 index 00000000..b6df2f1a --- /dev/null +++ b/web/auth/guards.py @@ -0,0 +1,269 @@ +"""Authentication guards and identity resolution for PolyWeather.""" + +import os +from typing import Any, Dict + +from fastapi import HTTPException, Request +from loguru import logger + +from src.auth.supabase_entitlement import SUPABASE_ENTITLEMENT, extract_bearer_token + + +def _env_bool(name: str, default: bool = False) -> bool: + raw = os.getenv(name) + if raw is None: + return default + return raw.strip().lower() in {"1", "true", "yes", "on"} + + +_ENTITLEMENT_HEADER = "x-polyweather-entitlement" +_FORWARDED_SUPABASE_USER_ID_HEADER = "x-polyweather-auth-user-id" +_FORWARDED_SUPABASE_EMAIL_HEADER = "x-polyweather-auth-email" +_OPS_ADMIN_EMAILS = { + item.strip().lower() + for item in str(os.getenv("POLYWEATHER_OPS_ADMIN_EMAILS") or "").split(",") + if item.strip() +} + + +# Config values imported lazily from web.core to avoid circular imports +# and allow monkeypatching to target web.core directly. +def _get_entitlement_token(): + import web.core as _core + return _core._ENTITLEMENT_TOKEN + + +def _get_supabase_auth_required(): + import web.core as _core + return _core._SUPABASE_AUTH_REQUIRED + + +def _get_entitlement_guard_enabled(): + import web.core as _core + return _core._ENTITLEMENT_GUARD_ENABLED + + +def _legacy_service_token_valid(request: Request) -> bool: + token = request.headers.get(_ENTITLEMENT_HEADER) + if not token: + token = extract_bearer_token(request.headers.get("authorization")) + token_hint = _get_entitlement_token() + return bool(token_hint and token == token_hint) + + +def _bind_forwarded_supabase_identity(request: Request) -> bool: + if not _legacy_service_token_valid(request): + return False + forwarded_user_id = str( + request.headers.get(_FORWARDED_SUPABASE_USER_ID_HEADER) or "" + ).strip() + if not forwarded_user_id: + return False + request.state.auth_user_id = forwarded_user_id + request.state.auth_email = str( + request.headers.get(_FORWARDED_SUPABASE_EMAIL_HEADER) or "" + ).strip() + return True + + +def _bind_optional_supabase_identity(request: Request) -> None: + if _bind_forwarded_supabase_identity(request): + return + if not SUPABASE_ENTITLEMENT.configured: + return + access_token = extract_bearer_token(request.headers.get("authorization")) + if not access_token: + return + identity = SUPABASE_ENTITLEMENT.get_identity(access_token) + if not identity: + return + request.state.auth_user_id = identity.user_id + request.state.auth_email = identity.email + request.state.auth_points = identity.points + request.state.auth_created_at = identity.created_at + from src.utils.online_tracker import record_activity + record_activity(identity.user_id) + + +def _resolve_auth_points(request: Request, account_db=None) -> int: + from src.database.db_manager import DBManager + + if account_db is None: + # imported lazily to avoid circular dependency at module level + from web.core import _account_db as _db + account_db = _db + + raw_points = getattr(request.state, "auth_points", 0) + try: + points = max(0, int(raw_points or 0)) + except Exception: + points = 0 + + user_id = str(getattr(request.state, "auth_user_id", "") or "").strip() + + if user_id: + try: + db_points = account_db.get_points_by_supabase_user_id(user_id) + if db_points > points: + request.state.auth_points = db_points + points = db_points + except Exception as exc: + logger.warning(f"auth points fallback failed user_id={user_id}: {exc}") + + if points <= 0: + email = str(getattr(request.state, "auth_email", "") or "").strip().lower() + if email: + try: + email_points = account_db.get_points_by_supabase_email(email) + if email_points > points: + request.state.auth_points = email_points + points = email_points + except Exception as exc: + logger.warning( + f"auth points email fallback failed email={email}: {exc}" + ) + + return points + + +def _resolve_weekly_profile(request: Request, account_db=None) -> Dict[str, Any]: + from src.database.db_manager import DBManager + + if account_db is None: + from web.core import _account_db as _db + account_db = _db + + user_id = str(getattr(request.state, "auth_user_id", "") or "").strip() + if not user_id: + return {"weekly_points": 0, "weekly_rank": None} + try: + profile = account_db.get_weekly_profile_by_supabase_user_id(user_id) + return { + "weekly_points": int(profile.get("weekly_points") or 0), + "weekly_rank": profile.get("weekly_rank"), + } + except Exception as exc: + logger.warning(f"auth weekly profile fallback failed user_id={user_id}: {exc}") + return {"weekly_points": 0, "weekly_rank": None} + + +def _assert_entitlement(request: Request) -> None: + if SUPABASE_ENTITLEMENT.enabled: + if _legacy_service_token_valid(request): + if _bind_forwarded_supabase_identity(request): + return + bearer_token = extract_bearer_token(request.headers.get("authorization")) + if not bearer_token or bearer_token == _get_entitlement_token(): + return + if not _get_supabase_auth_required(): + _bind_optional_supabase_identity(request) + return + if not SUPABASE_ENTITLEMENT.configured: + raise HTTPException( + status_code=503, + detail="Supabase auth is enabled but SUPABASE_URL / SUPABASE_ANON_KEY is not configured", + ) + + access_token = extract_bearer_token(request.headers.get("authorization")) + if not access_token: + raise HTTPException(status_code=401, detail="Unauthorized") + + identity = SUPABASE_ENTITLEMENT.get_identity(access_token) + if not identity: + raise HTTPException(status_code=401, detail="Unauthorized") + skip_subscription_gate = bool( + getattr(request.state, "skip_subscription_gate", False) + ) + if ( + not skip_subscription_gate + and not SUPABASE_ENTITLEMENT.has_active_subscription(identity.user_id) + ): + raise HTTPException(status_code=403, detail="Subscription required") + + request.state.auth_user_id = identity.user_id + request.state.auth_email = identity.email + request.state.auth_points = identity.points + request.state.auth_created_at = identity.created_at + from src.utils.online_tracker import record_activity + record_activity(identity.user_id) + return + + if not _get_entitlement_guard_enabled(): + return + + if not _get_entitlement_token(): + raise HTTPException( + status_code=503, + detail="Entitlement guard is enabled but backend token is not configured", + ) + + if not _legacy_service_token_valid(request): + raise HTTPException(status_code=401, detail="Unauthorized") + + +def _require_supabase_identity(request: Request) -> Dict[str, str]: + if not SUPABASE_ENTITLEMENT.enabled: + raise HTTPException( + status_code=503, detail="payment requires POLYWEATHER_AUTH_ENABLED=true" + ) + if not SUPABASE_ENTITLEMENT.configured: + raise HTTPException( + status_code=503, + detail="payment requires SUPABASE_URL and SUPABASE_ANON_KEY", + ) + + state_user_id = str(getattr(request.state, "auth_user_id", "") or "").strip() + if state_user_id: + state_email = str(getattr(request.state, "auth_email", "") or "").strip() + return {"user_id": state_user_id, "email": state_email} + + token = extract_bearer_token(request.headers.get("authorization")) + if token: + identity = SUPABASE_ENTITLEMENT.get_identity(token) + if identity: + return {"user_id": identity.user_id, "email": identity.email} + + legacy_ok = _legacy_service_token_valid(request) + if legacy_ok: + forwarded_user_id = str( + request.headers.get(_FORWARDED_SUPABASE_USER_ID_HEADER) or "" + ).strip() + if forwarded_user_id: + forwarded_email = str( + request.headers.get(_FORWARDED_SUPABASE_EMAIL_HEADER) or "" + ).strip() + return {"user_id": forwarded_user_id, "email": forwarded_email} + return {"user_id": "entitlement", "email": ""} + + logger.warning( + "payment auth identity missing state_user={} auth_bearer={} legacy_ok={} forwarded_user={}".format( + bool(state_user_id), + bool(token), + bool(legacy_ok), + bool( + str( + request.headers.get(_FORWARDED_SUPABASE_USER_ID_HEADER) or "" + ).strip() + ), + ) + ) + raise HTTPException(status_code=401, detail="Unauthorized") + + +def _require_ops_admin(request: Request) -> Dict[str, str]: + identity = _require_supabase_identity(request) + email = str(identity.get("email") or "").strip().lower() + if email and email in _OPS_ADMIN_EMAILS: + return identity + + user_id = identity.get("user_id") + if user_id and user_id.lower() == "entitlement": + raise HTTPException( + status_code=401, + detail="entitlement bearer is not suitable for ops admin authorization", + ) + + raise HTTPException( + status_code=403, + detail="ops access restricted to configured admin emails", + ) diff --git a/web/core.py b/web/core.py index a02cb47f..b226c4d6 100644 --- a/web/core.py +++ b/web/core.py @@ -1,38 +1,28 @@ """ PolyWeather Web Core Context + +Holds module-level singletons (app, config, weather collector, DB manager, cache) +and re-exports symbols from domain-specific modules for backward compatibility. """ -import json import os -import sqlite3 import threading -import time from collections import OrderedDict -from datetime import datetime, timezone from typing import Any, Dict, Optional -from fastapi import FastAPI, HTTPException, Request +from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware -from pydantic import BaseModel, Field from loguru import logger from src.utils.config_loader import load_config from src.utils.config_validation import validate_runtime_env from src.data_collection.weather_sources import WeatherDataCollector -from src.data_collection.country_networks import provider_coverage_summary -from src.data_collection.city_risk_profiles import CITY_RISK_PROFILES # noqa: F401 -from src.auth.supabase_entitlement import SUPABASE_ENTITLEMENT, extract_bearer_token -from src.utils.metrics import ( - build_metrics_summary, - counter_inc, - gauge_set, - histogram_observe, -) from src.database.db_manager import DBManager -from src.database.runtime_state import get_state_storage_mode -from src.payments import PAYMENT_CHECKOUT, PaymentCheckoutError # noqa: F401 from src.data_collection.city_registry import CITY_REGISTRY +# --------------------------------------------------------------------------- +# FastAPI app singleton +# --------------------------------------------------------------------------- app = FastAPI(title="PolyWeather Map", version="1.0") _cors_origins = os.getenv( @@ -47,6 +37,9 @@ app.add_middleware( allow_headers=["*"], ) +# --------------------------------------------------------------------------- +# Core singletons (must remain module-level for backward compat) +# --------------------------------------------------------------------------- _config = load_config() _config_validation = validate_runtime_env("web") for _warning in _config_validation.warnings: @@ -79,6 +72,9 @@ SETTLEMENT_SOURCE_LABELS: Dict[str, str] = { "wunderground": "Wunderground", } +# --------------------------------------------------------------------------- +# LRUDict — simple size-bounded cache +# --------------------------------------------------------------------------- class LRUDict: """Size-bounded ordered dict that evicts oldest entries on overflow.""" @@ -111,405 +107,112 @@ CACHE_TTL_ANKARA = 60 CACHE_TTL_KOREAN_AMOS = 60 _PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +# --------------------------------------------------------------------------- +# Middleware — imported from web.middleware.http +# --------------------------------------------------------------------------- +from web.middleware.http import metrics_middleware as _metrics_middleware +from web.middleware.http import etag_middleware as _etag_middleware -@app.middleware("http") -async def _metrics_middleware(request: Request, call_next): - started = time.perf_counter() - try: - response = await call_next(request) - except Exception: - duration_ms = (time.perf_counter() - started) * 1000.0 - counter_inc( - "polyweather_http_requests_total", - method=request.method, - path=request.url.path, - status="500", - ) - histogram_observe( - "polyweather_http_request_duration_ms", - duration_ms, - method=request.method, - path=request.url.path, - status="500", - ) - raise +app.middleware("http")(_metrics_middleware) +app.middleware("http")(_etag_middleware) - duration_ms = (time.perf_counter() - started) * 1000.0 - status_code = str(response.status_code) - counter_inc( - "polyweather_http_requests_total", - method=request.method, - path=request.url.path, - status=status_code, - ) - histogram_observe( - "polyweather_http_request_duration_ms", - duration_ms, - method=request.method, - path=request.url.path, - status=status_code, - ) - return response +# Re-export middleware for external consumers +metrics_middleware = _metrics_middleware +etag_middleware = _etag_middleware +# --------------------------------------------------------------------------- +# Schemas — re-exported from web.schemas +# --------------------------------------------------------------------------- +from web.schemas.auth import ( # noqa: E402, F401 + AnalyticsEventRequest, + FeedbackRewardRequest, + GrantPointsRequest, + ReferralApplyRequest, + TelegramBindTokenRequest, + TelegramLoginRequest, + UserFeedbackRequest, +) +from web.schemas.payments import ( # noqa: E402, F401 + ConfirmPaymentTxRequest, + CreatePaymentIntentRequest, + SubmitPaymentTxRequest, + ValidatePaymentTxRequest, + WalletBindRequest, + WalletChallengeRequest, + WalletUnbindRequest, + WalletVerifyRequest, +) -@app.middleware("http") -async def _etag_middleware(request: Request, call_next): - """Add ETag to GET /api/* responses; return 304 on If-None-Match hit.""" - response = await call_next(request) +# --------------------------------------------------------------------------- +# Auth guards — imported from web.auth.guards, wrappers for account_db +# --------------------------------------------------------------------------- +from src.auth.supabase_entitlement import SUPABASE_ENTITLEMENT # noqa: E402 - if request.method != "GET" or response.status_code != 200: - return response +import web.auth.guards as _auth_guards # noqa: E402 - path = request.url.path - if not path.startswith("/api/") or path.endswith("/stream"): - return response - - body = getattr(response, "body", None) or b"" - if not body: - return response - - try: - import hashlib - - etag = hashlib.md5(body).hexdigest() - except Exception: - return response - - etag_value = f'"{etag}"' - if_none_match = request.headers.get("If-None-Match", "") - if if_none_match == etag_value: - from fastapi.responses import Response - - return Response(status_code=304, headers={"ETag": etag_value}) - - response.headers["ETag"] = etag_value - response.headers["Cache-Control"] = "private, max-age=30" - return response - - -def _env_bool(name: str, default: bool = False) -> bool: - raw = os.getenv(name) - if raw is None: - return default - return raw.strip().lower() in {"1", "true", "yes", "on"} - - -_ENTITLEMENT_GUARD_ENABLED = _env_bool("POLYWEATHER_REQUIRE_ENTITLEMENT", False) -_ENTITLEMENT_HEADER = "x-polyweather-entitlement" -_ENTITLEMENT_TOKEN = (os.getenv("POLYWEATHER_BACKEND_ENTITLEMENT_TOKEN") or "").strip() -_FORWARDED_SUPABASE_USER_ID_HEADER = "x-polyweather-auth-user-id" -_FORWARDED_SUPABASE_EMAIL_HEADER = "x-polyweather-auth-email" -_SUPABASE_AUTH_REQUIRED = _env_bool( +# Auth config flags — defined directly here so tests can monkeypatch web.core +_SUPABASE_AUTH_REQUIRED = _auth_guards._env_bool( "POLYWEATHER_AUTH_REQUIRED", SUPABASE_ENTITLEMENT.enabled, ) -_OPS_ADMIN_EMAILS = { - item.strip().lower() - for item in str(os.getenv("POLYWEATHER_OPS_ADMIN_EMAILS") or "").split(",") - if item.strip() -} +_ENTITLEMENT_GUARD_ENABLED = _auth_guards._env_bool("POLYWEATHER_REQUIRE_ENTITLEMENT", False) +_ENTITLEMENT_TOKEN = (os.getenv("POLYWEATHER_BACKEND_ENTITLEMENT_TOKEN") or "").strip() -def _legacy_service_token_valid(request: Request) -> bool: - token = request.headers.get(_ENTITLEMENT_HEADER) - if not token: - token = extract_bearer_token(request.headers.get("authorization")) - return bool(_ENTITLEMENT_TOKEN and token == _ENTITLEMENT_TOKEN) +def _legacy_service_token_valid(request): + return _auth_guards._legacy_service_token_valid(request) -def _bind_forwarded_supabase_identity(request: Request) -> bool: - if not _legacy_service_token_valid(request): - return False - forwarded_user_id = str( - request.headers.get(_FORWARDED_SUPABASE_USER_ID_HEADER) or "" - ).strip() - if not forwarded_user_id: - return False - request.state.auth_user_id = forwarded_user_id - request.state.auth_email = str( - request.headers.get(_FORWARDED_SUPABASE_EMAIL_HEADER) or "" - ).strip() - return True +def _bind_forwarded_supabase_identity(request): + return _auth_guards._bind_forwarded_supabase_identity(request) -def _bind_optional_supabase_identity(request: Request) -> None: - if _bind_forwarded_supabase_identity(request): - return - if not SUPABASE_ENTITLEMENT.configured: - return - access_token = extract_bearer_token(request.headers.get("authorization")) - if not access_token: - return - identity = SUPABASE_ENTITLEMENT.get_identity(access_token) - if not identity: - return - request.state.auth_user_id = identity.user_id - request.state.auth_email = identity.email - request.state.auth_points = identity.points - request.state.auth_created_at = identity.created_at - from src.utils.online_tracker import record_activity - record_activity(identity.user_id) +def _bind_optional_supabase_identity(request): + return _auth_guards._bind_optional_supabase_identity(request) -def _resolve_auth_points(request: Request) -> int: - raw_points = getattr(request.state, "auth_points", 0) - try: - points = max(0, int(raw_points or 0)) - except Exception: - points = 0 - - user_id = str(getattr(request.state, "auth_user_id", "") or "").strip() - - if user_id: - try: - db_points = _account_db.get_points_by_supabase_user_id(user_id) - if db_points > points: - request.state.auth_points = db_points - points = db_points - except Exception as exc: - logger.warning(f"auth points fallback failed user_id={user_id}: {exc}") - - if points <= 0: - email = str(getattr(request.state, "auth_email", "") or "").strip().lower() - if email: - try: - email_points = _account_db.get_points_by_supabase_email(email) - if email_points > points: - request.state.auth_points = email_points - points = email_points - except Exception as exc: - logger.warning( - f"auth points email fallback failed email={email}: {exc}" - ) - - return points +def _resolve_auth_points(request): + return _auth_guards._resolve_auth_points(request, account_db=_account_db) -def _resolve_weekly_profile(request: Request) -> Dict[str, Any]: - user_id = str(getattr(request.state, "auth_user_id", "") or "").strip() - if not user_id: - return {"weekly_points": 0, "weekly_rank": None} - try: - profile = _account_db.get_weekly_profile_by_supabase_user_id(user_id) - return { - "weekly_points": int(profile.get("weekly_points") or 0), - "weekly_rank": profile.get("weekly_rank"), - } - except Exception as exc: - logger.warning(f"auth weekly profile fallback failed user_id={user_id}: {exc}") - return {"weekly_points": 0, "weekly_rank": None} +def _resolve_weekly_profile(request): + return _auth_guards._resolve_weekly_profile(request, account_db=_account_db) -def _assert_entitlement(request: Request) -> None: - if SUPABASE_ENTITLEMENT.enabled: - if _legacy_service_token_valid(request): - if _bind_forwarded_supabase_identity(request): - return - bearer_token = extract_bearer_token(request.headers.get("authorization")) - if not bearer_token or bearer_token == _ENTITLEMENT_TOKEN: - return - if not _SUPABASE_AUTH_REQUIRED: - _bind_optional_supabase_identity(request) - return - if not SUPABASE_ENTITLEMENT.configured: - raise HTTPException( - status_code=503, - detail="Supabase auth is enabled but SUPABASE_URL / SUPABASE_ANON_KEY is not configured", - ) - - access_token = extract_bearer_token(request.headers.get("authorization")) - if not access_token: - raise HTTPException(status_code=401, detail="Unauthorized") - - identity = SUPABASE_ENTITLEMENT.get_identity(access_token) - if not identity: - raise HTTPException(status_code=401, detail="Unauthorized") - skip_subscription_gate = bool( - getattr(request.state, "skip_subscription_gate", False) - ) - if ( - not skip_subscription_gate - and not SUPABASE_ENTITLEMENT.has_active_subscription(identity.user_id) - ): - raise HTTPException(status_code=403, detail="Subscription required") - - request.state.auth_user_id = identity.user_id - request.state.auth_email = identity.email - request.state.auth_points = identity.points - request.state.auth_created_at = identity.created_at - from src.utils.online_tracker import record_activity - record_activity(identity.user_id) - return - - if not _ENTITLEMENT_GUARD_ENABLED: - return - - if not _ENTITLEMENT_TOKEN: - raise HTTPException( - status_code=503, - detail="Entitlement guard is enabled but backend token is not configured", - ) - - if not _legacy_service_token_valid(request): - raise HTTPException(status_code=401, detail="Unauthorized") +def _assert_entitlement(request): + return _auth_guards._assert_entitlement(request) -def _require_supabase_identity(request: Request) -> Dict[str, str]: - if not SUPABASE_ENTITLEMENT.enabled: - raise HTTPException( - status_code=503, detail="payment requires POLYWEATHER_AUTH_ENABLED=true" - ) - if not SUPABASE_ENTITLEMENT.configured: - raise HTTPException( - status_code=503, - detail="payment requires SUPABASE_URL and SUPABASE_ANON_KEY", - ) +def _require_supabase_identity(request): + return _auth_guards._require_supabase_identity(request) - state_user_id = str(getattr(request.state, "auth_user_id", "") or "").strip() - if state_user_id: - state_email = str(getattr(request.state, "auth_email", "") or "").strip() - return {"user_id": state_user_id, "email": state_email} - token = extract_bearer_token(request.headers.get("authorization")) - if token: - identity = SUPABASE_ENTITLEMENT.get_identity(token) - if identity: - return {"user_id": identity.user_id, "email": identity.email} +def _require_ops_admin(request): + return _auth_guards._require_ops_admin(request) - legacy_ok = _legacy_service_token_valid(request) - if legacy_ok: - forwarded_user_id = str( - request.headers.get(_FORWARDED_SUPABASE_USER_ID_HEADER) or "" - ).strip() - if forwarded_user_id: - forwarded_email = str( - request.headers.get(_FORWARDED_SUPABASE_EMAIL_HEADER) or "" - ).strip() - return {"user_id": forwarded_user_id, "email": forwarded_email} - # Entitlement token is valid but forwarded headers are missing. - # Return a placeholder identity — callers (e.g. _require_ops_admin) - # can decide whether to accept it. - return {"user_id": "entitlement", "email": ""} - logger.warning( - "payment auth identity missing state_user={} auth_bearer={} legacy_ok={} forwarded_user={}".format( - bool(state_user_id), - bool(token), - bool(legacy_ok), - bool( - str( - request.headers.get(_FORWARDED_SUPABASE_USER_ID_HEADER) or "" - ).strip() - ), - ) +# --------------------------------------------------------------------------- +# Diagnostics — re-exported from web.diagnostics.health +# --------------------------------------------------------------------------- +from web.diagnostics.health import ( # noqa: E402 + build_health_payload as _build_health_payload_raw, + build_system_status_payload as _build_system_status_payload_raw, +) + + +def build_health_payload(): + return _build_health_payload_raw(_account_db, len(CITIES)) + + +def build_system_status_payload(): + return _build_system_status_payload_raw( + _account_db, _config, _weather, _cache, len(_cache), len(CITIES), CITY_REGISTRY ) - raise HTTPException(status_code=401, detail="Unauthorized") - - -def _require_ops_admin(request: Request) -> Dict[str, str]: - is_entitlement = _legacy_service_token_valid(request) - identity = _require_supabase_identity(request) - if not _OPS_ADMIN_EMAILS: - raise HTTPException( - status_code=503, - detail="ops admin is not configured; set POLYWEATHER_OPS_ADMIN_EMAILS", - ) - email = str(identity.get("email") or "").strip().lower() - if email and email in _OPS_ADMIN_EMAILS: - return identity - # If identity lacks an email (e.g. pure entitlement-token auth with - # missing forwarded headers), loosen the requirement: entitlement token - # alone is sufficient admin proof when admin list is configured. - if is_entitlement: - granted = { - "user_id": "admin:entitlement", - "email": next(iter(_OPS_ADMIN_EMAILS)), - } - return granted - raise HTTPException(status_code=403, detail="ops admin required") - - -class WalletChallengeRequest(BaseModel): - address: str = Field(..., min_length=8) - - -class WalletVerifyRequest(BaseModel): - address: str = Field(..., min_length=8) - nonce: str = Field(..., min_length=6) - signature: str = Field(..., min_length=20) - - -class WalletUnbindRequest(BaseModel): - address: str = Field(..., min_length=8) - - -class CreatePaymentIntentRequest(BaseModel): - plan_code: str = Field(default="pro_monthly", min_length=2) - payment_mode: str = Field(default="strict") - allowed_wallet: Optional[str] = None - token_address: Optional[str] = None - chain_id: Optional[int] = None - use_points: bool = False - points_to_consume: Optional[int] = None - metadata: Dict[str, Any] = Field(default_factory=dict) - - -class SubmitPaymentTxRequest(BaseModel): - tx_hash: str = Field(..., min_length=10) - from_address: Optional[str] = None - - -class ValidatePaymentTxRequest(BaseModel): - tx_hash: str = Field(..., min_length=10) - - -class ConfirmPaymentTxRequest(BaseModel): - tx_hash: Optional[str] = None - - -class ReferralApplyRequest(BaseModel): - code: str = Field(..., min_length=3, max_length=32) - - -class TelegramLoginRequest(BaseModel): - id: int - first_name: Optional[str] = None - last_name: Optional[str] = None - username: Optional[str] = None - photo_url: Optional[str] = None - auth_date: int - hash: str = Field(..., min_length=10) - - -class TelegramBindTokenRequest(BaseModel): - token: str = Field(..., min_length=8) - - -class AnalyticsEventRequest(BaseModel): - event_type: str = Field(..., min_length=3, max_length=64) - client_id: Optional[str] = Field(default=None, max_length=128) - session_id: Optional[str] = Field(default=None, max_length=128) - payload: Dict[str, Any] = Field(default_factory=dict) - - -class UserFeedbackRequest(BaseModel): - category: str = Field(default="bug", min_length=2, max_length=40) - message: str = Field(..., min_length=3, max_length=5000) - source: str = Field(default="terminal", max_length=40) - contact: Optional[str] = Field(default=None, max_length=180) - context: Dict[str, Any] = Field(default_factory=dict) - - -class GrantPointsRequest(BaseModel): - email: str = Field(..., min_length=3) - points: int = Field(..., gt=0, le=100000) - - -class FeedbackRewardRequest(BaseModel): - points: int = Field(..., gt=0, le=100000) - reason: str = Field(default="", max_length=500) +# --------------------------------------------------------------------------- +# Utility helpers +# --------------------------------------------------------------------------- def _sf(v) -> Optional[float]: if v is None: return None @@ -523,335 +226,11 @@ def _is_excluded_model_name(model_name: str) -> bool: return False -def _sqlite_health() -> Dict[str, Any]: - try: - with sqlite3.connect(_account_db.db_path, timeout=0.05) as conn: - conn.execute("SELECT 1").fetchone() - return {"ok": True, "db_path": _account_db.db_path} - except Exception as exc: - return {"ok": False, "db_path": _account_db.db_path, "error": str(exc)} - - -def _cache_summary() -> Dict[str, Any]: - from web.analysis_service import get_analysis_cache_stats - - open_meteo_forecast_entries = len(getattr(_weather, "_open_meteo_cache", {}) or {}) - open_meteo_ensemble_entries = len(getattr(_weather, "_ensemble_cache", {}) or {}) - open_meteo_multi_model_entries = len( - getattr(_weather, "_multi_model_cache", {}) or {} - ) - metar_entries = len(getattr(_weather, "_metar_cache", {}) or {}) - taf_entries = len(getattr(_weather, "_taf_cache", {}) or {}) - settlement_entries = len(getattr(_weather, "_settlement_cache", {}) or {}) - - gauge_set("polyweather_api_cache_entries", len(_cache)) - gauge_set( - "polyweather_open_meteo_forecast_cache_entries", open_meteo_forecast_entries - ) - gauge_set( - "polyweather_open_meteo_ensemble_cache_entries", open_meteo_ensemble_entries - ) - gauge_set( - "polyweather_open_meteo_multi_model_cache_entries", - open_meteo_multi_model_entries, - ) - gauge_set("polyweather_metar_cache_entries", metar_entries) - gauge_set("polyweather_taf_cache_entries", taf_entries) - gauge_set("polyweather_settlement_cache_entries", settlement_entries) - return { - "api_cache_entries": len(_cache), - "open_meteo_forecast_entries": open_meteo_forecast_entries, - "open_meteo_ensemble_entries": open_meteo_ensemble_entries, - "open_meteo_multi_model_entries": open_meteo_multi_model_entries, - "metar_entries": metar_entries, - "taf_entries": taf_entries, - "settlement_entries": settlement_entries, - "analysis": get_analysis_cache_stats(), - } - - -def _feature_flags_summary() -> Dict[str, Any]: - return { - "auth_enabled": bool(SUPABASE_ENTITLEMENT.enabled), - "auth_required": bool(_SUPABASE_AUTH_REQUIRED), - "entitlement_guard_enabled": bool(_ENTITLEMENT_GUARD_ENABLED), - "payment_enabled": bool(getattr(PAYMENT_CHECKOUT, "enabled", False)), - "state_storage_mode": get_state_storage_mode(), - } - - -def _integration_summary() -> Dict[str, Any]: - weather_cfg = _config.get("weather", {}) if isinstance(_config, dict) else {} - return { - "supabase_configured": bool(SUPABASE_ENTITLEMENT.configured), - "telegram_bot_configured": bool( - (_config.get("telegram", {}) or {}).get("bot_token") - ), - "walletconnect_configured": bool( - os.getenv("NEXT_PUBLIC_WALLETCONNECT_PROJECT_ID") - ), - "weather_sources": { - "openweather": bool(weather_cfg.get("openweather_api_key")), - "wunderground": bool(weather_cfg.get("wunderground_api_key")), - "visualcrossing": bool(weather_cfg.get("visualcrossing_api_key")), - }, - } - - -def _probability_summary() -> Dict[str, Any]: - return { - "engine_mode": "legacy", - } - - -def _read_json_file(path: str) -> Optional[Dict[str, Any]]: - if not os.path.exists(path): - return None - try: - with open(path, "r", encoding="utf-8") as fh: - payload = json.load(fh) - if isinstance(payload, dict): - return payload - except Exception: - return None - return None - - -def _table_date_summary(conn: sqlite3.Connection, table_name: str) -> Dict[str, Any]: - try: - row = conn.execute( - f""" - SELECT COUNT(*) AS row_count, - COUNT(DISTINCT city) AS cities_count, - MIN(target_date) AS min_date, - MAX(target_date) AS max_date - FROM {table_name} - """ - ).fetchone() - except Exception as exc: - return {"ok": False, "error": str(exc), "row_count": 0, "cities_count": 0} - - return { - "ok": True, - "row_count": int(row["row_count"] or 0), - "cities_count": int(row["cities_count"] or 0), - "min_date": row["min_date"], - "max_date": row["max_date"], - } - - -def _truth_source_counts(conn: sqlite3.Connection) -> Dict[str, int]: - try: - rows = conn.execute( - """ - SELECT COALESCE(NULLIF(TRIM(settlement_source), ''), 'unknown') AS settlement_source, - COUNT(*) AS row_count - FROM truth_records_store - GROUP BY COALESCE(NULLIF(TRIM(settlement_source), ''), 'unknown') - ORDER BY row_count DESC, settlement_source ASC - """ - ).fetchall() - except Exception: - return {} - return {str(row["settlement_source"]): int(row["row_count"] or 0) for row in rows} - - -def _truth_revisions_summary(conn: sqlite3.Connection) -> Dict[str, Any]: - try: - row = conn.execute( - """ - SELECT COUNT(*) AS row_count, - MAX(updated_at) AS last_updated_at - FROM truth_revisions_store - """ - ).fetchone() - except Exception as exc: - return {"ok": False, "error": str(exc), "row_count": 0} - return { - "ok": True, - "row_count": int(row["row_count"] or 0), - "last_updated_at": row["last_updated_at"], - } - - -def _city_coverage_summary(conn: sqlite3.Connection) -> Dict[str, Any]: - truth_rows = conn.execute( - """ - SELECT city, COUNT(*) AS row_count, MIN(target_date) AS min_date, MAX(target_date) AS max_date - FROM truth_records_store - GROUP BY city - """ - ).fetchall() - feature_rows = conn.execute( - """ - SELECT city, COUNT(*) AS row_count, MIN(target_date) AS min_date, MAX(target_date) AS max_date - FROM training_feature_records_store - GROUP BY city - """ - ).fetchall() - - truth_index = { - str(row["city"]): { - "truth_rows": int(row["row_count"] or 0), - "truth_min_date": row["min_date"], - "truth_max_date": row["max_date"], - } - for row in truth_rows - } - feature_index = { - str(row["city"]): { - "feature_rows": int(row["row_count"] or 0), - "feature_min_date": row["min_date"], - "feature_max_date": row["max_date"], - } - for row in feature_rows - } - - entries = [] - for city, meta in CITY_REGISTRY.items(): - truth_payload = truth_index.get(city, {}) - feature_payload = feature_index.get(city, {}) - entries.append( - { - "city": city, - "name": str(meta.get("name") or city), - "settlement_source": str(meta.get("settlement_source") or "metar"), - "settlement_station_code": str( - meta.get("settlement_station_code") or meta.get("icao") or "" - ), - "truth_rows": int(truth_payload.get("truth_rows") or 0), - "feature_rows": int(feature_payload.get("feature_rows") or 0), - "truth_min_date": truth_payload.get("truth_min_date"), - "truth_max_date": truth_payload.get("truth_max_date"), - "feature_min_date": feature_payload.get("feature_min_date"), - "feature_max_date": feature_payload.get("feature_max_date"), - } - ) - - highlighted = [ - entry for entry in entries if entry["city"] in {"taipei", "shenzhen"} - ] - gaps = sorted( - entries, - key=lambda entry: ( - entry["feature_rows"] > 0, - entry["truth_rows"] > 0, - entry["truth_rows"], - entry["feature_rows"], - entry["city"], - ), - )[:10] - return { - "total_cities": len(entries), - "with_truth_rows": sum(1 for entry in entries if entry["truth_rows"] > 0), - "with_feature_rows": sum(1 for entry in entries if entry["feature_rows"] > 0), - "entries": entries, - "highlighted": highlighted, - "top_gaps": gaps, - } - - -def _model_city_coverage_summary( - city_entries: Any, -) -> Dict[str, Any]: - rows = [] - for entry in city_entries or []: - city = str(entry.get("city") or "").strip().lower() - rows.append( - { - "city": city, - "name": entry.get("name") or city, - "settlement_source": entry.get("settlement_source"), - "truth_rows": int(entry.get("truth_rows") or 0), - "feature_rows": int(entry.get("feature_rows") or 0), - } - ) - - weakest = sorted( - rows, - key=lambda row: ( - row["truth_rows"] > 0, - row["truth_rows"], - row["city"], - ), - )[:12] - strongest = sorted( - rows, - key=lambda row: ( - -row["truth_rows"], - row["city"], - ), - )[:8] - return { - "weakest": weakest, - "strongest": strongest, - } - - -def _training_data_summary() -> Dict[str, Any]: - db_path = _account_db.db_path - truth_records = {"ok": False, "row_count": 0, "cities_count": 0} - truth_revisions = {"ok": False, "row_count": 0} - training_features = {"ok": False, "row_count": 0, "cities_count": 0} - try: - with sqlite3.connect(db_path) as conn: - conn.row_factory = sqlite3.Row - truth_records = _table_date_summary(conn, "truth_records_store") - if truth_records.get("ok"): - truth_records["source_counts"] = _truth_source_counts(conn) - truth_revisions = _truth_revisions_summary(conn) - training_features = _table_date_summary( - conn, "training_feature_records_store" - ) - city_coverage = _city_coverage_summary(conn) - except Exception as exc: - return { - "db_path": db_path, - "db_ok": False, - "error": str(exc), - "truth_records": truth_records, - "truth_revisions": truth_revisions, - "training_features": training_features, - "city_coverage": {}, - "model_city_coverage": {}, - } - - return { - "db_path": db_path, - "db_ok": True, - "truth_records": truth_records, - "truth_revisions": truth_revisions, - "training_features": training_features, - "city_coverage": city_coverage, - "model_city_coverage": _model_city_coverage_summary( - city_coverage.get("entries") or [], - ), - } - - -def build_health_payload() -> Dict[str, Any]: - db = _sqlite_health() - return { - "status": "ok" if db.get("ok") else "degraded", - "time_utc": datetime.now(timezone.utc).isoformat(), - "db": db, - "state_storage_mode": get_state_storage_mode(), - "cities_count": len(CITIES), - } - - -def build_system_status_payload() -> Dict[str, Any]: - return { - "status": build_health_payload()["status"], - "time_utc": datetime.now(timezone.utc).isoformat(), - "state_storage_mode": get_state_storage_mode(), - "db": _sqlite_health(), - "features": _feature_flags_summary(), - "integrations": _integration_summary(), - "cache": _cache_summary(), - "metrics": build_metrics_summary(), - "probability": _probability_summary(), - "training_data": _training_data_summary(), - "station_networks": provider_coverage_summary(), - "cities_count": len(CITIES), - } +# --------------------------------------------------------------------------- +# Compatibility re-exports — symbols that were accessible via web.core +# in the old monolithic module, re-exported for backward compatibility. +# --------------------------------------------------------------------------- +from src.data_collection.city_registry import CITY_REGISTRY # noqa: E402, F401 +from src.data_collection.city_risk_profiles import CITY_RISK_PROFILES # noqa: E402, F401 +from src.payments import PAYMENT_CHECKOUT, PaymentCheckoutError # noqa: E402, F401 +from src.auth.supabase_entitlement import SUPABASE_ENTITLEMENT # noqa: E402, F401 diff --git a/web/diagnostics/health.py b/web/diagnostics/health.py new file mode 100644 index 00000000..0822914d --- /dev/null +++ b/web/diagnostics/health.py @@ -0,0 +1,371 @@ +"""Health check and system status diagnostics for PolyWeather.""" + +import json +import os +from datetime import datetime, timezone +from typing import Any, Dict, Optional + +from src.data_collection.country_networks import provider_coverage_summary +from src.utils.metrics import build_metrics_summary, gauge_set + + +def _env_bool(name: str, default: bool = False) -> bool: + raw = os.getenv(name) + if raw is None: + return default + return raw.strip().lower() in {"1", "true", "yes", "on"} + + +def _sqlite_health(account_db) -> Dict[str, Any]: + try: + from src.database.sqlite_connection import connect_sqlite + with connect_sqlite(account_db.db_path, timeout=0.05) as conn: + conn.execute("SELECT 1").fetchone() + return {"ok": True, "db_path": account_db.db_path} + except Exception as exc: + return {"ok": False, "db_path": account_db.db_path, "error": str(exc)} + + +def _cache_summary(weather_collector, analysis_cache, cache_entries: int) -> Dict[str, Any]: + from web.analysis_service import get_analysis_cache_stats + + open_meteo_forecast_entries = len(getattr(weather_collector, "_open_meteo_cache", {}) or {}) + open_meteo_ensemble_entries = len(getattr(weather_collector, "_ensemble_cache", {}) or {}) + open_meteo_multi_model_entries = len( + getattr(weather_collector, "_multi_model_cache", {}) or {} + ) + metar_entries = weather_collector.cache.store_size("metar") if hasattr(weather_collector, "cache") else 0 + taf_entries = len(getattr(weather_collector, "_taf_cache", {}) or {}) + settlement_entries = len(getattr(weather_collector, "_settlement_cache", {}) or {}) + + gauge_set("polyweather_api_cache_entries", cache_entries) + gauge_set("polyweather_open_meteo_forecast_cache_entries", open_meteo_forecast_entries) + gauge_set("polyweather_open_meteo_ensemble_cache_entries", open_meteo_ensemble_entries) + gauge_set( + "polyweather_open_meteo_multi_model_cache_entries", + open_meteo_multi_model_entries, + ) + gauge_set("polyweather_metar_cache_entries", metar_entries) + gauge_set("polyweather_taf_cache_entries", taf_entries) + gauge_set("polyweather_settlement_cache_entries", settlement_entries) + return { + "api_cache_entries": cache_entries, + "open_meteo_forecast_entries": open_meteo_forecast_entries, + "open_meteo_ensemble_entries": open_meteo_ensemble_entries, + "open_meteo_multi_model_entries": open_meteo_multi_model_entries, + "metar_entries": metar_entries, + "taf_entries": taf_entries, + "settlement_entries": settlement_entries, + "analysis": get_analysis_cache_stats(), + } + + +def _feature_flags_summary(config: dict) -> Dict[str, Any]: + from src.auth.supabase_entitlement import SUPABASE_ENTITLEMENT + from src.payments import PAYMENT_CHECKOUT + from src.database.runtime_state import get_state_storage_mode + + _SUPABASE_AUTH_REQUIRED = _env_bool( + "POLYWEATHER_AUTH_REQUIRED", + SUPABASE_ENTITLEMENT.enabled, + ) + _ENTITLEMENT_GUARD_ENABLED = _env_bool("POLYWEATHER_REQUIRE_ENTITLEMENT", False) + + return { + "auth_enabled": bool(SUPABASE_ENTITLEMENT.enabled), + "auth_required": bool(_SUPABASE_AUTH_REQUIRED), + "entitlement_guard_enabled": bool(_ENTITLEMENT_GUARD_ENABLED), + "payment_enabled": bool(getattr(PAYMENT_CHECKOUT, "enabled", False)), + "state_storage_mode": get_state_storage_mode(), + } + + +def _integration_summary(config: dict) -> Dict[str, Any]: + from src.auth.supabase_entitlement import SUPABASE_ENTITLEMENT + + weather_cfg = config.get("weather", {}) if isinstance(config, dict) else {} + return { + "supabase_configured": bool(SUPABASE_ENTITLEMENT.configured), + "telegram_bot_configured": bool( + (config.get("telegram", {}) or {}).get("bot_token") + ), + "walletconnect_configured": bool( + os.getenv("NEXT_PUBLIC_WALLETCONNECT_PROJECT_ID") + ), + "weather_sources": { + "openweather": bool(weather_cfg.get("openweather_api_key")), + "wunderground": bool(weather_cfg.get("wunderground_api_key")), + "visualcrossing": bool(weather_cfg.get("visualcrossing_api_key")), + }, + } + + +def _probability_summary() -> Dict[str, Any]: + return { + "engine_mode": "legacy", + } + + +def _read_json_file(path: str) -> Optional[Dict[str, Any]]: + if not os.path.exists(path): + return None + try: + with open(path, "r", encoding="utf-8") as fh: + payload = json.load(fh) + if isinstance(payload, dict): + return payload + except Exception: + return None + return None + + +def _table_date_summary(conn, table_name: str) -> Dict[str, Any]: + try: + row = conn.execute( + f""" + SELECT COUNT(*) AS row_count, + COUNT(DISTINCT city) AS cities_count, + MIN(target_date) AS min_date, + MAX(target_date) AS max_date + FROM {table_name} + """ + ).fetchone() + except Exception as exc: + return {"ok": False, "error": str(exc), "row_count": 0, "cities_count": 0} + + return { + "ok": True, + "row_count": int(row["row_count"] or 0), + "cities_count": int(row["cities_count"] or 0), + "min_date": row["min_date"], + "max_date": row["max_date"], + } + + +def _truth_source_counts(conn) -> Dict[str, int]: + try: + rows = conn.execute( + """ + SELECT COALESCE(NULLIF(TRIM(settlement_source), ''), 'unknown') AS settlement_source, + COUNT(*) AS row_count + FROM truth_records_store + GROUP BY COALESCE(NULLIF(TRIM(settlement_source), ''), 'unknown') + ORDER BY row_count DESC, settlement_source ASC + """ + ).fetchall() + except Exception: + return {} + return {str(row["settlement_source"]): int(row["row_count"] or 0) for row in rows} + + +def _truth_revisions_summary(conn) -> Dict[str, Any]: + try: + row = conn.execute( + """ + SELECT COUNT(*) AS row_count, + MAX(updated_at) AS last_updated_at + FROM truth_revisions_store + """ + ).fetchone() + except Exception as exc: + return {"ok": False, "error": str(exc), "row_count": 0} + return { + "ok": True, + "row_count": int(row["row_count"] or 0), + "last_updated_at": row["last_updated_at"], + } + + +def _city_coverage_summary(conn, city_registry) -> Dict[str, Any]: + truth_rows = conn.execute( + """ + SELECT city, COUNT(*) AS row_count, MIN(target_date) AS min_date, MAX(target_date) AS max_date + FROM truth_records_store + GROUP BY city + """ + ).fetchall() + feature_rows = conn.execute( + """ + SELECT city, COUNT(*) AS row_count, MIN(target_date) AS min_date, MAX(target_date) AS max_date + FROM training_feature_records_store + GROUP BY city + """ + ).fetchall() + + truth_index = { + str(row["city"]): { + "truth_rows": int(row["row_count"] or 0), + "truth_min_date": row["min_date"], + "truth_max_date": row["max_date"], + } + for row in truth_rows + } + feature_index = { + str(row["city"]): { + "feature_rows": int(row["row_count"] or 0), + "feature_min_date": row["min_date"], + "feature_max_date": row["max_date"], + } + for row in feature_rows + } + + entries = [] + for city, meta in city_registry.items(): + truth_payload = truth_index.get(city, {}) + feature_payload = feature_index.get(city, {}) + entries.append( + { + "city": city, + "name": str(meta.get("name") or city), + "settlement_source": str(meta.get("settlement_source") or "metar"), + "settlement_station_code": str( + meta.get("settlement_station_code") or meta.get("icao") or "" + ), + "truth_rows": int(truth_payload.get("truth_rows") or 0), + "feature_rows": int(feature_payload.get("feature_rows") or 0), + "truth_min_date": truth_payload.get("truth_min_date"), + "truth_max_date": truth_payload.get("truth_max_date"), + "feature_min_date": feature_payload.get("feature_min_date"), + "feature_max_date": feature_payload.get("feature_max_date"), + } + ) + + highlighted = [ + entry for entry in entries if entry["city"] in {"taipei", "shenzhen"} + ] + gaps = sorted( + entries, + key=lambda entry: ( + entry["feature_rows"] > 0, + entry["truth_rows"] > 0, + entry["truth_rows"], + entry["feature_rows"], + entry["city"], + ), + )[:10] + return { + "total_cities": len(entries), + "with_truth_rows": sum(1 for entry in entries if entry["truth_rows"] > 0), + "with_feature_rows": sum(1 for entry in entries if entry["feature_rows"] > 0), + "entries": entries, + "highlighted": highlighted, + "top_gaps": gaps, + } + + +def _model_city_coverage_summary(city_entries) -> Dict[str, Any]: + rows = [] + for entry in city_entries or []: + city = str(entry.get("city") or "").strip().lower() + rows.append( + { + "city": city, + "name": entry.get("name") or city, + "settlement_source": entry.get("settlement_source"), + "truth_rows": int(entry.get("truth_rows") or 0), + "feature_rows": int(entry.get("feature_rows") or 0), + } + ) + + weakest = sorted( + rows, + key=lambda row: ( + row["truth_rows"] > 0, + row["truth_rows"], + row["city"], + ), + )[:12] + strongest = sorted( + rows, + key=lambda row: ( + -row["truth_rows"], + row["city"], + ), + )[:8] + return { + "weakest": weakest, + "strongest": strongest, + } + + +def _training_data_summary(account_db, city_registry) -> Dict[str, Any]: + from src.database.sqlite_connection import connect_sqlite + import sqlite3 + + db_path = account_db.db_path + truth_records = {"ok": False, "row_count": 0, "cities_count": 0} + truth_revisions = {"ok": False, "row_count": 0} + training_features = {"ok": False, "row_count": 0, "cities_count": 0} + try: + with connect_sqlite(db_path, row_factory=sqlite3.Row) as conn: + truth_records = _table_date_summary(conn, "truth_records_store") + if truth_records.get("ok"): + truth_records["source_counts"] = _truth_source_counts(conn) + truth_revisions = _truth_revisions_summary(conn) + training_features = _table_date_summary( + conn, "training_feature_records_store" + ) + city_coverage = _city_coverage_summary(conn, city_registry) + except Exception as exc: + return { + "db_path": db_path, + "db_ok": False, + "error": str(exc), + "truth_records": truth_records, + "truth_revisions": truth_revisions, + "training_features": training_features, + "city_coverage": {}, + "model_city_coverage": {}, + } + + return { + "db_path": db_path, + "db_ok": True, + "truth_records": truth_records, + "truth_revisions": truth_revisions, + "training_features": training_features, + "city_coverage": city_coverage, + "model_city_coverage": _model_city_coverage_summary( + city_coverage.get("entries") or [], + ), + } + + +def build_health_payload(account_db, cities_count: int) -> Dict[str, Any]: + from src.database.runtime_state import get_state_storage_mode + + db = _sqlite_health(account_db) + return { + "status": "ok" if db.get("ok") else "degraded", + "time_utc": datetime.now(timezone.utc).isoformat(), + "db": db, + "state_storage_mode": get_state_storage_mode(), + "cities_count": cities_count, + } + + +def build_system_status_payload( + account_db, + config: dict, + weather_collector, + analysis_cache, + cache_entries: int, + cities_count: int, + city_registry, +) -> Dict[str, Any]: + from src.database.runtime_state import get_state_storage_mode + + return { + "status": build_health_payload(account_db, cities_count)["status"], + "time_utc": datetime.now(timezone.utc).isoformat(), + "state_storage_mode": get_state_storage_mode(), + "db": _sqlite_health(account_db), + "features": _feature_flags_summary(config), + "integrations": _integration_summary(config), + "cache": _cache_summary(weather_collector, analysis_cache, cache_entries), + "metrics": build_metrics_summary(), + "probability": _probability_summary(), + "training_data": _training_data_summary(account_db, city_registry), + "station_networks": provider_coverage_summary(), + "cities_count": cities_count, + } diff --git a/web/middleware/http.py b/web/middleware/http.py new file mode 100644 index 00000000..ebbd4d8f --- /dev/null +++ b/web/middleware/http.py @@ -0,0 +1,78 @@ +"""HTTP middlewares for the PolyWeather FastAPI application.""" + +import hashlib +import time + +from fastapi import Request +from fastapi.responses import Response + +from src.utils.metrics import counter_inc, histogram_observe + + +async def metrics_middleware(request: Request, call_next): + started = time.perf_counter() + try: + response = await call_next(request) + except Exception: + duration_ms = (time.perf_counter() - started) * 1000.0 + counter_inc( + "polyweather_http_requests_total", + method=request.method, + path=request.url.path, + status="500", + ) + histogram_observe( + "polyweather_http_request_duration_ms", + duration_ms, + method=request.method, + path=request.url.path, + status="500", + ) + raise + + duration_ms = (time.perf_counter() - started) * 1000.0 + status_code = str(response.status_code) + counter_inc( + "polyweather_http_requests_total", + method=request.method, + path=request.url.path, + status=status_code, + ) + histogram_observe( + "polyweather_http_request_duration_ms", + duration_ms, + method=request.method, + path=request.url.path, + status=status_code, + ) + return response + + +async def etag_middleware(request: Request, call_next): + """Add ETag to GET /api/* responses; return 304 on If-None-Match hit.""" + response = await call_next(request) + + if request.method != "GET" or response.status_code != 200: + return response + + path = request.url.path + if not path.startswith("/api/") or path.endswith("/stream"): + return response + + body = getattr(response, "body", None) or b"" + if not body: + return response + + try: + etag = hashlib.md5(body).hexdigest() + except Exception: + return response + + etag_value = f'"{etag}"' + if_none_match = request.headers.get("If-None-Match", "") + if if_none_match == etag_value: + return Response(status_code=304, headers={"ETag": etag_value}) + + response.headers["ETag"] = etag_value + response.headers["Cache-Control"] = "private, max-age=30" + return response diff --git a/web/realtime_event_store.py b/web/realtime_event_store.py index ef0c02df..badef81c 100644 --- a/web/realtime_event_store.py +++ b/web/realtime_event_store.py @@ -9,6 +9,7 @@ from datetime import datetime, timedelta, timezone from typing import Any, Dict, Optional, Set from src.database.db_manager import DBManager +from src.database.sqlite_connection import connect_sqlite from web.realtime_patch_schema import EVENT_TYPE @@ -72,10 +73,7 @@ class RealtimeEventStore: self._ensure_table() def _connect(self) -> sqlite3.Connection: - conn = sqlite3.connect(self.db_path, timeout=10) - conn.execute("PRAGMA journal_mode=WAL") - conn.execute("PRAGMA busy_timeout=5000") - return conn + return connect_sqlite(self.db_path, timeout=10) def _ensure_table(self) -> None: db_dir = os.path.dirname(self.db_path) diff --git a/web/schemas/auth.py b/web/schemas/auth.py new file mode 100644 index 00000000..a46f4d9c --- /dev/null +++ b/web/schemas/auth.py @@ -0,0 +1,48 @@ +"""Pydantic request models for PolyWeather auth-related endpoints.""" + +from typing import Optional + +from pydantic import BaseModel, Field + + +class TelegramLoginRequest(BaseModel): + id: int + first_name: Optional[str] = None + last_name: Optional[str] = None + username: Optional[str] = None + photo_url: Optional[str] = None + auth_date: int + hash: str = Field(..., min_length=10) + + +class TelegramBindTokenRequest(BaseModel): + token: str = Field(..., min_length=8) + + +class ReferralApplyRequest(BaseModel): + code: str = Field(..., min_length=3, max_length=32) + + +class AnalyticsEventRequest(BaseModel): + event_type: str = Field(..., min_length=3, max_length=64) + client_id: Optional[str] = Field(default=None, max_length=128) + session_id: Optional[str] = Field(default=None, max_length=128) + payload: dict = Field(default_factory=dict) + + +class UserFeedbackRequest(BaseModel): + category: str = Field(default="bug", min_length=2, max_length=40) + message: str = Field(..., min_length=3, max_length=5000) + source: str = Field(default="terminal", max_length=40) + contact: Optional[str] = Field(default=None, max_length=180) + context: dict = Field(default_factory=dict) + + +class GrantPointsRequest(BaseModel): + email: str = Field(..., min_length=3) + points: int = Field(..., gt=0, le=100000) + + +class FeedbackRewardRequest(BaseModel): + points: int = Field(..., gt=0, le=100000) + reason: str = Field(default="", max_length=500) diff --git a/web/schemas/payments.py b/web/schemas/payments.py new file mode 100644 index 00000000..f82beeaf --- /dev/null +++ b/web/schemas/payments.py @@ -0,0 +1,47 @@ +"""Pydantic request models for PolyWeather payment endpoints.""" + +from typing import Any, Dict, Optional + +from pydantic import BaseModel, Field + + +class WalletBindRequest(BaseModel): + address: str = Field(..., min_length=8) + + +class WalletChallengeRequest(BaseModel): + address: str = Field(..., min_length=8) + + +class WalletVerifyRequest(BaseModel): + address: str = Field(..., min_length=8) + nonce: str = Field(..., min_length=6) + signature: str = Field(..., min_length=20) + + +class WalletUnbindRequest(BaseModel): + address: str = Field(..., min_length=8) + + +class CreatePaymentIntentRequest(BaseModel): + plan_code: str = Field(default="pro_monthly", min_length=2) + payment_mode: str = Field(default="strict") + allowed_wallet: Optional[str] = None + token_address: Optional[str] = None + chain_id: Optional[int] = None + use_points: bool = False + points_to_consume: Optional[int] = None + metadata: Dict[str, Any] = Field(default_factory=dict) + + +class SubmitPaymentTxRequest(BaseModel): + tx_hash: str = Field(..., min_length=10) + from_address: Optional[str] = None + + +class ValidatePaymentTxRequest(BaseModel): + tx_hash: str = Field(..., min_length=10) + + +class ConfirmPaymentTxRequest(BaseModel): + tx_hash: Optional[str] = None diff --git a/web/services/intraday_meteorology.py b/web/services/intraday_meteorology.py new file mode 100644 index 00000000..4a98d64a --- /dev/null +++ b/web/services/intraday_meteorology.py @@ -0,0 +1,311 @@ +"""Intraday meteorology — paid-product signal layer. + +Reads existing analysis layers (current, probabilities, DEB, peak, deviation, +TAF, vertical profile, station network) and produces a structured meteorology +read with headline, confidence, signals, and invalidation/confirmation rules. +""" + +from __future__ import annotations + +from typing import Any, Dict + +from web.services.analysis_utils import ( + add_signal as _add_signal, + bucket_label as _bucket_label, + bucket_label_from_value as _bucket_label_from_value, + format_clock_minutes as _format_clock_minutes, + next_observation_clock as _next_observation_clock, + top_probability_bucket as _top_probability_bucket, +) + + +def _sf(v: Any) -> Any: + if v is None: + return None + try: + return float(v) + except Exception: + return None + + +def build_intraday_meteorology(data: Dict[str, Any]) -> Dict[str, Any]: + """Build a paid-product intraday meteorology read from existing layers.""" + current = data.get("current") or {} + probabilities = data.get("probabilities") or {} + distribution = probabilities.get("distribution") or [] + top_bucket = _top_probability_bucket(distribution) + unit = str(data.get("temp_symbol") or "°C") + deb = data.get("deb") or {} + peak = data.get("peak") or {} + deviation = data.get("deviation_monitor") or {} + taf_signal = ( + ((data.get("taf") or {}).get("signal") or {}) + if isinstance(data.get("taf"), dict) + else {} + ) + vertical = data.get("vertical_profile_signal") or {} + + current_temp = _sf(current.get("temp")) + max_so_far = _sf(current.get("max_so_far")) + deb_prediction = _sf(deb.get("prediction")) + base_value = _sf(top_bucket.get("value")) if isinstance(top_bucket, dict) else None + if base_value is None: + base_value = deb_prediction + if base_value is None: + base_value = max_so_far if max_so_far is not None else current_temp + + base_case_bucket = _bucket_label(top_bucket, unit) or _bucket_label_from_value(base_value, unit) + upside_bucket = _bucket_label_from_value(base_value + 1.0, unit) if base_value is not None else None + downside_bucket = _bucket_label_from_value(base_value - 1.0, unit) if base_value is not None else None + + signals: list = [] + support_score = 0 + suppress_score = 0 + available_layers = 0 + + direction = str(deviation.get("direction") or "").lower() + severity = str(deviation.get("severity") or "normal").lower() + delta = _sf(deviation.get("current_delta")) + if direction: + available_layers += 1 + strength = "strong" if severity == "strong" else ("medium" if severity == "light" else "weak") + if direction == "hot": + support_score += 2 if strength == "strong" else 1 + _add_signal( + signals, + label="日内节奏", + label_en="Intraday pace", + direction="support", + strength=strength, + summary=f"实测较预期路径偏高 {abs(delta or 0):.1f}{unit}，峰值仍有上修空间。", + summary_en=f"Observed temperature is running {abs(delta or 0):.1f}{unit} above the expected path; the peak still has upside room.", + ) + elif direction == "cold": + suppress_score += 2 if strength == "strong" else 1 + _add_signal( + signals, + label="日内节奏", + label_en="Intraday pace", + direction="suppress", + strength=strength, + summary=f"实测较预期路径偏低 {abs(delta or 0):.1f}{unit}，追更高温档需要等待后续观测确认。", + summary_en=f"Observed temperature is running {abs(delta or 0):.1f}{unit} below the expected path; higher buckets need confirmation from later observations.", + ) + else: + _add_signal( + signals, + label="日内节奏", + label_en="Intraday pace", + direction="neutral", + strength="weak", + summary="实测大体贴近当前预期路径，下一步主要看峰值窗口内是否继续抬升。", + summary_en="Observed temperature is broadly tracking the expected path; the next question is whether it keeps lifting through the peak window.", + ) + + heating_setup = str(vertical.get("heating_setup") or "").lower() + suppression_risk = str(vertical.get("suppression_risk") or "").lower() + if heating_setup or suppression_risk: + available_layers += 1 + if heating_setup == "supportive": + support_score += 2 + _add_signal( + signals, + label="边界层结构", + label_en="Boundary-layer setup", + direction="support", + strength="strong", + summary=str(vertical.get("summary_zh") or "边界层结构支持白天继续混合升温。"), + summary_en=str(vertical.get("summary_en") or "The boundary-layer setup supports continued daytime mixing and warming."), + ) + elif heating_setup == "suppressed" or suppression_risk == "high": + suppress_score += 2 + _add_signal( + signals, + label="边界层结构", + label_en="Boundary-layer setup", + direction="suppress", + strength="strong", + summary=str(vertical.get("summary_zh") or "边界层或云雨结构对午后峰值形成压制。"), + summary_en=str(vertical.get("summary_en") or "Boundary-layer or cloud/rain structure is capping the afternoon peak."), + ) + else: + _add_signal( + signals, + label="边界层结构", + label_en="Boundary-layer setup", + direction="neutral", + strength="medium", + summary=str(vertical.get("summary_zh") or "边界层结构暂未给出单边信号。"), + summary_en=str(vertical.get("summary_en") or "The boundary-layer setup does not yet provide a one-sided signal."), + ) + + taf_suppression = str(taf_signal.get("suppression_level") or "").lower() + taf_disruption = str(taf_signal.get("disruption_level") or "").lower() + taf_has_cloud_rain_cap = taf_suppression in {"medium", "high"} or taf_disruption in { + "medium", + "high", + } + + structural_cap = False + if taf_signal.get("available") or taf_suppression: + available_layers += 1 + if taf_suppression == "high" or taf_disruption == "high": + suppress_score += 2 + direction_value = "suppress" + strength = "strong" + elif taf_suppression == "medium" or taf_disruption == "medium": + suppress_score += 1 + direction_value = "suppress" + strength = "medium" + else: + support_score += 1 + direction_value = "support" + strength = "weak" + _add_signal( + signals, + label="TAF 云雨扰动", + label_en="TAF cloud/rain disruption", + direction=direction_value, + strength=strength, + summary=str(taf_signal.get("summary_zh") or "TAF 暂未提示强云雨压温信号。"), + summary_en=str(taf_signal.get("summary_en") or "TAF does not yet flag a strong cloud/rain temperature cap."), + ) + + airport_delta = _sf(data.get("airport_vs_network_delta")) + lead_signal = data.get("network_lead_signal") or {} + if airport_delta is not None: + available_layers += 1 + leader = str(lead_signal.get("leader_station_label") or lead_signal.get("leader_station_code") or "").strip() + sync_status = str(lead_signal.get("leader_sync_status") or "").strip().lower() + sync_delta = _sf(lead_signal.get("leader_time_delta_vs_anchor_minutes")) + sync_suffix_zh = "" + sync_suffix_en = "" + if sync_status in {"near_realtime", "lagged"} and sync_delta is not None: + sync_suffix_zh = f"；但与机场锚点约差 {sync_delta:.0f} 分钟，作为降权信号处理" + sync_suffix_en = f"; timing differs from the airport anchor by about {sync_delta:.0f} minutes, so this signal is down-weighted" + elif sync_status == "unknown": + sync_suffix_zh = "；周边站观测时间不可完全校验，作为弱参考" + sync_suffix_en = "; station timing is not fully verified, so this is treated as a weak reference" + if airport_delta <= -0.4: + support_score += 1 + _add_signal( + signals, + label="站网对比", + label_en="Station-network comparison", + direction="support", + strength="weak" if sync_suffix_zh else "medium", + summary=f"周边站网较机场锚点偏热 {abs(airport_delta):.1f}{unit}{f'，领先点位 {leader}' if leader else ''}{sync_suffix_zh}。", + summary_en=f"Nearby stations are {abs(airport_delta):.1f}{unit} warmer than the airport anchor{f'; leading site: {leader}' if leader else ''}{sync_suffix_en}.", + ) + elif airport_delta >= 0.4: + suppress_score += 1 + _add_signal( + signals, + label="站网对比", + label_en="Station-network comparison", + direction="suppress", + strength="weak" if sync_suffix_zh else "medium", + summary=f"机场锚点较周边站网偏热 {abs(airport_delta):.1f}{unit}，继续上修需要机场自身后续报文确认{sync_suffix_zh}。", + summary_en=f"The airport anchor is {abs(airport_delta):.1f}{unit} warmer than nearby stations; further upside needs confirmation from later airport reports{sync_suffix_en}.", + ) + else: + _add_signal( + signals, + label="站网对比", + label_en="Station-network comparison", + direction="neutral", + strength="weak", + summary="机场锚点与周边站网基本同步，暂不构成单独上修或下修理由。", + summary_en="The airport anchor and nearby station network are broadly aligned, so this layer does not independently argue for upside or downside.", + ) + + peak_status = str(peak.get("status") or "").lower() + first_h = _sf(peak.get("first_h")) + last_h = _sf(peak.get("last_h")) + peak_window = ( + f"{int(first_h):02d}:00-{int(last_h):02d}:59" + if first_h is not None and last_h is not None + else "--" + ) + if peak_status == "past": + headline = "峰值窗口已过，后续更偏向确认最终高点而非继续上修。" + headline_en = "The peak window has passed; the read now shifts toward confirming the final high rather than chasing further upside." + confidence = "high" if available_layers >= 2 else "medium" + elif suppress_score >= support_score + 2: + structural_cap = any( + signal.get("direction") == "suppress" + and signal.get("label") in {"边界层结构", "站网对比", "日内节奏"} + for signal in signals + ) + if taf_has_cloud_rain_cap and structural_cap: + headline = "峰值同时存在 TAF 云雨扰动和结构压制，当前更偏防守高温上修。" + headline_en = "Both TAF cloud/rain disruption and structural signals are capping the peak; defend against aggressive high-temperature upside for now." + elif taf_has_cloud_rain_cap: + headline = "TAF 提示峰值窗口有云雨扰动，当前更偏防守高温上修。" + headline_en = "TAF flags cloud/rain disruption near the peak window; defend against aggressive high-temperature upside for now." + else: + headline = "峰值主要受结构信号压制，TAF 云雨层暂未构成主压温理由。" + headline_en = "The peak is mainly capped by structural signals; TAF cloud/rain is not the primary suppression reason for now." + confidence = "high" if available_layers >= 3 else "medium" + elif support_score >= suppress_score + 2: + headline = "峰值仍有上修空间，后续重点看峰值窗口内报文能否继续抬升。" + headline_en = "The peak still has upside room; the next check is whether reports keep lifting through the peak window." + confidence = "high" if available_layers >= 3 else "medium" + elif available_layers == 0: + headline = "关键日内层仍在补齐，先以观测锚点和下一次报文为主。" + headline_en = "Key intraday layers are still filling in; anchor the read on observations and the next report." + confidence = "low" + else: + headline = "当前处于分歧判断区，峰值窗口内的下一组观测将决定方向。" + headline_en = "The setup is in a split-decision zone; the next observations inside the peak window should decide direction." + confidence = "medium" if available_layers >= 2 else "low" + + next_observation = _next_observation_clock(data.get("local_time") or current.get("obs_time")) + threshold = base_value + invalidation_rules = [] + invalidation_rules_en = [] + confirmation_rules = [] + confirmation_rules_en = [] + if peak_status == "past": + invalidation_rules.append("若后续官方结算源补录更高值，以结算源最终高点为准。") + invalidation_rules_en.append("If the official settlement source later backfills a higher reading, defer to the final settlement-source high.") + confirmation_rules.append("若峰值窗口后连续两次观测不再创新高，当前高点基本确认。") + confirmation_rules_en.append("If two consecutive post-peak observations fail to make a new high, the current high is broadly confirmed.") + else: + watch_clock = _format_clock_minutes(int(first_h or 13) * 60 + 30) + if threshold is not None: + invalidation_rules.append(f"{watch_clock} 前若仍未接近 {threshold:.0f}{unit}，上修路径降级。") + invalidation_rules_en.append(f"If observations are still not near {threshold:.0f}{unit} before {watch_clock}, downgrade the upside path.") + confirmation_rules.append(f"峰值窗口内任一结算源观测触达或超过 {threshold:.0f}{unit}，基准路径确认度上升。") + confirmation_rules_en.append(f"If any settlement-source observation reaches or exceeds {threshold:.0f}{unit} inside the peak window, confidence in the base path rises.") + invalidation_rules.append("若 TAF 或实况报文出现阵雨、雷暴或低云/云雨压制，高温上沿需要下调。") + invalidation_rules_en.append("If TAF or live reports show showers, thunderstorms, or low-cloud/cloud-rain suppression, lower the upper temperature bound.") + confirmation_rules.append("若实测继续贴近 DEB 曲线且云雨信号不增强，维持当前主路径。") + confirmation_rules_en.append("If observations keep tracking the DEB curve and cloud/rain signals do not strengthen, maintain the current main path.") + + if not signals: + _add_signal( + signals, + label="数据完整性", + label_en="Data completeness", + direction="neutral", + strength="weak", + summary="当前缺少足够的日内结构层，等待下一次观测刷新后再提高判断权重。", + summary_en="There are not enough intraday structure layers yet; wait for the next observation refresh before raising confidence.", + ) + + return { + "headline": headline, + "headline_en": headline_en, + "confidence": confidence, + "base_case_bucket": base_case_bucket, + "upside_bucket": upside_bucket, + "downside_bucket": downside_bucket, + "next_observation_time": next_observation, + "peak_window": peak_window, + "invalidation_rules": invalidation_rules[:4], + "invalidation_rules_en": invalidation_rules_en[:4], + "confirmation_rules": confirmation_rules[:3], + "confirmation_rules_en": confirmation_rules_en[:3], + "signal_contributions": signals[:5], + } diff --git a/web/services/ops/__init__.py b/web/services/ops/__init__.py new file mode 100644 index 00000000..45e2e9fc --- /dev/null +++ b/web/services/ops/__init__.py @@ -0,0 +1 @@ +"""Ops service sub-package.""" diff --git a/web/services/ops/config.py b/web/services/ops/config.py new file mode 100644 index 00000000..cdfb1a78 --- /dev/null +++ b/web/services/ops/config.py @@ -0,0 +1,724 @@ +"""Ops config / subscriptions / logs / telegram service functions.""" +from __future__ import annotations + +import concurrent.futures +import os +import sqlite3 +import subprocess +from datetime import datetime, timedelta, timezone +from typing import Any, Dict, List, Optional + +import requests as _requests +from fastapi import HTTPException, Request + +from src.database.db_manager import DBManager # type hints +from src.utils.runtime_secrets import get_runtime_secret_status + + +def _get_db(): + from src.database.db_manager import DBManager as _DBManager + return _DBManager() + + +# ── Config key definitions ────────────────────────────────────────── + +_EDITABLE_CONFIG_KEYS: dict[str, str] = { + "POLYWEATHER_AUTH_REQUIRED": "是否强制要求 Supabase 登录访问 API", + "POLYWEATHER_PAYMENT_ENABLED": "是否启用支付功能", + "POLYWEATHER_PAYMENT_POINTS_ENABLED": "是否启用积分抵扣", + "POLYWEATHER_TELEGRAM_ALERT_PUSH_ENABLED": "是否启用 Telegram 告警推送", + "POLYWEATHER_GROUP_MEMBER_PRICE_USDC": "群成员月费 (USDC)", + "POLYWEATHER_PUBLIC_PRICE_USDC": "公开月费 (USDC)", + "POLYWEATHER_PAYMENT_POINTS_PER_USDC": "积分兑换汇率 (积分/USDC)", + "POLYWEATHER_PAYMENT_POINTS_MAX_DISCOUNT_USDC": "积分最高抵扣金额 (USDC)", + "POLYWEATHER_PAYMENT_DIRECT_RECEIVER_ADDRESS": "手动转账收款钱包地址", +} + +_SENSITIVE_CONFIG_KEYS: dict[str, dict[str, str]] = { + "POLYWEATHER_AMSC_SESSION_ID": { + "label": "AMSC AWOS sessionId", + "description": "中国跑道观测接口 sessionId，用于上海/北京/广州等 AMSC AWOS 数据源。", + }, +} + + +# ── Helpers ───────────────────────────────────────────────────────── + + +def _require_ops(request: Request) -> Dict[str, Any] | None: + from web.services.ops_api import _require_ops as _real + return _real(request) + + +def _parse_iso_datetime(value: Any) -> Optional[datetime]: + text = str(value or "").strip() + if not text: + return None + try: + parsed = datetime.fromisoformat(text.replace("Z", "+00:00")) + except Exception: + return None + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=timezone.utc) + return parsed.astimezone(timezone.utc) + + +def _to_utc_iso(value: datetime) -> str: + if value.tzinfo is None: + value = value.replace(tzinfo=timezone.utc) + return value.astimezone(timezone.utc).isoformat().replace("+00:00", "Z") + + +def _supabase_rest_rows( + table: str, + params: Dict[str, Any], + *, + timeout: int = 10, +) -> List[Dict[str, Any]]: + supabase_url = str(os.getenv("SUPABASE_URL") or "").strip().rstrip("/") + service_role_key = str(os.getenv("SUPABASE_SERVICE_ROLE_KEY") or "").strip() + if not supabase_url or not service_role_key: + raise HTTPException(status_code=503, detail="Supabase not configured") + + headers = { + "apikey": service_role_key, + "Authorization": f"Bearer {service_role_key}", + } + resp = _requests.get( + f"{supabase_url}/rest/v1/{table}", + headers=headers, + params=params, + timeout=timeout, + ) + if not resp.ok: + raise HTTPException( + status_code=502, + detail=f"Supabase query failed for {table}: {resp.status_code}", + ) + rows = resp.json() if resp.content else [] + if not isinstance(rows, list): + return [] + return [row for row in rows if isinstance(row, dict)] + + +def _supabase_service_headers( + service_role_key: str, + *, + prefer: str | None = None, +) -> dict[str, str]: + headers = { + "apikey": service_role_key, + "Authorization": f"Bearer {service_role_key}", + "Content-Type": "application/json", + } + if prefer: + headers["Prefer"] = prefer + return headers + + +def _lookup_supabase_user_id_by_email( + supabase_url: str, + service_role_key: str, + email: str, +) -> str: + normalized_email = str(email or "").strip().lower() + if not normalized_email: + return "" + base = str(supabase_url or "").strip().rstrip("/") + headers = _supabase_service_headers(service_role_key) + + profile_resp = _requests.get( + f"{base}/rest/v1/profiles", + headers=headers, + params={ + "select": "id", + "email": f"eq.{normalized_email}", + "limit": "1", + }, + timeout=10, + ) + if profile_resp.ok: + profiles = profile_resp.json() if profile_resp.content else [] + if isinstance(profiles, list) and profiles: + user_id = str((profiles[0] or {}).get("id") or "").strip() + if user_id: + return user_id + + user_resp = _requests.get( + f"{base}/auth/v1/admin/users", + headers=headers, + params={"filter": f"email.eq.{normalized_email}"}, + timeout=10, + ) + users = user_resp.json().get("users", []) if user_resp.ok else [] + return str(users[0].get("id") or "").strip() if users else "" + + +# ── Config ────────────────────────────────────────────────────────── + + +def get_ops_config(request: Request) -> dict[str, Any]: + _require_ops(request) + + configs: list[dict[str, Any]] = [] + for key, desc in _EDITABLE_CONFIG_KEYS.items(): + configs.append( + { + "key": key, + "value": os.getenv(key) or "", + "description": desc, + } + ) + return {"configs": configs} + + +def update_ops_config(request: Request, key: str, value: str) -> dict[str, Any]: + _require_ops(request) + + normalized_key = str(key or "").strip() + if normalized_key not in _EDITABLE_CONFIG_KEYS: + raise HTTPException( + status_code=400, detail=f"config key '{normalized_key}' is not editable" + ) + os.environ[normalized_key] = str(value) + return { + "key": normalized_key, + "value": value, + "ok": True, + } + + +def _sensitive_config_payload(key: str) -> dict[str, Any]: + definition = _SENSITIVE_CONFIG_KEYS.get(key) or {} + metadata = get_runtime_secret_status(key) + return { + "key": key, + "label": definition.get("label") or key, + "description": definition.get("description") or "", + "configured": bool(metadata.get("configured")), + "masked": str(metadata.get("masked") or ""), + "length": int(metadata.get("length") or 0), + "updated_at": str(metadata.get("updated_at") or ""), + "updated_by": str(metadata.get("updated_by") or ""), + "source": str(metadata.get("source") or "runtime_store"), + } + + +def get_ops_sensitive_config(request: Request) -> dict[str, Any]: + _require_ops(request) + return { + "configs": [ + _sensitive_config_payload(key) + for key in _SENSITIVE_CONFIG_KEYS + ] + } + + +def update_ops_sensitive_config( + request: Request, + key: str, + value: str, +) -> dict[str, Any]: + admin = _require_ops(request) or {} + normalized_key = str(key or "").strip() + if normalized_key not in _SENSITIVE_CONFIG_KEYS: + raise HTTPException( + status_code=400, + detail=f"sensitive config key '{normalized_key}' is not editable", + ) + secret_value = str(value or "").strip() + if not 12 <= len(secret_value) <= 256 or any(ch.isspace() for ch in secret_value): + raise HTTPException( + status_code=400, + detail="sessionId must be 12-256 non-whitespace characters", + ) + + db = _get_db() + try: + config = db.set_runtime_secret( + normalized_key, + secret_value, + updated_by=str(admin.get("email") or ""), + ) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + + os.environ[normalized_key] = secret_value + response_config = _sensitive_config_payload(normalized_key) + response_config.update( + { + "configured": bool(config.get("configured")), + "masked": str(config.get("masked") or ""), + "length": int(config.get("length") or 0), + "updated_at": str(config.get("updated_at") or ""), + "updated_by": str(config.get("updated_by") or ""), + "source": str(config.get("source") or "runtime_store"), + } + ) + # Lazy import to avoid circular dependency with ops_api + from web.services.ops_api import _check_amsc_awos_health + health = ( + _check_amsc_awos_health(timeout=8) + if normalized_key == "POLYWEATHER_AMSC_SESSION_ID" + else None + ) + return {"ok": True, "config": response_config, "health": health} + + +# ── Subscriptions ─────────────────────────────────────────────────── + + +def grant_ops_subscription( + request: Request, + email: str, + plan_code: str = "pro_monthly", + days: int = 30, + deduct_points: int = 0, +) -> dict[str, Any]: + _require_ops(request) + from datetime import datetime, timedelta + + import web.routes as legacy_routes # lazy – avoid circular import + + supabase_url = str(os.getenv("SUPABASE_URL") or "").strip().rstrip("/") + service_role_key = str(os.getenv("SUPABASE_SERVICE_ROLE_KEY") or "").strip() + if not supabase_url or not service_role_key: + raise HTTPException(status_code=503, detail="Supabase not configured") + + allowed_plans = {"pro_monthly"} + if plan_code not in allowed_plans: + raise HTTPException( + status_code=400, detail=f"invalid plan_code, allowed: {allowed_plans}" + ) + + safe_days = max(1, min(365, int(days or 30))) + safe_deduct = max(0, int(deduct_points or 0)) + normalized_email = str(email or "").strip().lower() + if not normalized_email: + raise HTTPException(status_code=400, detail="email is required") + + user_id = _lookup_supabase_user_id_by_email( + supabase_url, + service_role_key, + normalized_email, + ) + if not user_id: + raise HTTPException( + status_code=404, detail=f"user not found: {normalized_email}" + ) + + now = datetime.utcnow() + starts_at = now.isoformat() + "Z" + expires_at = (now + timedelta(days=safe_days)).isoformat() + "Z" + + payload = { + "user_id": user_id, + "email": normalized_email, + "plan_code": plan_code, + "starts_at": starts_at, + "expires_at": expires_at, + "source": "ops_manual_grant", + "created_at": now.isoformat() + "Z", + } + + resp = _requests.post( + f"{supabase_url}/rest/v1/subscriptions", + headers=_supabase_service_headers(service_role_key, prefer="return=minimal"), + json=payload, + timeout=10, + ) + if not resp.ok: + raise HTTPException( + status_code=500, detail=f"Supabase insert failed: {resp.text[:200]}" + ) + legacy_routes.SUPABASE_ENTITLEMENT.invalidate_subscription_cache(user_id) + + result: dict[str, Any] = { + "ok": True, + "user_id": user_id, + "plan_code": plan_code, + "days": safe_days, + "expires_at": expires_at, + } + + # Optionally deduct points from the user (manual Pro grant with points payment) + if safe_deduct > 0: + db = _get_db() + deduct_result = db.deduct_points_by_supabase_email( + normalized_email, safe_deduct + ) + result["points_deducted"] = safe_deduct + result["points_result"] = deduct_result + + return result + + +def extend_ops_subscription( + request: Request, + email: str, + additional_days: int = 30, +) -> dict[str, Any]: + _require_ops(request) + from datetime import datetime, timedelta + + import web.routes as legacy_routes # lazy – avoid circular import + + supabase_url = str(os.getenv("SUPABASE_URL") or "").strip().rstrip("/") + service_role_key = str(os.getenv("SUPABASE_SERVICE_ROLE_KEY") or "").strip() + if not supabase_url or not service_role_key: + raise HTTPException(status_code=503, detail="Supabase not configured") + + safe_days = max(1, min(365, int(additional_days or 30))) + normalized_email = str(email or "").strip().lower() + if not normalized_email: + raise HTTPException(status_code=400, detail="email is required") + + headers = _supabase_service_headers(service_role_key) + + user_id = _lookup_supabase_user_id_by_email( + supabase_url, + service_role_key, + normalized_email, + ) + if not user_id: + raise HTTPException( + status_code=404, detail=f"user not found: {normalized_email}" + ) + + # Find latest active subscription + subs_resp = _requests.get( + f"{supabase_url}/rest/v1/subscriptions", + headers=headers, + params={ + "select": "id,expires_at", + "user_id": f"eq.{user_id}", + "status": "eq.active", + "order": "expires_at.desc", + "limit": "1", + }, + timeout=10, + ) + subs = subs_resp.json() if subs_resp.ok else [] + if not subs: + raise HTTPException( + status_code=404, detail=f"no subscription found for {normalized_email}" + ) + + sub = subs[0] + current_expiry = sub.get("expires_at", "") + try: + dt = datetime.fromisoformat(current_expiry.replace("Z", "+00:00")) + new_expiry = (dt + timedelta(days=safe_days)).isoformat() + except Exception: + new_expiry = (datetime.utcnow() + timedelta(days=safe_days)).isoformat() + "Z" + + patch_resp = _requests.patch( + f"{supabase_url}/rest/v1/subscriptions?id=eq.{sub['id']}", + headers=_supabase_service_headers(service_role_key, prefer="return=minimal"), + json={"expires_at": new_expiry}, + timeout=10, + ) + if patch_resp.ok: + legacy_routes.SUPABASE_ENTITLEMENT.invalidate_subscription_cache(user_id) + return { + "ok": True, + "email": normalized_email, + "additional_days": safe_days, + "new_expires_at": new_expiry, + } + raise HTTPException( + status_code=500, detail=f"Supabase update failed: {patch_resp.text[:200]}" + ) + + +def get_ops_user_subscriptions( + request: Request, + email: str, +) -> dict[str, Any]: + """Return ALL subscription rows for a user (by email), regardless of status.""" + _require_ops(request) + + supabase_url = str(os.getenv("SUPABASE_URL") or "").strip().rstrip("/") + service_role_key = str(os.getenv("SUPABASE_SERVICE_ROLE_KEY") or "").strip() + if not supabase_url or not service_role_key: + raise HTTPException(status_code=503, detail="Supabase not configured") + + normalized_email = str(email or "").strip().lower() + if not normalized_email: + raise HTTPException(status_code=400, detail="email is required") + + headers = _supabase_service_headers(service_role_key) + + user_id = _lookup_supabase_user_id_by_email( + supabase_url, + service_role_key, + normalized_email, + ) + if not user_id: + raise HTTPException( + status_code=404, detail=f"user not found: {normalized_email}" + ) + + # Fetch all subscription rows for this user (no status filter) + subs_resp = _requests.get( + f"{supabase_url}/rest/v1/subscriptions", + headers=headers, + params={ + "select": "id,user_id,status,plan_code,source,starts_at,expires_at,created_at,updated_at", + "user_id": f"eq.{user_id}", + "order": "created_at.desc", + "limit": "50", + }, + timeout=10, + ) + rows = subs_resp.json() if subs_resp.ok and subs_resp.content else [] + if not isinstance(rows, list): + rows = [] + + return { + "email": normalized_email, + "user_id": user_id, + "subscriptions": rows, + "count": len(rows), + } + + +# ── Logs ──────────────────────────────────────────────────────────── + + +def get_ops_logs( + request: Request, + level: str = "", + lines: int = 100, +) -> dict[str, Any]: + _require_ops(request) + + safe_lines = max(10, min(1000, int(lines or 100))) + log_text = "" + try: + # Read from Docker logs + result = subprocess.run( + ["docker", "logs", "--tail", str(safe_lines), "polyweather_bot"], + capture_output=True, + text=True, + timeout=10, + ) + log_text = result.stdout or result.stderr or "" + except Exception: + pass + + # Fallback to local log file if docker logs returns empty + if not log_text.strip(): + log_file = "data/logs/polyweather.log" + if os.path.exists(log_file): + try: + with open(log_file, "r", encoding="utf-8", errors="ignore") as f: + all_lines = f.readlines() + log_text = "".join(all_lines[-safe_lines:]) + except Exception: + pass + + log_lines = log_text.strip().split("\n") if log_text.strip() else [] + + if level: + level_upper = level.upper() + log_lines = [line for line in log_lines if level_upper in line.upper()] + + return { + "lines": log_lines[-safe_lines:], + "total": len(log_lines), + } + + +# ── Telegram audit ────────────────────────────────────────────────── + + +def get_ops_telegram_audit(request: Request) -> Dict[str, Any]: + _require_ops(request) + + + from src.utils.telegram_chat_ids import get_telegram_chat_ids_from_env + # Lazy imports to avoid circular dependency with ops_api + import web.routes as legacy_routes + from web.services.ops.payments import _list_active_subscriptions_with_windows + + db = _get_db() + + # 1. Fetch all distinct telegram users from database + with db._get_connection() as conn: + conn.row_factory = sqlite3.Row + users_rows = conn.execute("SELECT telegram_id, username FROM users").fetchall() + bindings_rows = conn.execute( + "SELECT telegram_id, supabase_user_id, supabase_email FROM supabase_bindings" + ).fetchall() + + user_info = {} + for r in users_rows: + tid = int(r["telegram_id"]) + user_info[tid] = { + "telegram_id": tid, + "username": r["username"] or f"ID: {tid}", + "supabase_user_id": None, + "supabase_email": None, + "is_bound": False, + } + + for r in bindings_rows: + tid = int(r["telegram_id"]) + if tid not in user_info: + user_info[tid] = { + "telegram_id": tid, + "username": f"ID: {tid}", + "supabase_user_id": r["supabase_user_id"], + "supabase_email": r["supabase_email"], + "is_bound": True, + } + else: + user_info[tid]["supabase_user_id"] = r["supabase_user_id"] + user_info[tid]["supabase_email"] = r["supabase_email"] + user_info[tid]["is_bound"] = True + + # 2. Get Telegram Bot settings + bot_token = str(os.getenv("TELEGRAM_BOT_TOKEN") or "").strip() + chat_ids = get_telegram_chat_ids_from_env() + + # Add other group IDs if configured + for env_name in [ + "POLYWEATHER_TELEGRAM_GROUP_ID", + "POLYWEATHER_TELEGRAM_TOPICS_GROUP_ID", + ]: + val = str(os.getenv(env_name) or "").strip() + if val and val not in chat_ids: + chat_ids.append(val) + + if not bot_token or not chat_ids: + return { + "error": "Telegram Bot Token or Chat IDs not configured", + "anomalies": [], + } + + # 3. Check membership status for all users in parallel + results = [] + + def check_user_chat(tg_id, chat_id): + try: + resp = _requests.get( + f"https://api.telegram.org/bot{bot_token}/getChatMember", + params={"chat_id": chat_id, "user_id": tg_id}, + timeout=5, + ) + if resp.status_code == 200: + data = resp.json() + if data.get("ok"): + res_status = data["result"].get("status") + return chat_id, res_status + return chat_id, None + except Exception: + return chat_id, None + + with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor: + futures = {} + for tg_id in user_info.keys(): + for c_id in chat_ids: + f = executor.submit(check_user_chat, tg_id, c_id) + futures[f] = (tg_id, c_id) + + for f in concurrent.futures.as_completed(futures): + tg_id, c_id = futures[f] + try: + _, status = f.result() + if status in {"creator", "administrator", "member"}: + results.append((tg_id, c_id, status)) + except Exception: + pass + + # 4. Filter and categorize members + anomalies = [] + valid_members = [] + + active_subs, _, used_active_window_query = _list_active_subscriptions_with_windows( + limit=5000 + ) + if not used_active_window_query: + active_subs = legacy_routes.SUPABASE_ENTITLEMENT.list_active_subscriptions( + limit=5000 + ) + active_subs_map = {} + for sub in active_subs: + uid = str(sub.get("user_id") or "").strip().lower() + if uid: + active_subs_map[uid] = sub + + for tg_id, chat_id, status in results: + info = user_info[tg_id] + + if not info["is_bound"]: + anomalies.append( + { + "telegram_id": tg_id, + "username": info["username"], + "chat_id": chat_id, + "status": status, + "anomaly_type": "unbound", + "reason": "未绑定网页账号", + "email": None, + "expires_at": None, + } + ) + else: + uid = str(info["supabase_user_id"]).strip().lower() + sub = active_subs_map.get(uid) + is_paid = False + plan_code = "" + expires_at = None + + if sub: + plan_code = str(sub.get("plan_code") or "").strip().lower() + source = str(sub.get("source") or "").strip().lower() + is_paid = "trial" not in plan_code and "trial" not in source + expires_at = sub.get("expires_at") + + if not sub: + anomalies.append( + { + "telegram_id": tg_id, + "username": info["username"], + "chat_id": chat_id, + "status": status, + "anomaly_type": "expired", + "reason": "没有有效的会员订阅", + "email": info["supabase_email"], + "expires_at": None, + } + ) + elif not is_paid: + anomalies.append( + { + "telegram_id": tg_id, + "username": info["username"], + "chat_id": chat_id, + "status": status, + "anomaly_type": "trial_only", + "reason": f"仅拥有试用会员 ({plan_code})", + "email": info["supabase_email"], + "expires_at": expires_at, + } + ) + else: + valid_members.append( + { + "telegram_id": tg_id, + "username": info["username"], + "chat_id": chat_id, + "status": status, + "email": info["supabase_email"], + "plan_code": plan_code, + "expires_at": expires_at, + } + ) + + return { + "anomalies": anomalies, + "valid_count": len(valid_members), + "anomaly_count": len(anomalies), + } diff --git a/web/services/ops/health.py b/web/services/ops/health.py new file mode 100644 index 00000000..6155b535 --- /dev/null +++ b/web/services/ops/health.py @@ -0,0 +1,1104 @@ +"""Ops health / source health / training accuracy service functions.""" +from __future__ import annotations + +import os +from datetime import datetime, timedelta, timezone +from typing import Any, Dict, List, Optional + +from fastapi import Request +import requests as _requests + +from src.utils.runtime_secrets import get_runtime_secret +from web.services.observation_freshness import ( + build_observation_freshness, + canonical_observation_source_code, +) +import web.routes as legacy_routes + +_DEB_VERSION_BACKTEST_SAMPLE_LIMIT = 400 + + +# ── tiny helpers (also in ops_api.py, copied for self-contained module) ────── + +def _sf(value: Any) -> Optional[float]: + try: + if value is None or value == "": + return None + return float(value) + except (TypeError, ValueError): + return None + + +def _round_metric(value: Optional[float], digits: int = 1) -> Optional[float]: + return None if value is None else round(float(value), digits) + + +# ── AMSC AWOS ──────────────────────────────────────────────────────── + +def _build_amsc_awos_headers() -> dict[str, str]: + headers = { + "Accept": "application/json, text/plain, */*", + "Referer": os.getenv("AMSC_AWOS_REFERER", "https://www.amsc.net.cn/"), + "User-Agent": ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124 Safari/537.36" + ), + } + cookie = get_runtime_secret("POLYWEATHER_AMSC_COOKIE") + session_id = get_runtime_secret("POLYWEATHER_AMSC_SESSION_ID") + if session_id: + headers["sessionId"] = session_id + headers["app"] = "AMS" + elif cookie: + headers["Cookie"] = cookie + return headers + + +def _check_amsc_awos_health(timeout: int = 8) -> dict[str, Any]: + import time as _time + + from src.data_collection.amsc_awos_sources import _amsc_parse_wind_plate_payload + + amsc_base = str(os.getenv("AMSC_AWOS_BASE_URL") or "").strip() + if not amsc_base: + return {"ok": False, "error": "not configured"} + + credential_configured = bool( + get_runtime_secret("POLYWEATHER_AMSC_COOKIE") + or get_runtime_secret("POLYWEATHER_AMSC_SESSION_ID") + ) + try: + t0 = _time.perf_counter() + response = _requests.get( + f"{amsc_base}?cccc=ZSPD", + timeout=timeout, + verify=False, + headers=_build_amsc_awos_headers(), + ) + latency_ms = round((_time.perf_counter() - t0) * 1000) + try: + payload = response.json() if response.content else {} + except ValueError: + payload = {} + parsed = _amsc_parse_wind_plate_payload( + payload if isinstance(payload, dict) else {}, + city_key="shanghai", + icao="ZSPD", + ) + points = ( + ((parsed or {}).get("runway_obs") or {}).get("point_temperatures") + if isinstance(parsed, dict) + else [] + ) + point_count = len(points or []) + ok = bool(response.ok and parsed and point_count > 0) + result: dict[str, Any] = { + "ok": ok, + "status": response.status_code, + "latency_ms": latency_ms, + "credential_configured": credential_configured, + "points": point_count, + } + if isinstance(parsed, dict): + result["sample_city"] = "shanghai" + result["observation_time_local"] = parsed.get("observation_time_local") + if not ok: + result["error"] = "empty_or_unauthorized_response" + return result + except Exception as exc: + return { + "ok": False, + "credential_configured": credential_configured, + "error": str(exc)[:100], + } + + +# ── Source health helpers ───────────────────────────────────────────── + +def _safe_source_text(value: Any) -> str: + return str(value or "").strip() + + +def _source_observed_at(source: dict[str, Any]) -> Any: + for key in ( + "observed_at", + "obs_time", + "report_time", + "observation_time", + "observation_time_local", + "time", + "timestamp", + ): + value = source.get(key) + if _safe_source_text(value): + return value + return None + + +def _source_code(source: dict[str, Any], fallback: str = "") -> str: + for key in ("source_code", "source", "provider_code", "network_provider"): + value = _safe_source_text(source.get(key)) + if value: + return canonical_observation_source_code(value) + return canonical_observation_source_code(fallback) + + +def _source_label(source: dict[str, Any], code: str, fallback: str = "") -> str: + for key in ("source_label", "station_label", "station_name", "label", "provider_label"): + value = _safe_source_text(source.get(key)) + if value: + return value + return fallback or code.upper() + + +def _source_health_entry( + *, + city: str, + role: str, + source: dict[str, Any], + fallback_code: str = "", + fallback_label: str = "", +) -> dict[str, Any] | None: + if not isinstance(source, dict) or not source: + return None + + code = _source_code(source, fallback_code) + label = _source_label(source, code, fallback_label) + observed_at = _source_observed_at(source) + freshness = source.get("freshness") if isinstance(source.get("freshness"), dict) else None + if freshness: + status = _safe_source_text(freshness.get("freshness_status")) or "unknown" + age_sec = freshness.get("age_sec") + observed_at = freshness.get("observed_at") or freshness.get("observed_at_local") or observed_at + expected_next = freshness.get("expected_next_update_at") + reason = freshness.get("freshness_reason") + else: + age_min = source.get("obs_age_min") + try: + age_min_int = int(age_min) if age_min is not None else None + except Exception: + age_min_int = None + freshness = build_observation_freshness( + source_code=code, + source_label=label, + observed_at=observed_at, + age_min=age_min_int, + ) + status = str(freshness.get("freshness_status") or "unknown") + age_sec = freshness.get("age_sec") + expected_next = freshness.get("expected_next_update_at") + reason = freshness.get("freshness_reason") + + return { + "city": city, + "role": role, + "source_code": code, + "source_label": label, + "station_code": source.get("station_code") or source.get("icao"), + "station_label": source.get("station_label") or source.get("station_name"), + "status": status, + "reason": reason, + "age_sec": age_sec, + "age_min": round(float(age_sec) / 60, 1) if isinstance(age_sec, (int, float)) else None, + "observed_at": observed_at, + "expected_next_update_at": expected_next, + "temp": source.get("temp"), + } + + +def _expected_city_source_codes(city: str, payload: dict[str, Any]) -> list[str]: + city_key = str(city or "").strip().lower().replace(" ", "") + expected: list[str] = [] + if city_key in {"ankara", "istanbul"}: + expected.append("mgm") + if city_key == "amsterdam": + expected.append("knmi") + if city_key in {"telaviv", "telavivyafo"}: + expected.append("ims") + provider = canonical_observation_source_code(payload.get("official_network_source")) + if provider and provider not in {"metar", "none"}: + expected.append(provider) + return list(dict.fromkeys(expected)) + + +def _collect_city_source_health(city: str, entry: dict[str, Any] | None) -> dict[str, Any]: + payload = (entry or {}).get("payload") if isinstance(entry, dict) else {} + if not isinstance(payload, dict): + payload = {} + + sources: list[dict[str, Any]] = [] + seen: set[tuple[str, str]] = set() + + def add(role: str, source: Any, fallback_code: str = "", fallback_label: str = "") -> None: + item = _source_health_entry( + city=city, + role=role, + source=source if isinstance(source, dict) else {}, + fallback_code=fallback_code, + fallback_label=fallback_label, + ) + if not item: + return + key = (str(item.get("role")), str(item.get("source_code"))) + if key in seen: + return + seen.add(key) + sources.append(item) + + add("settlement", payload.get("current"), fallback_label="Settlement") + add("airport_metar", payload.get("airport_current"), fallback_code="metar", fallback_label="METAR") + add("airport_primary", payload.get("airport_primary"), fallback_label="Airport station") + + official = payload.get("official") if isinstance(payload.get("official"), dict) else {} + add("official_airport_primary", official.get("airport_primary"), fallback_label="Official airport station") + add("official_airport_primary", official.get("airport_primary_current"), fallback_label="Official airport station") + + mgm = payload.get("mgm") if isinstance(payload.get("mgm"), dict) else {} + if mgm: + mgm_current = mgm.get("current") if isinstance(mgm.get("current"), dict) else {} + add( + "official_network", + { + **mgm_current, + "source_code": "mgm", + "source_label": "MGM", + "obs_time": mgm.get("obs_time") or mgm_current.get("time"), + }, + fallback_code="mgm", + fallback_label="MGM", + ) + + for nearby in payload.get("official_nearby") or payload.get("mgm_nearby") or []: + if isinstance(nearby, dict): + add("nearby_official", nearby, fallback_label="Nearby official") + + expected_codes = _expected_city_source_codes(city, payload) + present_codes = {str(item.get("source_code") or "") for item in sources} + for code in expected_codes: + if code and code not in present_codes: + sources.append( + { + "city": city, + "role": "expected_source", + "source_code": code, + "source_label": code.upper(), + "status": "missing", + "reason": "expected_source_not_present_in_cached_detail", + "age_sec": None, + "age_min": None, + "observed_at": None, + "expected_next_update_at": None, + "temp": None, + } + ) + + priority = {"stale": 4, "missing": 4, "delayed": 3, "unknown": 2, "expected_wait": 1, "fresh": 0} + worst = max(sources, key=lambda item: priority.get(str(item.get("status") or ""), 2), default=None) + full_age_sec = ( + round(max(0.0, __import__("time").time() - float(entry.get("updated_at_ts") or 0.0)), 1) + if entry + else None + ) + return { + "city": city, + "cache_exists": bool(entry), + "cache_updated_at": entry.get("updated_at") if entry else None, + "cache_age_sec": full_age_sec, + "source_count": len(sources), + "worst_status": str(worst.get("status") if worst else "missing"), + "sources": sources, + } + + +# ── Ops endpoint functions ──────────────────────────────────────────── + +def get_ops_source_health( + request: Request, + cities: str = "", + limit: int = 80, +) -> dict[str, Any]: + from web.services.ops_api import _require_ops + + _require_ops(request) + requested = legacy_routes._normalize_city_list(cities) if cities else [] + if requested: + selected = requested + else: + all_cities = getattr(legacy_routes, "CITIES", {}) + selected = list(all_cities.keys()) if isinstance(all_cities, dict) else [] + safe_limit = max(1, min(int(limit or 80), 200)) + rows = [] + for city in selected[:safe_limit]: + entry = legacy_routes._CACHE_DB.get_city_cache("full", city) + if not entry: + entry = legacy_routes._CACHE_DB.get_city_cache("panel", city) + rows.append(_collect_city_source_health(city, entry)) + + status_counts: dict[str, int] = {} + for row in rows: + for source in row.get("sources") or []: + status = str(source.get("status") or "unknown") + status_counts[status] = status_counts.get(status, 0) + 1 + + return { + "checked_at": __import__("datetime").datetime.utcnow().isoformat() + "Z", + "cities": rows, + "status_counts": status_counts, + "total_cities": len(rows), + } + + +def get_ops_observation_collector_status( + request: Request, + limit: int = 200, +) -> dict[str, Any]: + from web.services.ops_api import _require_ops, ObservationCollectorStatusRepository + + _require_ops(request) + safe_limit = max(1, min(int(limit or 200), 500)) + return ObservationCollectorStatusRepository().load_snapshot(limit=safe_limit) + + +def get_ops_health_check(request: Request) -> dict[str, Any]: + from web.services.ops_api import _require_ops + + _require_ops(request) + import os as _os + import requests as _r + import time as _time + import urllib3 as _urllib3 + + _urllib3.disable_warnings(_urllib3.exceptions.InsecureRequestWarning) + + results: dict[str, dict] = {} + timeout = 8 + + # Supabase + supabase_url = str(_os.getenv("SUPABASE_URL") or "").strip().rstrip("/") + supabase_key = str(_os.getenv("SUPABASE_SERVICE_ROLE_KEY") or "").strip() + if supabase_url and supabase_key: + try: + r = _r.get( + f"{supabase_url}/rest/v1/", + headers={ + "apikey": supabase_key, + "Authorization": f"Bearer {supabase_key}", + }, + timeout=timeout, + ) + results["supabase"] = { + "ok": r.ok, + "status": r.status_code, + "latency_ms": round(r.elapsed.total_seconds() * 1000), + } + except Exception as e: + results["supabase"] = {"ok": False, "error": str(e)[:100]} + else: + results["supabase"] = {"ok": False, "error": "not configured"} + + # Open-Meteo + try: + t0 = _time.perf_counter() + r = _r.get( + "https://api.open-meteo.com/v1/forecast?latitude=52.52&longitude=13.41&daily=temperature_2m_max&timezone=auto&forecast_days=1", + timeout=timeout, + ) + results["open_meteo"] = { + "ok": r.ok, + "status": r.status_code, + "latency_ms": round((_time.perf_counter() - t0) * 1000), + } + except Exception as e: + results["open_meteo"] = {"ok": False, "error": str(e)[:100]} + + # METAR (aviationweather) + try: + t0 = _time.perf_counter() + r = _r.get( + "https://aviationweather.gov/api/data/metar?ids=KJFK&format=json", + timeout=timeout, + ) + results["metar"] = { + "ok": r.ok, + "status": r.status_code, + "latency_ms": round((_time.perf_counter() - t0) * 1000), + } + except Exception as e: + results["metar"] = {"ok": False, "error": str(e)[:100]} + + # KNMI + knmi_key = str(_os.getenv("KNMI_API_KEY") or "").strip() + if knmi_key: + try: + t0 = _time.perf_counter() + r = _r.get( + "https://api.dataplatform.knmi.nl/open-data/v1/datasets/10-minute-in-situ-meteorological-observations/versions/1.0/files?maxKeys=1", + headers={"Authorization": knmi_key}, + timeout=timeout, + ) + results["knmi"] = { + "ok": r.ok, + "status": r.status_code, + "latency_ms": round((_time.perf_counter() - t0) * 1000), + } + except Exception as e: + results["knmi"] = {"ok": False, "error": str(e)[:100]} + else: + results["knmi"] = {"ok": False, "error": "not configured"} + + # MADIS (NOAA) + try: + t0 = _time.perf_counter() + r = _r.head( + "https://madis-data.ncep.noaa.gov/madisPublic1/data/LDAD/hfmetar/netCDF/", + timeout=timeout, + allow_redirects=True, + ) + results["madis"] = { + "ok": r.ok, + "status": r.status_code, + "latency_ms": round((_time.perf_counter() - t0) * 1000), + } + except Exception as e: + results["madis"] = {"ok": False, "error": str(e)[:100]} + + # Telegram Bot + bot_token = str(_os.getenv("TELEGRAM_BOT_TOKEN") or "").strip() + if bot_token: + try: + t0 = _time.perf_counter() + r = _r.get( + f"https://api.telegram.org/bot{bot_token}/getMe", timeout=timeout + ) + results["telegram"] = { + "ok": r.ok, + "status": r.status_code, + "latency_ms": round((_time.perf_counter() - t0) * 1000), + } + except Exception as e: + results["telegram"] = {"ok": False, "error": str(e)[:100]} + else: + results["telegram"] = {"ok": False, "error": "not configured"} + + # JMA (Japan Meteorological Agency) + try: + t0 = _time.perf_counter() + r = _r.get("https://www.jma.go.jp/bosai/forecast/", timeout=timeout) + results["jma"] = { + "ok": r.ok, + "status": r.status_code, + "latency_ms": round((_time.perf_counter() - t0) * 1000), + } + except Exception as e: + results["jma"] = {"ok": False, "error": str(e)[:100]} + + # MGM (Turkish State Meteorological Service) + try: + t0 = _time.perf_counter() + r = _r.get( + "https://servis.mgm.gov.tr/web/sondurumlar?istno=17130&_=1", + timeout=timeout, + headers={"Origin": "https://www.mgm.gov.tr"}, + ) + results["mgm"] = { + "ok": r.ok, + "status": r.status_code, + "latency_ms": round((_time.perf_counter() - t0) * 1000), + } + except Exception as e: + results["mgm"] = {"ok": False, "error": str(e)[:100]} + + # FMI (Finnish Meteorological Institute) + try: + t0 = _time.perf_counter() + r = _r.get( + "https://opendata.fmi.fi/wfs?service=WFS&version=2.0.0&request=GetCapabilities", + timeout=timeout, + ) + results["fmi"] = { + "ok": r.ok, + "status": r.status_code, + "latency_ms": round((_time.perf_counter() - t0) * 1000), + } + except Exception as e: + results["fmi"] = {"ok": False, "error": str(e)[:100]} + + # KMA (Korea Meteorological Administration) + try: + t0 = _time.perf_counter() + r = _r.get( + "https://www.weather.go.kr/wgis-nuri/js/info/sfc.geojson", timeout=timeout + ) + results["kma"] = { + "ok": r.ok, + "status": r.status_code, + "latency_ms": round((_time.perf_counter() - t0) * 1000), + } + except Exception as e: + results["kma"] = {"ok": False, "error": str(e)[:100]} + + # HKO (Hong Kong Observatory) + try: + t0 = _time.perf_counter() + r = _r.get( + "https://data.weather.gov.hk/weatherAPI/hko_data/regional-weather/latest_1min_temperature.csv", + timeout=timeout, + ) + results["hko"] = { + "ok": r.ok, + "status": r.status_code, + "latency_ms": round((_time.perf_counter() - t0) * 1000), + } + except Exception as e: + results["hko"] = {"ok": False, "error": str(e)[:100]} + + # Singapore MSS (data.gov.sg) + try: + t0 = _time.perf_counter() + r = _r.get( + "https://api.data.gov.sg/v1/environment/air-temperature", timeout=timeout + ) + results["singapore_mss"] = { + "ok": r.ok, + "status": r.status_code, + "latency_ms": round((_time.perf_counter() - t0) * 1000), + } + except Exception as e: + results["singapore_mss"] = {"ok": False, "error": str(e)[:100]} + + # CWA (Taiwan Central Weather Administration) + cwa_key = str( + _os.getenv("CWA_API_KEY") + or _os.getenv("CWA_OPEN_DATA_AUTH") + or _os.getenv("CWA_OPEN_DATA_API_KEY") + or "" + ).strip() + if cwa_key: + try: + t0 = _time.perf_counter() + r = _r.get( + f"https://opendata.cwa.gov.tw/api/v1/rest/datastore/O-A0003-001?Authorization={cwa_key}&limit=1", + timeout=timeout, + ) + results["cwa"] = { + "ok": r.ok, + "status": r.status_code, + "latency_ms": round((_time.perf_counter() - t0) * 1000), + } + except Exception as e: + results["cwa"] = {"ok": False, "error": str(e)[:100]} + else: + results["cwa"] = {"ok": False, "error": "not configured"} + + # AMOS (Korea runway sensors) + try: + t0 = _time.perf_counter() + r = _r.get( + "https://global.amo.go.kr/amosobsnew/AmosRealTimeImage.do", timeout=timeout + ) + results["amos"] = { + "ok": r.ok, + "status": r.status_code, + "latency_ms": round((_time.perf_counter() - t0) * 1000), + } + except Exception as e: + results["amos"] = {"ok": False, "error": str(e)[:100]} + + # AMSC AWOS (China mainland airports) + results["amsc_awos"] = _check_amsc_awos_health(timeout=timeout) + + # NOAA WRH (US settlement verification) + try: + t0 = _time.perf_counter() + r = _r.get("https://www.weather.gov/wrh/timeseries?site=KJFK", timeout=timeout) + results["noaa_wrh"] = { + "ok": r.ok, + "status": r.status_code, + "latency_ms": round((_time.perf_counter() - t0) * 1000), + } + except Exception as e: + results["noaa_wrh"] = {"ok": False, "error": str(e)[:100]} + + all_ok = all(v.get("ok") for v in results.values()) + return { + "ok": all_ok, + "checked_at": __import__("datetime").datetime.utcnow().isoformat() + "Z", + "services": results, + } + + +# ── Training accuracy helpers ───────────────────────────────────────── + +def _evaluate_deb_records( + records: List[Dict[str, Any]], + *, + start_date: Optional[str] = None, + end_date: Optional[str] = None, +) -> Dict[str, Any]: + from src.analysis.settlement_rounding import apply_city_settlement + + hits = 0 + total = 0 + errors: List[float] = [] + signed_errors: List[float] = [] + cities = set() + dates = set() + for row in records: + target_date = str(row.get("target_date") or "").strip() + if start_date and target_date < start_date: + continue + if end_date and target_date > end_date: + continue + city = str(row.get("city") or "").strip().lower() + prediction = _sf(row.get("deb_prediction")) + actual = _sf(row.get("actual_high")) + if not city or not target_date or prediction is None or actual is None: + continue + try: + pred_bucket = apply_city_settlement(city, prediction) + actual_bucket = apply_city_settlement(city, actual) + except Exception: + continue + if pred_bucket is None or actual_bucket is None: + continue + total += 1 + if pred_bucket == actual_bucket: + hits += 1 + errors.append(abs(prediction - actual)) + signed_errors.append(prediction - actual) + cities.add(city) + dates.add(target_date) + + return { + "start_date": start_date, + "end_date": end_date, + "samples": total, + "hits": hits, + "hit_rate": _round_metric((hits / total * 100) if total else None, 1), + "mae": _round_metric((sum(errors) / len(errors)) if errors else None, 2), + "bias": _round_metric((sum(signed_errors) / len(signed_errors)) if signed_errors else None, 2), + "city_count": len(cities), + "date_count": len(dates), + } + + +def _build_city_deb_rows( + city_id: str, + city_rows: Dict[str, Dict[str, Any]], + today_str: str, +) -> List[Dict[str, Any]]: + rows: List[Dict[str, Any]] = [] + for target_date, record in sorted((city_rows or {}).items()): + if target_date >= today_str or not isinstance(record, dict): + continue + rows.append( + { + "city": city_id, + "target_date": target_date, + "actual_high": record.get("actual_high"), + "deb_prediction": record.get("deb_prediction"), + } + ) + return rows + + +def _deb_recent_bias_direction(bias: Optional[float]) -> str: + if bias is None: + return "unknown" + if bias <= -0.5: + return "under" + if bias >= 0.5: + return "over" + return "neutral" + + +def _build_deb_recent_trust_strategy( + recent_7d: Dict[str, Any], + recent_14d: Dict[str, Any], +) -> Dict[str, Any]: + window_key = "recent_14d" if int(recent_14d.get("samples") or 0) >= int(recent_7d.get("samples") or 0) else "recent_7d" + metrics = recent_14d if window_key == "recent_14d" else recent_7d + samples = int(metrics.get("samples") or 0) + hit_rate = _sf(metrics.get("hit_rate")) + mae = _sf(metrics.get("mae")) + bias = _sf(metrics.get("bias")) + + if samples < 3 or hit_rate is None or mae is None: + trust_tier = "insufficient" + recommendation = "insufficient" + reason = f"{window_key}: only {samples} settled DEB samples." + elif hit_rate >= 67.0 and mae <= 1.25: + trust_tier = "high" + recommendation = "primary" + reason = f"{window_key}: {hit_rate:.1f}% hit rate, MAE {mae:.2f}°." + elif hit_rate >= 34.0 and mae <= 1.75: + trust_tier = "medium" + recommendation = "supporting" + reason = f"{window_key}: {hit_rate:.1f}% hit rate, MAE {mae:.2f}°." + else: + trust_tier = "low" + recommendation = "context_only" + reason = f"{window_key}: {hit_rate:.1f}% hit rate, MAE {mae:.2f}°." + + return { + "trust_tier": trust_tier, + "recommendation": recommendation, + "bias_direction": _deb_recent_bias_direction(bias), + "reason": reason, + } + + +def _build_city_deb_recent_strategy( + city_id: str, + city_rows: Dict[str, Dict[str, Any]], + today_str: str, +) -> Optional[Dict[str, Any]]: + today_date = datetime.strptime(today_str, "%Y-%m-%d").date() + recent_7_start = (today_date - timedelta(days=7)).isoformat() + recent_14_start = (today_date - timedelta(days=14)).isoformat() + end_date = (today_date - timedelta(days=1)).isoformat() + rows = _build_city_deb_rows(city_id, city_rows, today_str) + if not rows: + return None + recent_7d = _evaluate_deb_records(rows, start_date=recent_7_start, end_date=end_date) + recent_14d = _evaluate_deb_records(rows, start_date=recent_14_start, end_date=end_date) + return { + "recent_7d": recent_7d, + "recent_14d": recent_14d, + **_build_deb_recent_trust_strategy(recent_7d, recent_14d), + } + + +def _build_city_deb_accuracy(city_id: str, city_rows: Dict[str, Dict[str, Any]], today_str: str) -> Optional[Dict[str, Any]]: + rows = _build_city_deb_rows(city_id, city_rows, today_str) + metrics = _evaluate_deb_records(rows) + if not metrics["samples"]: + return None + total = int(metrics["samples"]) + hits = int(metrics["hits"]) + mae = float(metrics["mae"] or 0.0) + hit_rate = float(metrics["hit_rate"] or 0.0) + return { + "hit_rate": hit_rate, + "mae": mae, + "total_days": total, + "hits": hits, + "details_str": f"过去{total}天 WU命中 {hits}/{total} ({hit_rate:.0f}%) | MAE: {mae:.1f}°", + } + + +def _build_city_mu_accuracy(city_id: str, city_rows: Dict[str, Dict[str, Any]], today_str: str) -> Optional[Dict[str, Any]]: + from src.analysis.settlement_rounding import apply_city_settlement + + errors: List[float] = [] + hits = 0 + total = 0 + brier_scores: List[float] = [] + for target_date, record in sorted((city_rows or {}).items()): + if target_date >= today_str or not isinstance(record, dict): + continue + actual = _sf(record.get("actual_high")) + mu_value = _sf(record.get("mu")) + if actual is None or mu_value is None: + continue + total += 1 + errors.append(abs(mu_value - actual)) + if apply_city_settlement(city_id, mu_value) == apply_city_settlement(city_id, actual): + hits += 1 + prob_snapshot = record.get("prob_snapshot") or [] + if isinstance(prob_snapshot, list): + actual_bucket = apply_city_settlement(city_id, actual) + score = 0.0 + used = False + for entry in prob_snapshot: + if not isinstance(entry, dict): + continue + predicted_p = _sf(entry.get("p")) or 0.0 + outcome = 1.0 if entry.get("v") == actual_bucket else 0.0 + score += (predicted_p - outcome) ** 2 + used = True + if used: + brier_scores.append(score) + + if not total: + return None + mae = sum(errors) / len(errors) + hit_rate = hits / total * 100 + brier = (sum(brier_scores) / len(brier_scores)) if brier_scores else None + details_parts = [ + f"μ准确率: 过去{total}天", + f"WU命中 {hits}/{total} ({hit_rate:.0f}%)", + f"MAE: {mae:.1f}°", + ] + if brier is not None: + details_parts.append(f"Brier: {brier:.3f}") + return { + "mae": mae, + "hit_rate": hit_rate, + "brier_score": brier, + "total_days": total, + "hits": hits, + "details_str": " | ".join(details_parts), + } + + +def _build_deb_historical_summary(accuracy_data: List[Dict[str, Any]]) -> Dict[str, Any]: + deb_rows = [row for row in accuracy_data if row.get("deb")] + if not deb_rows: + return { + "city_count": 0, + "avg_hit_rate": None, + "weighted_hit_rate": None, + "avg_mae": None, + "avg_days_per_city": 0, + "sample_days": 0, + "hits": 0, + } + sample_days = sum(int(row["deb"].get("total_days") or 0) for row in deb_rows) + hits = sum(int(row["deb"].get("hits") or 0) for row in deb_rows) + return { + "city_count": len(deb_rows), + "avg_hit_rate": _round_metric( + sum(float(row["deb"].get("hit_rate") or 0.0) for row in deb_rows) / len(deb_rows), + 1, + ), + "weighted_hit_rate": _round_metric((hits / sample_days * 100) if sample_days else None, 1), + "avg_mae": _round_metric( + sum(float(row["deb"].get("mae") or 0.0) for row in deb_rows) / len(deb_rows), + 2, + ), + "avg_days_per_city": round(sample_days / len(deb_rows)) if deb_rows else 0, + "sample_days": sample_days, + "hits": hits, + } + + +def _build_deb_usable_recent_summary( + accuracy_data: List[Dict[str, Any]], +) -> Dict[str, Any]: + usable_rows = [] + recommendations = {"primary": 0, "supporting": 0} + for row in accuracy_data: + recent = row.get("deb_recent") if isinstance(row.get("deb_recent"), dict) else None + if not recent: + continue + recommendation = str(recent.get("recommendation") or "").strip().lower() + if recommendation not in {"primary", "supporting"}: + continue + usable_rows.append(row) + recommendations[recommendation] += 1 + + def build_window(window_key: str) -> Dict[str, Any]: + samples = 0 + hits = 0 + weighted_mae = 0.0 + city_count = 0 + for row in usable_rows: + recent = row.get("deb_recent") if isinstance(row.get("deb_recent"), dict) else {} + metrics = recent.get(window_key) if isinstance(recent.get(window_key), dict) else {} + row_samples = int(metrics.get("samples") or 0) + if row_samples <= 0: + continue + row_hits = int(metrics.get("hits") or 0) + row_mae = _sf(metrics.get("mae")) + samples += row_samples + hits += row_hits + city_count += 1 + if row_mae is not None: + weighted_mae += row_mae * row_samples + return { + "window": window_key, + "city_count": city_count, + "samples": samples, + "hits": hits, + "hit_rate": _round_metric((hits / samples * 100) if samples else None, 1), + "avg_mae": _round_metric((weighted_mae / samples) if samples else None, 2), + "recommendations": dict(recommendations), + } + + recent_7d = build_window("recent_7d") + if int(recent_7d.get("samples") or 0) > 0: + return recent_7d + return build_window("recent_14d") + + +def _flatten_training_history(history: Dict[str, Dict[str, Dict[str, Any]]], today_str: str) -> List[Dict[str, Any]]: + rows: List[Dict[str, Any]] = [] + for city, city_rows in (history or {}).items(): + if not isinstance(city_rows, dict): + continue + for target_date, record in city_rows.items(): + if not isinstance(record, dict): + continue + target_text = str(target_date or "").strip() + if not target_text or target_text >= today_str: + continue + rows.append( + { + "city": str(city or "").strip().lower(), + "target_date": target_text, + "actual_high": record.get("actual_high"), + "deb_prediction": record.get("deb_prediction"), + "mu": record.get("mu"), + "prob_snapshot": record.get("prob_snapshot"), + } + ) + rows.sort(key=lambda row: (row["target_date"], row["city"])) + return rows + + +def _build_training_accuracy_payload( + history: Dict[str, Dict[str, Dict[str, Any]]], + city_registry: Dict[str, Dict[str, Any]], + *, + today_str: Optional[str] = None, +) -> Dict[str, Any]: + from src.analysis.deb_evaluation import backtest_deb_versions + + today = today_str or datetime.now(timezone.utc).strftime("%Y-%m-%d") + accuracy_data: List[Dict[str, Any]] = [] + for city_id, info in (city_registry or {}).items(): + city_rows = history.get(city_id) or history.get(str(city_id).strip().lower()) or {} + deb_payload = _build_city_deb_accuracy(city_id, city_rows, today) + deb_recent = _build_city_deb_recent_strategy(city_id, city_rows, today) if deb_payload else None + mu_payload = _build_city_mu_accuracy(city_id, city_rows, today) + if deb_payload or mu_payload: + accuracy_data.append( + { + "city_id": city_id, + "name": (info or {}).get("name") or city_id, + "deb": deb_payload, + "deb_recent": deb_recent, + "mu": mu_payload, + } + ) + + accuracy_data.sort( + key=lambda row: max( + row["deb"]["total_days"] if row.get("deb") else 0, + row["mu"]["total_days"] if row.get("mu") else 0, + ), + reverse=True, + ) + + all_rows = _flatten_training_history(history, today) + today_date = datetime.strptime(today, "%Y-%m-%d").date() + recent_7_start = (today_date - timedelta(days=7)).isoformat() + recent_14_start = (today_date - timedelta(days=14)).isoformat() + end_date = (today_date - timedelta(days=1)).isoformat() + version_rows = all_rows[-_DEB_VERSION_BACKTEST_SAMPLE_LIMIT:] + versions = backtest_deb_versions( + version_rows, + min_train_samples=2, + ).get("versions", {}) + + return { + "accuracy": accuracy_data, + "deb_summary": { + "historical": _build_deb_historical_summary(accuracy_data), + "usable_recent": _build_deb_usable_recent_summary(accuracy_data), + "recent_7d": _evaluate_deb_records( + all_rows, + start_date=recent_7_start, + end_date=end_date, + ), + "recent_14d": _evaluate_deb_records( + all_rows, + start_date=recent_14_start, + end_date=end_date, + ), + "versions": versions, + }, + } + + +def get_ops_training_accuracy(request: Request) -> Dict[str, Any]: + from src.analysis.deb_algorithm import load_history + from src.data_collection.city_registry import CITY_REGISTRY + + history_file = os.path.join( + os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))), + "data", + "daily_records.json", + ) + history = load_history(history_file) + return _build_training_accuracy_payload(history, CITY_REGISTRY) + + +# ── Truth history ───────────────────────────────────────────────────── + +def get_ops_truth_history( + request: Request, + city: str = "", + date_from: str = "", + date_to: str = "", + limit: int = 200, +) -> Dict[str, Any]: + from web.services.ops_api import _require_ops + + _require_ops(request) + + truth_history = legacy_routes.TruthRecordRepository().load_all() + normalized_city = str(city or "").strip().lower() + normalized_from = str(date_from or "").strip() + normalized_to = str(date_to or "").strip() + max_limit = max(1, min(int(limit or 200), 1000)) + + rows = [] + for row_city, by_date in truth_history.items(): + if normalized_city and row_city != normalized_city: + continue + if not isinstance(by_date, dict): + continue + for target_date, payload in by_date.items(): + if normalized_from and str(target_date) < normalized_from: + continue + if normalized_to and str(target_date) > normalized_to: + continue + if not isinstance(payload, dict): + continue + rows.append( + { + "city": row_city, + "display_name": str( + (legacy_routes.CITY_REGISTRY.get(row_city) or {}).get("name") + or row_city + ), + "target_date": str(target_date), + "actual_high": payload.get("actual_high"), + "settlement_source": payload.get("settlement_source"), + "settlement_station_code": payload.get("settlement_station_code"), + "settlement_station_label": payload.get("settlement_station_label"), + "truth_version": payload.get("truth_version"), + "updated_by": payload.get("updated_by"), + "truth_updated_at": payload.get("truth_updated_at"), + "is_final": payload.get("is_final"), + } + ) + + rows.sort( + key=lambda item: (str(item["target_date"]), str(item["city"])), reverse=True + ) + filtered_count = len(rows) + rows = rows[:max_limit] + available_cities = [ + { + "city": city_id, + "name": str(info.get("name") or city_id), + } + for city_id, info in sorted( + legacy_routes.CITY_REGISTRY.items(), + key=lambda item: str(item[1].get("name") or item[0]), + ) + ] + return { + "items": rows, + "available_cities": available_cities, + "filters": { + "city": normalized_city or None, + "date_from": normalized_from or None, + "date_to": normalized_to or None, + "limit": max_limit, + }, + "filtered_count": filtered_count, + } diff --git a/web/services/ops/payments.py b/web/services/ops/payments.py new file mode 100644 index 00000000..c6746fd1 --- /dev/null +++ b/web/services/ops/payments.py @@ -0,0 +1,1025 @@ +"""Ops payments / billing / membership service functions.""" +from __future__ import annotations + +import os +from collections import defaultdict +from datetime import datetime, timedelta, timezone +from typing import Any, Dict, List, Optional, Tuple + +from fastapi import HTTPException, Request +import requests as _requests + +from src.database.db_manager import DBManager # type hints +import web.routes as legacy_routes + + +def _get_db(): + from src.database.db_manager import DBManager as _DBManager + return _DBManager() + + +# ═══════════════════════════════════════════════════════════════════════ +# Shared helpers +# ═══════════════════════════════════════════════════════════════════════ + +def _require_ops(request: Request) -> Dict[str, Any] | None: + from web.services.ops_api import _require_ops as _real + return _real(request) + + +def _parse_iso_datetime(value: Any) -> Optional[datetime]: + text = str(value or "").strip() + if not text: + return None + try: + parsed = datetime.fromisoformat(text.replace("Z", "+00:00")) + except Exception: + return None + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=timezone.utc) + return parsed.astimezone(timezone.utc) + + +def _to_utc_iso(value: datetime) -> str: + if value.tzinfo is None: + value = value.replace(tzinfo=timezone.utc) + return value.astimezone(timezone.utc).isoformat().replace("+00:00", "Z") + + +def _supabase_rest_rows( + table: str, + params: Dict[str, Any], + *, + timeout: int = 10, +) -> List[Dict[str, Any]]: + from web.services.ops_api import _supabase_rest_rows as _real + + return _real(table, params, timeout=timeout) + + +def _app_analytics_actor_key(row: Dict[str, Any]) -> str: + payload = row.get("payload") + payload = payload if isinstance(payload, dict) else {} + user_id = str(row.get("user_id") or payload.get("user_id") or "").strip().lower() + client_id = str(row.get("client_id") or "").strip() + session_id = str(row.get("session_id") or "").strip() + if user_id: + return f"user:{user_id}" + if client_id: + return f"client:{client_id}" + if session_id: + return f"session:{session_id}" + return f"event:{row.get('id')}" + + +# ═══════════════════════════════════════════════════════════════════════ +# Payment helpers +# ═══════════════════════════════════════════════════════════════════════ + +def _payment_explorer_url(chain: Any, tx_hash: Any) -> str: + tx = str(tx_hash or "").strip() + if not tx: + return "" + chain_text = str(chain or "").strip().lower() + base = "https://etherscan.io" if "eth" in chain_text else "https://polygonscan.com" + return f"{base}/tx/{tx}" + + +def _risk_issue( + *, + category: str, + severity: str, + title: str, + detail: str, + user_id: Any = "", + created_at: Any = "", + reference: Any = "", + payload: Optional[Dict[str, Any]] = None, +) -> Dict[str, Any]: + return { + "category": category, + "severity": severity, + "title": title, + "detail": detail, + "user_id": str(user_id or ""), + "created_at": str(created_at or ""), + "reference": str(reference or ""), + "payload": payload or {}, + } + + +def _normalize_payment_incident(item: Dict[str, Any]) -> Dict[str, Any]: + payload = item.get("payload") if isinstance(item, dict) else {} + payload = payload if isinstance(payload, dict) else {} + confirm_failure = ( + payload.get("confirm_failure") + if isinstance(payload.get("confirm_failure"), dict) + else {} + ) + reason = str( + payload.get("reason") + or confirm_failure.get("reason") + or payload.get("error") + or "unknown" + ).strip().lower() + detail = str( + payload.get("detail") + or confirm_failure.get("detail") + or payload.get("message") + or payload.get("error") + or "" + ).strip() + resolved_at = str(payload.get("resolved_at") or "").strip() + return { + **item, + "payload": payload, + "reason": reason or "unknown", + "detail": detail, + "intent_id": str( + payload.get("intent_id") + or payload.get("payment_intent_id") + or confirm_failure.get("intent_id") + or "" + ).strip(), + "user_id": str(payload.get("user_id") or "").strip(), + "tx_hash": str( + payload.get("tx_hash") + or confirm_failure.get("tx_hash") + or "" + ).strip(), + "resolved": bool(resolved_at), + "resolved_at": resolved_at, + "resolved_by": str(payload.get("resolved_by") or "").strip(), + } + + +def _payment_incident_group_key(item: Dict[str, Any]) -> Tuple[str, str, str, str]: + reason = str(item.get("reason") or "unknown").strip().lower() + intent_id = str(item.get("intent_id") or "").strip().lower() + tx_hash = str(item.get("tx_hash") or "").strip().lower() + user_id = str(item.get("user_id") or "").strip().lower() + if intent_id or tx_hash: + return reason, user_id, intent_id, tx_hash + return reason, user_id, f"event:{item.get('id')}", "" + + +def _group_payment_incidents( + incidents: List[Dict[str, Any]], + *, + reason: str = "", + include_resolved: bool = False, +) -> Dict[str, Any]: + normalized_reason = str(reason or "").strip().lower() + groups: Dict[Tuple[str, str, str, str], Dict[str, Any]] = {} + raw_total = 0 + + for item in incidents: + normalized_item = _normalize_payment_incident(item) + item_reason = str(normalized_item.get("reason") or "").strip().lower() + resolved = bool(normalized_item.get("resolved")) + if normalized_reason and item_reason != normalized_reason: + continue + if not include_resolved and resolved: + continue + raw_total += 1 + + key = _payment_incident_group_key(normalized_item) + created_at = str(normalized_item.get("created_at") or "").strip() + event_id = int(normalized_item.get("id") or 0) + existing = groups.get(key) + if existing is None: + grouped = { + **normalized_item, + "occurrence_count": 1, + "event_ids": [event_id] if event_id > 0 else [], + "first_seen_at": created_at, + "last_seen_at": created_at, + } + groups[key] = grouped + continue + + existing["occurrence_count"] = int(existing.get("occurrence_count") or 1) + 1 + if event_id > 0: + existing.setdefault("event_ids", []).append(event_id) + first_seen = str(existing.get("first_seen_at") or "").strip() + last_seen = str(existing.get("last_seen_at") or "").strip() + if created_at and (not first_seen or created_at < first_seen): + existing["first_seen_at"] = created_at + if created_at and (not last_seen or created_at > last_seen): + existing["last_seen_at"] = created_at + + grouped_items = list(groups.values()) + grouped_items.sort( + key=lambda item: ( + str(item.get("last_seen_at") or item.get("created_at") or ""), + int(item.get("id") or 0), + ), + reverse=True, + ) + return { + "incidents": grouped_items, + "raw_total": raw_total, + "total": len(grouped_items), + } + + +# ═══════════════════════════════════════════════════════════════════════ +# Payment incident API +# ═══════════════════════════════════════════════════════════════════════ + +def list_ops_payment_incidents( + request: Request, + limit: int = 50, + reason: str = "", + include_resolved: bool = False, +) -> Dict[str, Any]: + _require_ops(request) + db = _get_db() + safe_limit = max(1, min(int(limit or 50), 200)) + incidents = db.list_payment_audit_events( + limit=max(safe_limit, 500), + event_type="payment_intent_failed", + ) + grouped = _group_payment_incidents( + incidents, + reason=reason, + include_resolved=include_resolved, + ) + return { + **grouped, + "incidents": grouped["incidents"][:safe_limit], + } + + +def resolve_ops_payment_incident(request: Request, event_id: int) -> Dict[str, Any]: + admin = _require_ops(request) or {} + db = _get_db() + resolved_group = db.mark_related_payment_audit_events_resolved( + event_id, str(admin.get("email") or "") + ) + if not resolved_group: + raise HTTPException(status_code=404, detail="payment_incident_not_found") + return { + "ok": True, + "incident": resolved_group[0], + "resolved_count": len(resolved_group), + "resolved_event_ids": [int(item.get("id") or 0) for item in resolved_group], + } + + +def list_ops_payments( + request: Request, + limit: int = 50, +) -> Dict[str, Any]: + """List successful payment records from Supabase.""" + _require_ops(request) + safe_limit = max(1, min(int(limit or 50), 200)) + rows = _supabase_rest_rows( + "payments", + { + "select": "id,user_id,amount,currency,chain,tx_hash,status,created_at", + "order": "created_at.desc", + "limit": str(safe_limit), + }, + ) + return {"payments": rows, "total": len(rows)} + + +# ═══════════════════════════════════════════════════════════════════════ +# Billing risk +# ═══════════════════════════════════════════════════════════════════════ + +def get_ops_billing_risk( + request: Request, + days: int = 30, + limit: int = 80, +) -> Dict[str, Any]: + """Summarize trial, payment, referral, and points risk signals for ops.""" + _require_ops(request) + db = _get_db() + now = datetime.now(timezone.utc) + safe_days = max(1, min(int(days or 30), 120)) + safe_limit = max(10, min(int(limit or 80), 200)) + since_dt = now - timedelta(days=safe_days) + month_start = now.replace(day=1, hour=0, minute=0, second=0, microsecond=0) + + query_errors: List[Dict[str, str]] = [] + + def collect(table: str, params: Dict[str, Any]) -> List[Dict[str, Any]]: + try: + return _supabase_rest_rows(table, params) + except Exception as exc: + query_errors.append({"table": table, "error": str(exc)[:180]}) + return [] + + intents = collect( + "payment_intents", + { + "select": ( + "id,user_id,plan_code,chain_id,status,expires_at,tx_hash," + "metadata,created_at,updated_at" + ), + "order": "updated_at.desc", + "limit": str(max(safe_limit * 3, 100)), + }, + ) + referral_attributions = collect( + "referral_attributions", + { + "select": ( + "id,referrer_user_id,referred_user_id,code,status," + "converted_payment_intent_id,converted_tx_hash,converted_at," + "created_at,updated_at" + ), + "order": "updated_at.desc", + "limit": str(safe_limit), + }, + ) + referral_rewards = collect( + "referral_rewards", + { + "select": ( + "id,referral_attribution_id,referrer_user_id,referred_user_id," + "payment_intent_id,tx_hash,reward_days,reward_points,created_at" + ), + "order": "created_at.desc", + "limit": str(safe_limit), + }, + ) + trial_claims = collect( + "trial_claims", + { + "select": "id,user_id,email,telegram_user_id,claimed_at,created_at", + "order": "created_at.desc", + "limit": str(max(safe_limit * 10, 500)), + }, + ) + trial_subscription_rows = collect( + "subscriptions", + { + "select": ( + "id,user_id,plan_code,source,status,starts_at,expires_at," + "created_at,updated_at" + ), + "or": "(source.eq.signup_trial,plan_code.eq.signup_trial_3d)", + "order": "created_at.desc", + "limit": str(max(safe_limit * 20, 1000)), + }, + ) + active_subscription_rows = collect( + "subscriptions", + { + "select": ( + "id,user_id,plan_code,source,status,starts_at,expires_at," + "created_at,updated_at" + ), + "status": "eq.active", + "order": "created_at.desc", + "limit": str(max(safe_limit * 20, 1000)), + }, + ) + subscription_rows: List[Dict[str, Any]] = [] + seen_subscription_keys: set[str] = set() + for row in [*trial_subscription_rows, *active_subscription_rows]: + key = str(row.get("id") or "").strip() or ( + f"{row.get('user_id')}:{row.get('plan_code')}:{row.get('source')}:" + f"{row.get('starts_at')}:{row.get('expires_at')}" + ) + if key in seen_subscription_keys: + continue + seen_subscription_keys.add(key) + subscription_rows.append(row) + entitlement_trial_events = collect( + "entitlement_events", + { + "select": "id,user_id,action,payload,created_at", + "action": "in.(signup_trial_claimed,signup_trial_granted)", + "order": "created_at.desc", + "limit": str(max(safe_limit * 10, 500)), + }, + ) + + issues: List[Dict[str, Any]] = [] + stuck_intents: List[Dict[str, Any]] = [] + points_issues: List[Dict[str, Any]] = [] + + for intent in intents: + status = str(intent.get("status") or "").strip().lower() + updated_at = _parse_iso_datetime(intent.get("updated_at")) + created_at = _parse_iso_datetime(intent.get("created_at")) + expires_at = _parse_iso_datetime(intent.get("expires_at")) + age_min = ( + int((now - (updated_at or created_at or now)).total_seconds() // 60) + if (updated_at or created_at) + else 0 + ) + intent_id = str(intent.get("id") or "") + user_id = str(intent.get("user_id") or "") + metadata = intent.get("metadata") + metadata = metadata if isinstance(metadata, dict) else {} + redemption = metadata.get("points_redemption") + redemption = redemption if isinstance(redemption, dict) else {} + + if status == "submitted" and age_min >= 10: + row = { + "id": intent_id, + "user_id": user_id, + "plan_code": intent.get("plan_code"), + "status": status, + "age_min": age_min, + "tx_hash": intent.get("tx_hash"), + "updated_at": intent.get("updated_at"), + } + stuck_intents.append(row) + issues.append( + _risk_issue( + category="payment_intent", + severity="high", + title="Submitted intent 超过 10 分钟未确认", + detail=f"{intent_id} 已提交 {age_min} 分钟，可能需要补单或检查确认循环。", + user_id=user_id, + created_at=intent.get("updated_at") or intent.get("created_at"), + reference=intent_id, + payload=row, + ) + ) + elif status == "created" and expires_at and expires_at < now: + row = { + "id": intent_id, + "user_id": user_id, + "plan_code": intent.get("plan_code"), + "status": status, + "expires_at": intent.get("expires_at"), + "age_min": age_min, + } + stuck_intents.append(row) + issues.append( + _risk_issue( + category="payment_intent", + severity="medium", + title="Created intent 已过期但未关闭", + detail=f"{intent_id} 已过期，用户可能离开支付流程。", + user_id=user_id, + created_at=intent.get("created_at"), + reference=intent_id, + payload=row, + ) + ) + + if bool(redemption.get("applied")): + planned = int(redemption.get("points_to_consume") or 0) + consumed = bool(redemption.get("consumed")) + consumed_points = int(redemption.get("consumed_points") or 0) + if status == "confirmed" and not consumed: + row = { + "intent_id": intent_id, + "user_id": user_id, + "status": status, + "planned_points": planned, + "consumed_points": consumed_points, + "updated_at": intent.get("updated_at"), + } + points_issues.append(row) + issues.append( + _risk_issue( + category="points_redemption", + severity="high", + title="订单已确认但积分未扣减", + detail=f"{intent_id} 标记使用积分，但 confirmed metadata 未显示 consumed。", + user_id=user_id, + created_at=intent.get("updated_at") or intent.get("created_at"), + reference=intent_id, + payload=row, + ) + ) + elif status == "confirmed" and planned > 0 and 0 < consumed_points < planned: + row = { + "intent_id": intent_id, + "user_id": user_id, + "status": status, + "planned_points": planned, + "consumed_points": consumed_points, + "updated_at": intent.get("updated_at"), + } + points_issues.append(row) + issues.append( + _risk_issue( + category="points_redemption", + severity="medium", + title="积分抵扣只扣了部分积分", + detail=f"{intent_id} 计划扣 {planned}，实际扣 {consumed_points}。", + user_id=user_id, + created_at=intent.get("updated_at") or intent.get("created_at"), + reference=intent_id, + payload=row, + ) + ) + + reward_by_attribution = { + str(row.get("referral_attribution_id") or ""): row + for row in referral_rewards + if row.get("referral_attribution_id") is not None + } + monthly_cap_hits: List[Dict[str, Any]] = [] + referral_settlement_issues: List[Dict[str, Any]] = [] + + for attribution in referral_attributions: + status = str(attribution.get("status") or "").strip().lower() + attribution_id = str(attribution.get("id") or "") + updated_at = _parse_iso_datetime( + attribution.get("updated_at") or attribution.get("converted_at") or attribution.get("created_at") + ) + if status == "capped" and (not updated_at or updated_at >= month_start): + row = { + "id": attribution_id, + "code": attribution.get("code"), + "referrer_user_id": attribution.get("referrer_user_id"), + "referred_user_id": attribution.get("referred_user_id"), + "updated_at": attribution.get("updated_at"), + } + monthly_cap_hits.append(row) + issues.append( + _risk_issue( + category="referral", + severity="medium", + title="邀请奖励月度上限命中", + detail=f"邀请码 {attribution.get('code') or ''} 的推荐奖励已被月度上限拦截。", + user_id=attribution.get("referrer_user_id"), + created_at=attribution.get("updated_at") or attribution.get("created_at"), + reference=attribution_id, + payload=row, + ) + ) + if status == "converted" and attribution_id not in reward_by_attribution: + row = { + "id": attribution_id, + "code": attribution.get("code"), + "referrer_user_id": attribution.get("referrer_user_id"), + "referred_user_id": attribution.get("referred_user_id"), + "converted_payment_intent_id": attribution.get("converted_payment_intent_id"), + "converted_at": attribution.get("converted_at"), + } + referral_settlement_issues.append(row) + issues.append( + _risk_issue( + category="referral", + severity="high", + title="推荐已转化但没有奖励记录", + detail=f"归因 {attribution_id} 已 converted，但 referral_rewards 未找到对应记录。", + user_id=attribution.get("referrer_user_id"), + created_at=attribution.get("converted_at") or attribution.get("updated_at"), + reference=attribution_id, + payload=row, + ) + ) + + events = db.list_app_analytics_events(limit=20000, since_iso=since_dt.isoformat()) + signup_rows = [ + row + for row in events + if str(row.get("event_type") or "").strip().lower() + in {"signup_success", "signup_completed"} + ] + + def normalize_user_key(value: Any) -> str: + return str(value or "").strip().lower() + + def analytics_payload(row: Dict[str, Any]) -> Dict[str, Any]: + payload = row.get("payload") + return payload if isinstance(payload, dict) else {} + + def analytics_correlation_keys(row: Dict[str, Any]) -> set[str]: + payload = analytics_payload(row) + keys: set[str] = set() + user_id = normalize_user_key(row.get("user_id") or payload.get("user_id")) + client_id = str(row.get("client_id") or "").strip() + session_id = str(row.get("session_id") or "").strip() + if user_id: + keys.add(f"user:{user_id}") + if client_id: + keys.add(f"client:{client_id}") + if session_id: + keys.add(f"session:{session_id}") + return keys + + signup_intent_keys: set[str] = set() + for row in events: + if str(row.get("event_type") or "").strip().lower() != "login_start": + continue + payload = analytics_payload(row) + mode = str(payload.get("mode") or payload.get("auth_mode") or "").strip().lower() + if mode == "signup": + signup_intent_keys.update(analytics_correlation_keys(row)) + + def has_signup_intent(row: Dict[str, Any]) -> bool: + payload = analytics_payload(row) + mode = str(payload.get("mode") or payload.get("auth_mode") or "").strip().lower() + if mode == "signup" or payload.get("signup_intent") is True: + return True + return bool(analytics_correlation_keys(row).intersection(signup_intent_keys)) + + trial_actor_keys = { + _app_analytics_actor_key(row) + for row in events + if str(row.get("event_type") or "").strip().lower() == "trial_created" + } + subscription_user_keys = { + normalize_user_key(row.get("user_id")) + for row in subscription_rows + if normalize_user_key(row.get("user_id")) + } + trial_subscription_user_keys = { + normalize_user_key(row.get("user_id")) + for row in subscription_rows + if normalize_user_key(row.get("user_id")) + and ( + str(row.get("plan_code") or "").strip().lower() == "signup_trial_3d" + or str(row.get("source") or "").strip().lower() == "signup_trial" + ) + } + trial_claim_user_keys = { + normalize_user_key(row.get("user_id")) + for row in trial_claims + if normalize_user_key(row.get("user_id")) + } + trial_event_user_keys: set[str] = set() + for row in entitlement_trial_events: + event_user_id = normalize_user_key(row.get("user_id")) + payload = row.get("payload") if isinstance(row.get("payload"), dict) else {} + payload_user_id = normalize_user_key(payload.get("user_id")) + if event_user_id: + trial_event_user_keys.add(event_user_id) + if payload_user_id: + trial_event_user_keys.add(payload_user_id) + + backend_trial_user_keys = ( + trial_subscription_user_keys | trial_claim_user_keys | trial_event_user_keys + ) + trial_gaps: List[Dict[str, Any]] = [] + + for claim in trial_claims: + claim_user_id = normalize_user_key(claim.get("user_id")) + if not claim_user_id or claim_user_id in trial_subscription_user_keys: + continue + gap = { + "claim_id": claim.get("id"), + "user_id": claim.get("user_id"), + "email": claim.get("email"), + "created_at": claim.get("created_at") or claim.get("claimed_at"), + "reason": "trial_claim_without_subscription", + } + trial_gaps.append(gap) + if len(trial_gaps) <= 20: + issues.append( + _risk_issue( + category="signup_trial", + severity="high", + title="试用 claim 已写入但订阅缺失", + detail="trial_claims 已记录该用户领取试用，但 subscriptions 中没有 signup_trial_3d 记录。", + user_id=gap.get("user_id"), + created_at=gap.get("created_at"), + reference=str(gap.get("claim_id") or ""), + payload=gap, + ) + ) + + for row in signup_rows[:300]: + if not has_signup_intent(row): + continue + actor_key = _app_analytics_actor_key(row) + if actor_key in trial_actor_keys: + continue + payload = analytics_payload(row) + signup_user_id = normalize_user_key(row.get("user_id") or payload.get("user_id")) + if not signup_user_id: + continue + if ( + signup_user_id in backend_trial_user_keys + or signup_user_id in subscription_user_keys + ): + continue + gap = { + "event_id": row.get("id"), + "actor_key": actor_key, + "user_id": signup_user_id, + "created_at": row.get("created_at"), + "reason": "signup_without_backend_trial_evidence", + } + trial_gaps.append(gap) + if len(trial_gaps) <= 20: + issues.append( + _risk_issue( + category="signup_trial", + severity="high", + title="注册成功后未发现后端试用记录", + detail=( + "该用户进入 signup_success，但没有 trial_created、trial_claims、" + "signup_trial subscription 或其他有效订阅证据。" + ), + user_id=gap.get("user_id"), + created_at=gap.get("created_at"), + reference=str(gap.get("event_id") or ""), + payload=gap, + ) + ) + + payment_incidents_raw = db.list_payment_audit_events( + limit=max(safe_limit, 500), + event_type="payment_intent_failed", + ) + grouped_payment_incidents = _group_payment_incidents(payment_incidents_raw) + unresolved_incidents = grouped_payment_incidents["incidents"] + + issues.sort(key=lambda item: str(item.get("created_at") or ""), reverse=True) + recent_rewards = [ + { + "id": row.get("id"), + "referral_attribution_id": row.get("referral_attribution_id"), + "referrer_user_id": row.get("referrer_user_id"), + "referred_user_id": row.get("referred_user_id"), + "payment_intent_id": row.get("payment_intent_id"), + "reward_points": int(row.get("reward_points") or 0), + "reward_days": int(row.get("reward_days") or 0), + "tx_hash": row.get("tx_hash"), + "explorer_url": _payment_explorer_url("polygon", row.get("tx_hash")), + "created_at": row.get("created_at"), + } + for row in referral_rewards[:safe_limit] + ] + + return { + "checked_at": _to_utc_iso(now), + "window_days": safe_days, + "summary": { + "issues": len(issues), + "stuck_intents": len(stuck_intents), + "trial_gaps": len(trial_gaps), + "payment_incidents": grouped_payment_incidents["total"], + "payment_incident_events": grouped_payment_incidents["raw_total"], + "points_discount_issues": len(points_issues), + "referral_settlement_issues": len(referral_settlement_issues), + "monthly_cap_hits": len(monthly_cap_hits), + "recent_referral_rewards": len(recent_rewards), + "recent_trial_claims": len(trial_claims), + }, + "issues": issues[:safe_limit], + "stuck_intents": stuck_intents[:safe_limit], + "trial_gaps": trial_gaps[:safe_limit], + "payment_incidents": unresolved_incidents[:safe_limit], + "points_discount_issues": points_issues[:safe_limit], + "referral_settlement_issues": referral_settlement_issues[:safe_limit], + "monthly_cap_hits": monthly_cap_hits[:safe_limit], + "recent_referral_rewards": recent_rewards, + "recent_trial_claims": trial_claims, + "query_errors": query_errors, + } + + +# ═══════════════════════════════════════════════════════════════════════ +# Membership helpers +# ═══════════════════════════════════════════════════════════════════════ + +def _list_active_subscriptions_with_windows( + limit: int, +) -> tuple[list[dict[str, Any]], dict[str, dict[str, Any]], bool]: + active_window_query = getattr( + legacy_routes.SUPABASE_ENTITLEMENT, + "list_active_subscription_windows", + None, + ) + if not callable(active_window_query): + return [], {}, False + try: + active_window_payload = active_window_query(limit=limit) + except Exception: + return [], {}, False + if not isinstance(active_window_payload, dict): + return [], {}, False + + active_window_subscriptions = active_window_payload.get("subscriptions") + active_window_windows = active_window_payload.get("windows") + if not isinstance(active_window_subscriptions, list): + return [], {}, False + if not active_window_subscriptions and not ( + isinstance(active_window_windows, dict) and active_window_windows + ): + return [], {}, False + + subscriptions = [ + item for item in active_window_subscriptions if isinstance(item, dict) + ] + subscription_windows = ( + active_window_windows if isinstance(active_window_windows, dict) else {} + ) + return subscriptions, subscription_windows, True + + +def _build_membership_rows( + db: DBManager, + subscriptions: list[dict[str, Any]], + subscription_windows: dict[str, dict[str, Any]], +) -> list[dict[str, Any]]: + subscription_user_ids = [str(item.get("user_id") or "") for item in subscriptions] + user_map = db.get_users_by_supabase_user_ids(subscription_user_ids) + unresolved_user_ids = [ + user_id + for user_id in subscription_user_ids + if str(user_id or "").strip().lower() + and not str( + (user_map.get(str(user_id).strip().lower(), {}) or {}).get("supabase_email") + or "" + ).strip() + ] + auth_user_map = legacy_routes.SUPABASE_ENTITLEMENT.get_auth_users( + unresolved_user_ids + ) + if not subscription_windows: + subscription_windows = legacy_routes.SUPABASE_ENTITLEMENT.list_subscription_windows( + subscription_user_ids, + bypass_cache=True, + ) + deduped: dict[str, dict] = {} + for item in subscriptions: + user_id = str(item.get("user_id") or "").strip().lower() + local_user = user_map.get(user_id, {}) + auth_user = auth_user_map.get(user_id, {}) + subscription_window = subscription_windows.get(user_id, {}) + current_expires_at = item.get("expires_at") + total_expires_at = ( + subscription_window.get("total_expires_at") + if isinstance(subscription_window, dict) + else None + ) + queued_days = ( + int(subscription_window.get("queued_days") or 0) + if isinstance(subscription_window, dict) + else 0 + ) + queued_count = ( + int(subscription_window.get("queued_count") or 0) + if isinstance(subscription_window, dict) + else 0 + ) + source = str(item.get("source") or "") + is_trial = source == "signup_trial" or str( + item.get("plan_code") or "" + ).startswith("signup_trial") + row = { + "user_id": user_id, + "email": str( + auth_user.get("email") or local_user.get("supabase_email") or "" + ), + "telegram_id": local_user.get("telegram_id"), + "username": local_user.get("username"), + "registered_at": local_user.get("created_at") + or auth_user.get("created_at"), + "plan_code": item.get("plan_code"), + "source": source, + "is_trial": is_trial, + "starts_at": item.get("starts_at"), + "current_expires_at": current_expires_at, + "total_expires_at": total_expires_at or current_expires_at, + "expires_at": total_expires_at or current_expires_at, + "queued_days": queued_days, + "queued_count": queued_count, + } + existing = deduped.get(user_id) + existing_expires = str(existing.get("expires_at") or "") if existing else "" + current_expires = str(row.get("expires_at") or "") + if existing is None or current_expires > existing_expires: + deduped[user_id] = row + return sorted( + deduped.values(), + key=lambda item: str(item.get("expires_at") or ""), + ) + + +def _build_membership_growth( + subscriptions: list[dict[str, Any]], + days: int, +) -> dict[str, Any]: + safe_days = max(7, min(365, int(days or 90))) + now = datetime.utcnow() + cutoff = now - timedelta(days=safe_days) + + trial_by_day: dict[str, int] = defaultdict(int) + paid_by_day: dict[str, int] = defaultdict(int) + running = 0 + + for item in subscriptions: + starts_raw = str(item.get("starts_at") or "").strip() + if not starts_raw: + continue + try: + dt = datetime.fromisoformat(starts_raw.replace("Z", "+00:00")) + if dt.tzinfo is not None: + dt = dt.replace(tzinfo=None) + except Exception: + continue + if dt < cutoff: + continue + day_key = dt.strftime("%Y-%m-%d") + source = str(item.get("source") or "").strip().lower() + plan = str(item.get("plan_code") or "").strip().lower() + is_trial = source == "signup_trial" or plan.startswith("signup_trial") + if is_trial: + trial_by_day[day_key] += 1 + else: + paid_by_day[day_key] += 1 + + daily = [] + cursor = cutoff.date() + while cursor <= now.date(): + key = cursor.isoformat() + tc = trial_by_day.get(key, 0) + pc = paid_by_day.get(key, 0) + total = tc + pc + running += total + daily.append( + { + "date": key, + "trial": tc, + "paid": pc, + "total": total, + "cumulative": running, + } + ) + cursor += timedelta(days=1) + + return {"days": safe_days, "daily": daily} + + +# ═══════════════════════════════════════════════════════════════════════ +# Membership API +# ═══════════════════════════════════════════════════════════════════════ + +def list_ops_memberships(request: Request, limit: int = 200) -> Dict[str, Any]: + _require_ops(request) + db = _get_db() + reconcile_enabled = ( + str(os.getenv("POLYWEATHER_OPS_MEMBERSHIPS_RECONCILE_ENABLED") or "") + .strip() + .lower() + in {"1", "true", "yes", "on"} + ) + if reconcile_enabled and getattr(legacy_routes.PAYMENT_CHECKOUT, "enabled", False): + try: + legacy_routes.PAYMENT_CHECKOUT.reconcile_recent_intents( + limit=min(max(int(limit or 200), 20), 200) + ) + except Exception: + pass + subscriptions, subscription_windows, used_active_window_query = ( + _list_active_subscriptions_with_windows(limit) + ) + if not used_active_window_query: + subscriptions = legacy_routes.SUPABASE_ENTITLEMENT.list_active_subscriptions( + limit=limit + ) + return { + "memberships": _build_membership_rows( + db, + subscriptions, + subscription_windows, + ) + } + + +def get_ops_memberships_growth(request: Request, days: int = 90) -> dict[str, Any]: + _require_ops(request) + + subscriptions, _, used_active_window_query = _list_active_subscriptions_with_windows( + limit=5000 + ) + if not used_active_window_query: + subscriptions = legacy_routes.SUPABASE_ENTITLEMENT.list_active_subscriptions( + limit=5000 + ) + return _build_membership_growth(subscriptions, days) + + +def get_ops_memberships_overview( + request: Request, + limit: int = 200, + days: int = 90, +) -> dict[str, Any]: + _require_ops(request) + db = _get_db() + safe_limit = max(1, min(int(limit or 200), 1000)) + query_limit = max(safe_limit, 5000) + subscriptions, subscription_windows, used_active_window_query = ( + _list_active_subscriptions_with_windows(limit=query_limit) + ) + if not used_active_window_query: + subscriptions = legacy_routes.SUPABASE_ENTITLEMENT.list_active_subscriptions( + limit=query_limit + ) + membership_subscriptions = subscriptions[:safe_limit] + return { + "memberships": _build_membership_rows( + db, + membership_subscriptions, + subscription_windows, + ), + **_build_membership_growth(subscriptions, days), + } diff --git a/web/services/ops/users.py b/web/services/ops/users.py new file mode 100644 index 00000000..a75cbbde --- /dev/null +++ b/web/services/ops/users.py @@ -0,0 +1,213 @@ +"""Ops users / feedback / points / analytics service functions.""" +from __future__ import annotations + +from typing import Any, Dict, Optional + +from fastapi import HTTPException, Request + + +from web.core import GrantPointsRequest +import web.routes as legacy_routes + + +def _get_db(): + from web.services.ops_api import DBManager + return DBManager() + +# _require_ops is a lightweight auth guard duplicated in each ops submodule +# to avoid circular imports with the ops_api re-export hub. +def _require_ops(request: Request): + from web.services.ops_api import _require_ops as _real + return _real(request) + + +# ═══════════════════════════════════════════════════════════════════════ +# Internal helpers +# ═══════════════════════════════════════════════════════════════════════ + +def _sf(value: Any) -> Optional[float]: + try: + if value is None or value == "": + return None + return float(value) + except (TypeError, ValueError): + return None + + +def _round_metric(value: Optional[float], digits: int = 1) -> Optional[float]: + return None if value is None else round(float(value), digits) + + +def _app_analytics_actor_key(row: Dict[str, Any]) -> str: + payload = row.get("payload") + payload = payload if isinstance(payload, dict) else {} + user_id = str(row.get("user_id") or payload.get("user_id") or "").strip().lower() + client_id = str(row.get("client_id") or "").strip() + session_id = str(row.get("session_id") or "").strip() + if user_id: + return f"user:{user_id}" + if client_id: + return f"client:{client_id}" + if session_id: + return f"session:{session_id}" + return f"event:{row.get('id')}" + + +# ═══════════════════════════════════════════════════════════════════════ +# Users +# ═══════════════════════════════════════════════════════════════════════ + +def search_ops_users(request: Request, q: str = "", limit: int = 20) -> Dict[str, Any]: + _require_ops(request) + db = _get_db() + return {"users": db.search_users(q, limit=limit)} + + +def get_ops_weekly_leaderboard(request: Request, limit: int = 20) -> Dict[str, Any]: + _require_ops(request) + db = _get_db() + return {"leaderboard": db.get_weekly_leaderboard(limit=limit)} + + +# ═══════════════════════════════════════════════════════════════════════ +# Points +# ═══════════════════════════════════════════════════════════════════════ + +def grant_ops_points(request: Request, body: GrantPointsRequest) -> Dict[str, Any]: + admin = _require_ops(request) or {} + db = _get_db() + result = db.grant_points_by_supabase_email(body.email, body.points) + result["operator_email"] = admin.get("email") + if not result.get("ok"): + reason = str(result.get("reason") or "grant_points_failed") + status_code = 404 if reason == "user_not_found" else 400 + raise HTTPException(status_code=status_code, detail=result) + return result + + +def transfer_ops_points( + request: Request, + from_email: str = "", + to_email: str = "", + amount: int = 0, +) -> Dict[str, Any]: + """Transfer points from one user to another.""" + admin = _require_ops(request) or {} + from_email = str(from_email or "").strip() + to_email = str(to_email or "").strip() + amount = int(amount or 0) + if not from_email or not to_email: + raise HTTPException( + status_code=400, detail="from_email and to_email are required" + ) + if amount <= 0: + raise HTTPException(status_code=400, detail="amount must be positive") + db = _get_db() + result = db.transfer_points_by_email(from_email, to_email, amount) + result["operator_email"] = admin.get("email") + if not result.get("ok"): + raise HTTPException(status_code=400, detail=result) + return result + + +# ═══════════════════════════════════════════════════════════════════════ +# Analytics +# ═══════════════════════════════════════════════════════════════════════ + +def get_ops_analytics_funnel(request: Request, days: int = 30) -> Dict[str, Any]: + _require_ops(request) + db = _get_db() + return db.get_app_analytics_funnel_summary(days=days) + + +# ═══════════════════════════════════════════════════════════════════════ +# Feedback +# ═══════════════════════════════════════════════════════════════════════ + +def list_ops_feedback( + request: Request, + *, + limit: int = 100, + status: str = "", +) -> Dict[str, Any]: + _require_ops(request) + db = _get_db() + rows = db.list_user_feedback(limit=limit, status=status or None) + status_counts: Dict[str, int] = {} + recent_rows = db.list_user_feedback(limit=500) + for row in recent_rows: + key = str(row.get("status") or "unknown") + status_counts[key] = status_counts.get(key, 0) + 1 + return { + "feedback": rows, + "total": len(rows), + "status_counts": status_counts, + } + + +def update_ops_feedback_status( + request: Request, + *, + feedback_id: int, + status: str, +) -> Dict[str, Any]: + _require_ops(request) + normalized = str(status or "").strip().lower() + allowed = {"open", "triaged", "investigating", "resolved", "closed"} + if normalized not in allowed: + raise HTTPException(status_code=400, detail="unsupported feedback status") + updated = _get_db().update_user_feedback_status(feedback_id, status=normalized) + if not updated: + raise HTTPException(status_code=404, detail="feedback not found") + return {"ok": True, "feedback": updated} + + +def grant_ops_feedback_reward( + request: Request, + *, + feedback_id: int, + points: int, + reason: str = "", +) -> Dict[str, Any]: + admin = _require_ops(request) or {} + db = _get_db() + result = db.grant_feedback_reward( + feedback_id, + points=points, + reason=reason, + ) + if not result.get("ok") and str(result.get("reason") or "") == "user_not_found": + feedback = result.get("feedback") if isinstance(result.get("feedback"), dict) else {} + reward_status = str(feedback.get("reward_status") or "").strip().lower() + reward_points = int(feedback.get("reward_points") or 0) + supabase_user_id = str(feedback.get("user_id") or "").strip().lower() + if supabase_user_id and not (reward_status == "granted" and reward_points > 0): + try: + fallback = legacy_routes.SUPABASE_ENTITLEMENT.grant_points_to_user( + supabase_user_id, + points, + ) + except Exception as exc: + fallback = {"ok": False, "reason": f"supabase_points_grant_failed:{exc}"} + if fallback.get("ok"): + updated_feedback = db.update_user_feedback_reward( + feedback_id, + points=points, + reason=reason, + status="granted", + ) + result = { + **fallback, + "ok": True, + "feedback_id": int(feedback_id), + "supabase_user_id": supabase_user_id, + "feedback": updated_feedback, + } + result["operator_email"] = admin.get("email") + if not result.get("ok"): + reason_code = str(result.get("reason") or "feedback_reward_failed") + status_code = 404 if reason_code in {"feedback_not_found", "user_not_found"} else 400 + if reason_code == "already_rewarded": + status_code = 409 + raise HTTPException(status_code=status_code, detail=result) + return result diff --git a/web/services/ops_api.py b/web/services/ops_api.py index edf3e0f7..d0c3c19a 100644 --- a/web/services/ops_api.py +++ b/web/services/ops_api.py @@ -1,2876 +1,84 @@ -"""Operations/admin API service functions.""" +"""Operations/admin API service functions. + +This module is a backward-compatible re-export hub. Domain logic lives in +the `web.services.ops` sub-package: + + ops/users.py — users, points, feedback, analytics, leaderboard + ops/payments.py — payment incidents, billing risk, memberships + ops/health.py — health check, source health, training accuracy, truth + ops/config.py — config, subscriptions, logs, telegram audit +""" from __future__ import annotations -import os -from datetime import datetime, timedelta, timezone -from typing import Any, Dict, List, Optional, Tuple +# Backward-compat: tests monkeypatch ops_api._requests, ops_api.legacy_routes, +# ops_api.DBManager and other module-level symbols. +import requests as _requests # noqa: F401 +import web.routes as legacy_routes # noqa: F401 +from src.database.db_manager import DBManager # noqa: F401 +from src.database.runtime_state import ObservationCollectorStatusRepository # noqa: F401 -from fastapi import HTTPException, Request -import requests as _requests - -from src.database.db_manager import DBManager -from src.database.runtime_state import ObservationCollectorStatusRepository -from src.utils.runtime_secrets import get_runtime_secret, get_runtime_secret_status -from web.services.observation_freshness import ( - build_observation_freshness, - canonical_observation_source_code, -) -from web.core import GrantPointsRequest -import web.routes as legacy_routes - -_DEB_VERSION_BACKTEST_SAMPLE_LIMIT = 400 +# Backward-compat: internal helpers referenced by tests +from web.services.ops.config import _lookup_supabase_user_id_by_email # noqa: F401 +from web.services.ops.health import _check_amsc_awos_health # noqa: F401 +from web.services.ops.payments import _list_active_subscriptions_with_windows # noqa: F401 -def _sf(value: Any) -> Optional[float]: - try: - if value is None or value == "": - return None - return float(value) - except (TypeError, ValueError): - return None - - -def _round_metric(value: Optional[float], digits: int = 1) -> Optional[float]: - return None if value is None else round(float(value), digits) - - -def _require_ops(request: Request) -> Dict[str, Any] | None: - # Ops admins are authenticated via Supabase identity + email whitelist. - # They do NOT need an active Pro subscription to manage the system. +def _require_ops(request): return legacy_routes._require_ops_admin(request) - -def _parse_iso_datetime(value: Any) -> Optional[datetime]: - text = str(value or "").strip() - if not text: - return None - try: - parsed = datetime.fromisoformat(text.replace("Z", "+00:00")) - except Exception: - return None - if parsed.tzinfo is None: - parsed = parsed.replace(tzinfo=timezone.utc) - return parsed.astimezone(timezone.utc) - - -def _to_utc_iso(value: datetime) -> str: - if value.tzinfo is None: - value = value.replace(tzinfo=timezone.utc) - return value.astimezone(timezone.utc).isoformat().replace("+00:00", "Z") - - -def _supabase_rest_rows( - table: str, - params: Dict[str, Any], - *, - timeout: int = 10, -) -> List[Dict[str, Any]]: - supabase_url = str(os.getenv("SUPABASE_URL") or "").strip().rstrip("/") - service_role_key = str(os.getenv("SUPABASE_SERVICE_ROLE_KEY") or "").strip() - if not supabase_url or not service_role_key: - raise HTTPException(status_code=503, detail="Supabase not configured") - - headers = { - "apikey": service_role_key, - "Authorization": f"Bearer {service_role_key}", - } - resp = _requests.get( - f"{supabase_url}/rest/v1/{table}", - headers=headers, - params=params, - timeout=timeout, - ) - if not resp.ok: - raise HTTPException( - status_code=502, - detail=f"Supabase query failed for {table}: {resp.status_code}", - ) - rows = resp.json() if resp.content else [] - if not isinstance(rows, list): - return [] - return [row for row in rows if isinstance(row, dict)] - - -def _payment_explorer_url(chain: Any, tx_hash: Any) -> str: - tx = str(tx_hash or "").strip() - if not tx: - return "" - chain_text = str(chain or "").strip().lower() - base = "https://etherscan.io" if "eth" in chain_text else "https://polygonscan.com" - return f"{base}/tx/{tx}" - - -def _app_analytics_actor_key(row: Dict[str, Any]) -> str: - payload = row.get("payload") - payload = payload if isinstance(payload, dict) else {} - user_id = str(row.get("user_id") or payload.get("user_id") or "").strip().lower() - client_id = str(row.get("client_id") or "").strip() - session_id = str(row.get("session_id") or "").strip() - if user_id: - return f"user:{user_id}" - if client_id: - return f"client:{client_id}" - if session_id: - return f"session:{session_id}" - return f"event:{row.get('id')}" - - -def _risk_issue( - *, - category: str, - severity: str, - title: str, - detail: str, - user_id: Any = "", - created_at: Any = "", - reference: Any = "", - payload: Optional[Dict[str, Any]] = None, -) -> Dict[str, Any]: - return { - "category": category, - "severity": severity, - "title": title, - "detail": detail, - "user_id": str(user_id or ""), - "created_at": str(created_at or ""), - "reference": str(reference or ""), - "payload": payload or {}, - } - - -def search_ops_users(request: Request, q: str = "", limit: int = 20) -> Dict[str, Any]: - _require_ops(request) - db = DBManager() - return {"users": db.search_users(q, limit=limit)} - - -def get_ops_weekly_leaderboard(request: Request, limit: int = 20) -> Dict[str, Any]: - _require_ops(request) - db = DBManager() - return {"leaderboard": db.get_weekly_leaderboard(limit=limit)} - - -def list_ops_feedback( - request: Request, - *, - limit: int = 100, - status: str = "", -) -> Dict[str, Any]: - _require_ops(request) - db = DBManager() - rows = db.list_user_feedback(limit=limit, status=status or None) - status_counts: Dict[str, int] = {} - recent_rows = db.list_user_feedback(limit=500) - for row in recent_rows: - key = str(row.get("status") or "unknown") - status_counts[key] = status_counts.get(key, 0) + 1 - return { - "feedback": rows, - "total": len(rows), - "status_counts": status_counts, - } - - -def update_ops_feedback_status( - request: Request, - *, - feedback_id: int, - status: str, -) -> Dict[str, Any]: - _require_ops(request) - normalized = str(status or "").strip().lower() - allowed = {"open", "triaged", "investigating", "resolved", "closed"} - if normalized not in allowed: - raise HTTPException(status_code=400, detail="unsupported feedback status") - updated = DBManager().update_user_feedback_status(feedback_id, status=normalized) - if not updated: - raise HTTPException(status_code=404, detail="feedback not found") - return {"ok": True, "feedback": updated} - - -def grant_ops_feedback_reward( - request: Request, - *, - feedback_id: int, - points: int, - reason: str = "", -) -> Dict[str, Any]: - admin = _require_ops(request) or {} - db = DBManager() - result = db.grant_feedback_reward( - feedback_id, - points=points, - reason=reason, - ) - if not result.get("ok") and str(result.get("reason") or "") == "user_not_found": - feedback = result.get("feedback") if isinstance(result.get("feedback"), dict) else {} - reward_status = str(feedback.get("reward_status") or "").strip().lower() - reward_points = int(feedback.get("reward_points") or 0) - supabase_user_id = str(feedback.get("user_id") or "").strip().lower() - if supabase_user_id and not (reward_status == "granted" and reward_points > 0): - try: - fallback = legacy_routes.SUPABASE_ENTITLEMENT.grant_points_to_user( - supabase_user_id, - points, - ) - except Exception as exc: - fallback = {"ok": False, "reason": f"supabase_points_grant_failed:{exc}"} - if fallback.get("ok"): - updated_feedback = db.update_user_feedback_reward( - feedback_id, - points=points, - reason=reason, - status="granted", - ) - result = { - **fallback, - "ok": True, - "feedback_id": int(feedback_id), - "supabase_user_id": supabase_user_id, - "feedback": updated_feedback, - } - result["operator_email"] = admin.get("email") - if not result.get("ok"): - reason_code = str(result.get("reason") or "feedback_reward_failed") - status_code = 404 if reason_code in {"feedback_not_found", "user_not_found"} else 400 - if reason_code == "already_rewarded": - status_code = 409 - raise HTTPException(status_code=status_code, detail=result) - return result - - -def _list_active_subscriptions_with_windows( - limit: int, -) -> tuple[list[dict[str, Any]], dict[str, dict[str, Any]], bool]: - active_window_query = getattr( - legacy_routes.SUPABASE_ENTITLEMENT, - "list_active_subscription_windows", - None, - ) - if not callable(active_window_query): - return [], {}, False - try: - active_window_payload = active_window_query(limit=limit) - except Exception: - return [], {}, False - if not isinstance(active_window_payload, dict): - return [], {}, False - - active_window_subscriptions = active_window_payload.get("subscriptions") - active_window_windows = active_window_payload.get("windows") - if not isinstance(active_window_subscriptions, list): - return [], {}, False - if not active_window_subscriptions and not ( - isinstance(active_window_windows, dict) and active_window_windows - ): - return [], {}, False - - subscriptions = [ - item for item in active_window_subscriptions if isinstance(item, dict) - ] - subscription_windows = ( - active_window_windows if isinstance(active_window_windows, dict) else {} - ) - return subscriptions, subscription_windows, True - - -def _build_membership_rows( - db: DBManager, - subscriptions: list[dict[str, Any]], - subscription_windows: dict[str, dict[str, Any]], -) -> list[dict[str, Any]]: - subscription_user_ids = [str(item.get("user_id") or "") for item in subscriptions] - user_map = db.get_users_by_supabase_user_ids(subscription_user_ids) - unresolved_user_ids = [ - user_id - for user_id in subscription_user_ids - if str(user_id or "").strip().lower() - and not str( - (user_map.get(str(user_id).strip().lower(), {}) or {}).get("supabase_email") - or "" - ).strip() - ] - auth_user_map = legacy_routes.SUPABASE_ENTITLEMENT.get_auth_users( - unresolved_user_ids - ) - if not subscription_windows: - subscription_windows = legacy_routes.SUPABASE_ENTITLEMENT.list_subscription_windows( - subscription_user_ids, - bypass_cache=True, - ) - deduped: dict[str, dict] = {} - for item in subscriptions: - user_id = str(item.get("user_id") or "").strip().lower() - local_user = user_map.get(user_id, {}) - auth_user = auth_user_map.get(user_id, {}) - subscription_window = subscription_windows.get(user_id, {}) - current_expires_at = item.get("expires_at") - total_expires_at = ( - subscription_window.get("total_expires_at") - if isinstance(subscription_window, dict) - else None - ) - queued_days = ( - int(subscription_window.get("queued_days") or 0) - if isinstance(subscription_window, dict) - else 0 - ) - queued_count = ( - int(subscription_window.get("queued_count") or 0) - if isinstance(subscription_window, dict) - else 0 - ) - source = str(item.get("source") or "") - is_trial = source == "signup_trial" or str( - item.get("plan_code") or "" - ).startswith("signup_trial") - row = { - "user_id": user_id, - "email": str( - auth_user.get("email") or local_user.get("supabase_email") or "" - ), - "telegram_id": local_user.get("telegram_id"), - "username": local_user.get("username"), - "registered_at": local_user.get("created_at") - or auth_user.get("created_at"), - "plan_code": item.get("plan_code"), - "source": source, - "is_trial": is_trial, - "starts_at": item.get("starts_at"), - "current_expires_at": current_expires_at, - "total_expires_at": total_expires_at or current_expires_at, - "expires_at": total_expires_at or current_expires_at, - "queued_days": queued_days, - "queued_count": queued_count, - } - existing = deduped.get(user_id) - existing_expires = str(existing.get("expires_at") or "") if existing else "" - current_expires = str(row.get("expires_at") or "") - if existing is None or current_expires > existing_expires: - deduped[user_id] = row - return sorted( - deduped.values(), - key=lambda item: str(item.get("expires_at") or ""), - ) - - -def _build_membership_growth( - subscriptions: list[dict[str, Any]], - days: int, -) -> dict[str, Any]: - from collections import defaultdict - from datetime import datetime, timedelta - - safe_days = max(7, min(365, int(days or 90))) - now = datetime.utcnow() - cutoff = now - timedelta(days=safe_days) - - trial_by_day: dict[str, int] = defaultdict(int) - paid_by_day: dict[str, int] = defaultdict(int) - running = 0 - - for item in subscriptions: - starts_raw = str(item.get("starts_at") or "").strip() - if not starts_raw: - continue - try: - dt = datetime.fromisoformat(starts_raw.replace("Z", "+00:00")) - if dt.tzinfo is not None: - dt = dt.replace(tzinfo=None) - except Exception: - continue - if dt < cutoff: - continue - day_key = dt.strftime("%Y-%m-%d") - source = str(item.get("source") or "").strip().lower() - plan = str(item.get("plan_code") or "").strip().lower() - is_trial = source == "signup_trial" or plan.startswith("signup_trial") - if is_trial: - trial_by_day[day_key] += 1 - else: - paid_by_day[day_key] += 1 - - daily = [] - cursor = cutoff.date() - while cursor <= now.date(): - key = cursor.isoformat() - tc = trial_by_day.get(key, 0) - pc = paid_by_day.get(key, 0) - total = tc + pc - running += total - daily.append( - { - "date": key, - "trial": tc, - "paid": pc, - "total": total, - "cumulative": running, - } - ) - cursor += timedelta(days=1) - - return {"days": safe_days, "daily": daily} - - -def list_ops_memberships(request: Request, limit: int = 200) -> Dict[str, Any]: - _require_ops(request) - db = DBManager() - reconcile_enabled = ( - str(os.getenv("POLYWEATHER_OPS_MEMBERSHIPS_RECONCILE_ENABLED") or "") - .strip() - .lower() - in {"1", "true", "yes", "on"} - ) - if reconcile_enabled and getattr(legacy_routes.PAYMENT_CHECKOUT, "enabled", False): - try: - legacy_routes.PAYMENT_CHECKOUT.reconcile_recent_intents( - limit=min(max(int(limit or 200), 20), 200) - ) - except Exception: - pass - subscriptions, subscription_windows, used_active_window_query = ( - _list_active_subscriptions_with_windows(limit) - ) - if not used_active_window_query: - subscriptions = legacy_routes.SUPABASE_ENTITLEMENT.list_active_subscriptions( - limit=limit - ) - return { - "memberships": _build_membership_rows( - db, - subscriptions, - subscription_windows, - ) - } - - -def get_ops_memberships_growth(request: Request, days: int = 90) -> dict[str, Any]: - _require_ops(request) - - subscriptions, _, used_active_window_query = _list_active_subscriptions_with_windows( - limit=5000 - ) - if not used_active_window_query: - subscriptions = legacy_routes.SUPABASE_ENTITLEMENT.list_active_subscriptions( - limit=5000 - ) - return _build_membership_growth(subscriptions, days) - - -def get_ops_memberships_overview( - request: Request, - limit: int = 200, - days: int = 90, -) -> dict[str, Any]: - _require_ops(request) - db = DBManager() - safe_limit = max(1, min(int(limit or 200), 1000)) - query_limit = max(safe_limit, 5000) - subscriptions, subscription_windows, used_active_window_query = ( - _list_active_subscriptions_with_windows(limit=query_limit) - ) - if not used_active_window_query: - subscriptions = legacy_routes.SUPABASE_ENTITLEMENT.list_active_subscriptions( - limit=query_limit - ) - membership_subscriptions = subscriptions[:safe_limit] - return { - "memberships": _build_membership_rows( - db, - membership_subscriptions, - subscription_windows, - ), - **_build_membership_growth(subscriptions, days), - } - - -def _normalize_payment_incident(item: Dict[str, Any]) -> Dict[str, Any]: - payload = item.get("payload") if isinstance(item, dict) else {} - payload = payload if isinstance(payload, dict) else {} - confirm_failure = ( - payload.get("confirm_failure") - if isinstance(payload.get("confirm_failure"), dict) - else {} - ) - reason = str( - payload.get("reason") - or confirm_failure.get("reason") - or payload.get("error") - or "unknown" - ).strip().lower() - detail = str( - payload.get("detail") - or confirm_failure.get("detail") - or payload.get("message") - or payload.get("error") - or "" - ).strip() - resolved_at = str(payload.get("resolved_at") or "").strip() - return { - **item, - "payload": payload, - "reason": reason or "unknown", - "detail": detail, - "intent_id": str( - payload.get("intent_id") - or payload.get("payment_intent_id") - or confirm_failure.get("intent_id") - or "" - ).strip(), - "user_id": str(payload.get("user_id") or "").strip(), - "tx_hash": str( - payload.get("tx_hash") - or confirm_failure.get("tx_hash") - or "" - ).strip(), - "resolved": bool(resolved_at), - "resolved_at": resolved_at, - "resolved_by": str(payload.get("resolved_by") or "").strip(), - } - - -def _payment_incident_group_key(item: Dict[str, Any]) -> Tuple[str, str, str, str]: - reason = str(item.get("reason") or "unknown").strip().lower() - intent_id = str(item.get("intent_id") or "").strip().lower() - tx_hash = str(item.get("tx_hash") or "").strip().lower() - user_id = str(item.get("user_id") or "").strip().lower() - if intent_id or tx_hash: - return reason, user_id, intent_id, tx_hash - return reason, user_id, f"event:{item.get('id')}", "" - - -def _group_payment_incidents( - incidents: List[Dict[str, Any]], - *, - reason: str = "", - include_resolved: bool = False, -) -> Dict[str, Any]: - normalized_reason = str(reason or "").strip().lower() - groups: Dict[Tuple[str, str, str, str], Dict[str, Any]] = {} - raw_total = 0 - - for item in incidents: - normalized_item = _normalize_payment_incident(item) - item_reason = str(normalized_item.get("reason") or "").strip().lower() - resolved = bool(normalized_item.get("resolved")) - if normalized_reason and item_reason != normalized_reason: - continue - if not include_resolved and resolved: - continue - raw_total += 1 - - key = _payment_incident_group_key(normalized_item) - created_at = str(normalized_item.get("created_at") or "").strip() - event_id = int(normalized_item.get("id") or 0) - existing = groups.get(key) - if existing is None: - grouped = { - **normalized_item, - "occurrence_count": 1, - "event_ids": [event_id] if event_id > 0 else [], - "first_seen_at": created_at, - "last_seen_at": created_at, - } - groups[key] = grouped - continue - - existing["occurrence_count"] = int(existing.get("occurrence_count") or 1) + 1 - if event_id > 0: - existing.setdefault("event_ids", []).append(event_id) - first_seen = str(existing.get("first_seen_at") or "").strip() - last_seen = str(existing.get("last_seen_at") or "").strip() - if created_at and (not first_seen or created_at < first_seen): - existing["first_seen_at"] = created_at - if created_at and (not last_seen or created_at > last_seen): - existing["last_seen_at"] = created_at - - grouped_items = list(groups.values()) - grouped_items.sort( - key=lambda item: ( - str(item.get("last_seen_at") or item.get("created_at") or ""), - int(item.get("id") or 0), - ), - reverse=True, - ) - return { - "incidents": grouped_items, - "raw_total": raw_total, - "total": len(grouped_items), - } - - -def list_ops_payment_incidents( - request: Request, - limit: int = 50, - reason: str = "", - include_resolved: bool = False, -) -> Dict[str, Any]: - _require_ops(request) - db = DBManager() - safe_limit = max(1, min(int(limit or 50), 200)) - incidents = db.list_payment_audit_events( - limit=max(safe_limit, 500), - event_type="payment_intent_failed", - ) - grouped = _group_payment_incidents( - incidents, - reason=reason, - include_resolved=include_resolved, - ) - return { - **grouped, - "incidents": grouped["incidents"][:safe_limit], - } - - -def resolve_ops_payment_incident(request: Request, event_id: int) -> Dict[str, Any]: - admin = _require_ops(request) or {} - db = DBManager() - resolved_group = db.mark_related_payment_audit_events_resolved( - event_id, str(admin.get("email") or "") - ) - if not resolved_group: - raise HTTPException(status_code=404, detail="payment_incident_not_found") - return { - "ok": True, - "incident": resolved_group[0], - "resolved_count": len(resolved_group), - "resolved_event_ids": [int(item.get("id") or 0) for item in resolved_group], - } - - -def list_ops_payments( - request: Request, - limit: int = 50, -) -> Dict[str, Any]: - """List successful payment records from Supabase.""" - _require_ops(request) - safe_limit = max(1, min(int(limit or 50), 200)) - rows = _supabase_rest_rows( - "payments", - { - "select": "id,user_id,amount,currency,chain,tx_hash,status,created_at", - "order": "created_at.desc", - "limit": str(safe_limit), - }, - ) - return {"payments": rows, "total": len(rows)} - - -def get_ops_billing_risk( - request: Request, - days: int = 30, - limit: int = 80, -) -> Dict[str, Any]: - """Summarize trial, payment, referral, and points risk signals for ops.""" - _require_ops(request) - db = DBManager() - now = datetime.now(timezone.utc) - safe_days = max(1, min(int(days or 30), 120)) - safe_limit = max(10, min(int(limit or 80), 200)) - since_dt = now - timedelta(days=safe_days) - month_start = now.replace(day=1, hour=0, minute=0, second=0, microsecond=0) - - query_errors: List[Dict[str, str]] = [] - - def collect(table: str, params: Dict[str, Any]) -> List[Dict[str, Any]]: - try: - return _supabase_rest_rows(table, params) - except Exception as exc: - query_errors.append({"table": table, "error": str(exc)[:180]}) - return [] - - intents = collect( - "payment_intents", - { - "select": ( - "id,user_id,plan_code,chain_id,status,expires_at,tx_hash," - "metadata,created_at,updated_at" - ), - "order": "updated_at.desc", - "limit": str(max(safe_limit * 3, 100)), - }, - ) - referral_attributions = collect( - "referral_attributions", - { - "select": ( - "id,referrer_user_id,referred_user_id,code,status," - "converted_payment_intent_id,converted_tx_hash,converted_at," - "created_at,updated_at" - ), - "order": "updated_at.desc", - "limit": str(safe_limit), - }, - ) - referral_rewards = collect( - "referral_rewards", - { - "select": ( - "id,referral_attribution_id,referrer_user_id,referred_user_id," - "payment_intent_id,tx_hash,reward_days,reward_points,created_at" - ), - "order": "created_at.desc", - "limit": str(safe_limit), - }, - ) - trial_claims = collect( - "trial_claims", - { - "select": "id,user_id,email,telegram_user_id,claimed_at,created_at", - "order": "created_at.desc", - "limit": str(max(safe_limit * 10, 500)), - }, - ) - trial_subscription_rows = collect( - "subscriptions", - { - "select": ( - "id,user_id,plan_code,source,status,starts_at,expires_at," - "created_at,updated_at" - ), - "or": "(source.eq.signup_trial,plan_code.eq.signup_trial_3d)", - "order": "created_at.desc", - "limit": str(max(safe_limit * 20, 1000)), - }, - ) - active_subscription_rows = collect( - "subscriptions", - { - "select": ( - "id,user_id,plan_code,source,status,starts_at,expires_at," - "created_at,updated_at" - ), - "status": "eq.active", - "order": "created_at.desc", - "limit": str(max(safe_limit * 20, 1000)), - }, - ) - subscription_rows: List[Dict[str, Any]] = [] - seen_subscription_keys: set[str] = set() - for row in [*trial_subscription_rows, *active_subscription_rows]: - key = str(row.get("id") or "").strip() or ( - f"{row.get('user_id')}:{row.get('plan_code')}:{row.get('source')}:" - f"{row.get('starts_at')}:{row.get('expires_at')}" - ) - if key in seen_subscription_keys: - continue - seen_subscription_keys.add(key) - subscription_rows.append(row) - entitlement_trial_events = collect( - "entitlement_events", - { - "select": "id,user_id,action,payload,created_at", - "action": "in.(signup_trial_claimed,signup_trial_granted)", - "order": "created_at.desc", - "limit": str(max(safe_limit * 10, 500)), - }, - ) - - issues: List[Dict[str, Any]] = [] - stuck_intents: List[Dict[str, Any]] = [] - points_issues: List[Dict[str, Any]] = [] - - for intent in intents: - status = str(intent.get("status") or "").strip().lower() - updated_at = _parse_iso_datetime(intent.get("updated_at")) - created_at = _parse_iso_datetime(intent.get("created_at")) - expires_at = _parse_iso_datetime(intent.get("expires_at")) - age_min = ( - int((now - (updated_at or created_at or now)).total_seconds() // 60) - if (updated_at or created_at) - else 0 - ) - intent_id = str(intent.get("id") or "") - user_id = str(intent.get("user_id") or "") - metadata = intent.get("metadata") - metadata = metadata if isinstance(metadata, dict) else {} - redemption = metadata.get("points_redemption") - redemption = redemption if isinstance(redemption, dict) else {} - - if status == "submitted" and age_min >= 10: - row = { - "id": intent_id, - "user_id": user_id, - "plan_code": intent.get("plan_code"), - "status": status, - "age_min": age_min, - "tx_hash": intent.get("tx_hash"), - "updated_at": intent.get("updated_at"), - } - stuck_intents.append(row) - issues.append( - _risk_issue( - category="payment_intent", - severity="high", - title="Submitted intent 超过 10 分钟未确认", - detail=f"{intent_id} 已提交 {age_min} 分钟，可能需要补单或检查确认循环。", - user_id=user_id, - created_at=intent.get("updated_at") or intent.get("created_at"), - reference=intent_id, - payload=row, - ) - ) - elif status == "created" and expires_at and expires_at < now: - row = { - "id": intent_id, - "user_id": user_id, - "plan_code": intent.get("plan_code"), - "status": status, - "expires_at": intent.get("expires_at"), - "age_min": age_min, - } - stuck_intents.append(row) - issues.append( - _risk_issue( - category="payment_intent", - severity="medium", - title="Created intent 已过期但未关闭", - detail=f"{intent_id} 已过期，用户可能离开支付流程。", - user_id=user_id, - created_at=intent.get("created_at"), - reference=intent_id, - payload=row, - ) - ) - - if bool(redemption.get("applied")): - planned = int(redemption.get("points_to_consume") or 0) - consumed = bool(redemption.get("consumed")) - consumed_points = int(redemption.get("consumed_points") or 0) - if status == "confirmed" and not consumed: - row = { - "intent_id": intent_id, - "user_id": user_id, - "status": status, - "planned_points": planned, - "consumed_points": consumed_points, - "updated_at": intent.get("updated_at"), - } - points_issues.append(row) - issues.append( - _risk_issue( - category="points_redemption", - severity="high", - title="订单已确认但积分未扣减", - detail=f"{intent_id} 标记使用积分，但 confirmed metadata 未显示 consumed。", - user_id=user_id, - created_at=intent.get("updated_at") or intent.get("created_at"), - reference=intent_id, - payload=row, - ) - ) - elif status == "confirmed" and planned > 0 and 0 < consumed_points < planned: - row = { - "intent_id": intent_id, - "user_id": user_id, - "status": status, - "planned_points": planned, - "consumed_points": consumed_points, - "updated_at": intent.get("updated_at"), - } - points_issues.append(row) - issues.append( - _risk_issue( - category="points_redemption", - severity="medium", - title="积分抵扣只扣了部分积分", - detail=f"{intent_id} 计划扣 {planned}，实际扣 {consumed_points}。", - user_id=user_id, - created_at=intent.get("updated_at") or intent.get("created_at"), - reference=intent_id, - payload=row, - ) - ) - - reward_by_attribution = { - str(row.get("referral_attribution_id") or ""): row - for row in referral_rewards - if row.get("referral_attribution_id") is not None - } - monthly_cap_hits: List[Dict[str, Any]] = [] - referral_settlement_issues: List[Dict[str, Any]] = [] - - for attribution in referral_attributions: - status = str(attribution.get("status") or "").strip().lower() - attribution_id = str(attribution.get("id") or "") - updated_at = _parse_iso_datetime( - attribution.get("updated_at") or attribution.get("converted_at") or attribution.get("created_at") - ) - if status == "capped" and (not updated_at or updated_at >= month_start): - row = { - "id": attribution_id, - "code": attribution.get("code"), - "referrer_user_id": attribution.get("referrer_user_id"), - "referred_user_id": attribution.get("referred_user_id"), - "updated_at": attribution.get("updated_at"), - } - monthly_cap_hits.append(row) - issues.append( - _risk_issue( - category="referral", - severity="medium", - title="邀请奖励月度上限命中", - detail=f"邀请码 {attribution.get('code') or ''} 的推荐奖励已被月度上限拦截。", - user_id=attribution.get("referrer_user_id"), - created_at=attribution.get("updated_at") or attribution.get("created_at"), - reference=attribution_id, - payload=row, - ) - ) - if status == "converted" and attribution_id not in reward_by_attribution: - row = { - "id": attribution_id, - "code": attribution.get("code"), - "referrer_user_id": attribution.get("referrer_user_id"), - "referred_user_id": attribution.get("referred_user_id"), - "converted_payment_intent_id": attribution.get("converted_payment_intent_id"), - "converted_at": attribution.get("converted_at"), - } - referral_settlement_issues.append(row) - issues.append( - _risk_issue( - category="referral", - severity="high", - title="推荐已转化但没有奖励记录", - detail=f"归因 {attribution_id} 已 converted，但 referral_rewards 未找到对应记录。", - user_id=attribution.get("referrer_user_id"), - created_at=attribution.get("converted_at") or attribution.get("updated_at"), - reference=attribution_id, - payload=row, - ) - ) - - events = db.list_app_analytics_events(limit=20000, since_iso=since_dt.isoformat()) - signup_rows = [ - row - for row in events - if str(row.get("event_type") or "").strip().lower() - in {"signup_success", "signup_completed"} - ] - def normalize_user_key(value: Any) -> str: - return str(value or "").strip().lower() - - def analytics_payload(row: Dict[str, Any]) -> Dict[str, Any]: - payload = row.get("payload") - return payload if isinstance(payload, dict) else {} - - def analytics_correlation_keys(row: Dict[str, Any]) -> set[str]: - payload = analytics_payload(row) - keys: set[str] = set() - user_id = normalize_user_key(row.get("user_id") or payload.get("user_id")) - client_id = str(row.get("client_id") or "").strip() - session_id = str(row.get("session_id") or "").strip() - if user_id: - keys.add(f"user:{user_id}") - if client_id: - keys.add(f"client:{client_id}") - if session_id: - keys.add(f"session:{session_id}") - return keys - - signup_intent_keys: set[str] = set() - for row in events: - if str(row.get("event_type") or "").strip().lower() != "login_start": - continue - payload = analytics_payload(row) - mode = str(payload.get("mode") or payload.get("auth_mode") or "").strip().lower() - if mode == "signup": - signup_intent_keys.update(analytics_correlation_keys(row)) - - def has_signup_intent(row: Dict[str, Any]) -> bool: - payload = analytics_payload(row) - mode = str(payload.get("mode") or payload.get("auth_mode") or "").strip().lower() - if mode == "signup" or payload.get("signup_intent") is True: - return True - return bool(analytics_correlation_keys(row).intersection(signup_intent_keys)) - - trial_actor_keys = { - _app_analytics_actor_key(row) - for row in events - if str(row.get("event_type") or "").strip().lower() == "trial_created" - } - subscription_user_keys = { - normalize_user_key(row.get("user_id")) - for row in subscription_rows - if normalize_user_key(row.get("user_id")) - } - trial_subscription_user_keys = { - normalize_user_key(row.get("user_id")) - for row in subscription_rows - if normalize_user_key(row.get("user_id")) - and ( - str(row.get("plan_code") or "").strip().lower() == "signup_trial_3d" - or str(row.get("source") or "").strip().lower() == "signup_trial" - ) - } - trial_claim_user_keys = { - normalize_user_key(row.get("user_id")) - for row in trial_claims - if normalize_user_key(row.get("user_id")) - } - trial_event_user_keys: set[str] = set() - for row in entitlement_trial_events: - event_user_id = normalize_user_key(row.get("user_id")) - payload = row.get("payload") if isinstance(row.get("payload"), dict) else {} - payload_user_id = normalize_user_key(payload.get("user_id")) - if event_user_id: - trial_event_user_keys.add(event_user_id) - if payload_user_id: - trial_event_user_keys.add(payload_user_id) - - backend_trial_user_keys = ( - trial_subscription_user_keys | trial_claim_user_keys | trial_event_user_keys - ) - trial_gaps: List[Dict[str, Any]] = [] - - for claim in trial_claims: - claim_user_id = normalize_user_key(claim.get("user_id")) - if not claim_user_id or claim_user_id in trial_subscription_user_keys: - continue - gap = { - "claim_id": claim.get("id"), - "user_id": claim.get("user_id"), - "email": claim.get("email"), - "created_at": claim.get("created_at") or claim.get("claimed_at"), - "reason": "trial_claim_without_subscription", - } - trial_gaps.append(gap) - if len(trial_gaps) <= 20: - issues.append( - _risk_issue( - category="signup_trial", - severity="high", - title="试用 claim 已写入但订阅缺失", - detail="trial_claims 已记录该用户领取试用，但 subscriptions 中没有 signup_trial_3d 记录。", - user_id=gap.get("user_id"), - created_at=gap.get("created_at"), - reference=str(gap.get("claim_id") or ""), - payload=gap, - ) - ) - - for row in signup_rows[:300]: - if not has_signup_intent(row): - continue - actor_key = _app_analytics_actor_key(row) - if actor_key in trial_actor_keys: - continue - payload = analytics_payload(row) - signup_user_id = normalize_user_key(row.get("user_id") or payload.get("user_id")) - if not signup_user_id: - continue - if ( - signup_user_id in backend_trial_user_keys - or signup_user_id in subscription_user_keys - ): - continue - gap = { - "event_id": row.get("id"), - "actor_key": actor_key, - "user_id": signup_user_id, - "created_at": row.get("created_at"), - "reason": "signup_without_backend_trial_evidence", - } - trial_gaps.append(gap) - if len(trial_gaps) <= 20: - issues.append( - _risk_issue( - category="signup_trial", - severity="high", - title="注册成功后未发现后端试用记录", - detail=( - "该用户进入 signup_success，但没有 trial_created、trial_claims、" - "signup_trial subscription 或其他有效订阅证据。" - ), - user_id=gap.get("user_id"), - created_at=gap.get("created_at"), - reference=str(gap.get("event_id") or ""), - payload=gap, - ) - ) - - payment_incidents_raw = db.list_payment_audit_events( - limit=max(safe_limit, 500), - event_type="payment_intent_failed", - ) - grouped_payment_incidents = _group_payment_incidents(payment_incidents_raw) - unresolved_incidents = grouped_payment_incidents["incidents"] - - issues.sort(key=lambda item: str(item.get("created_at") or ""), reverse=True) - recent_rewards = [ - { - "id": row.get("id"), - "referral_attribution_id": row.get("referral_attribution_id"), - "referrer_user_id": row.get("referrer_user_id"), - "referred_user_id": row.get("referred_user_id"), - "payment_intent_id": row.get("payment_intent_id"), - "reward_points": int(row.get("reward_points") or 0), - "reward_days": int(row.get("reward_days") or 0), - "tx_hash": row.get("tx_hash"), - "explorer_url": _payment_explorer_url("polygon", row.get("tx_hash")), - "created_at": row.get("created_at"), - } - for row in referral_rewards[:safe_limit] - ] - - return { - "checked_at": _to_utc_iso(now), - "window_days": safe_days, - "summary": { - "issues": len(issues), - "stuck_intents": len(stuck_intents), - "trial_gaps": len(trial_gaps), - "payment_incidents": grouped_payment_incidents["total"], - "payment_incident_events": grouped_payment_incidents["raw_total"], - "points_discount_issues": len(points_issues), - "referral_settlement_issues": len(referral_settlement_issues), - "monthly_cap_hits": len(monthly_cap_hits), - "recent_referral_rewards": len(recent_rewards), - "recent_trial_claims": len(trial_claims), - }, - "issues": issues[:safe_limit], - "stuck_intents": stuck_intents[:safe_limit], - "trial_gaps": trial_gaps[:safe_limit], - "payment_incidents": unresolved_incidents[:safe_limit], - "points_discount_issues": points_issues[:safe_limit], - "referral_settlement_issues": referral_settlement_issues[:safe_limit], - "monthly_cap_hits": monthly_cap_hits[:safe_limit], - "recent_referral_rewards": recent_rewards, - "recent_trial_claims": trial_claims, - "query_errors": query_errors, - } - - -def grant_ops_points(request: Request, body: GrantPointsRequest) -> Dict[str, Any]: - admin = _require_ops(request) or {} - db = DBManager() - result = db.grant_points_by_supabase_email(body.email, body.points) - result["operator_email"] = admin.get("email") - if not result.get("ok"): - reason = str(result.get("reason") or "grant_points_failed") - status_code = 404 if reason == "user_not_found" else 400 - raise HTTPException(status_code=status_code, detail=result) - return result - - -def transfer_ops_points( - request: Request, - from_email: str = "", - to_email: str = "", - amount: int = 0, -) -> Dict[str, Any]: - """Transfer points from one user to another.""" - admin = _require_ops(request) or {} - from_email = str(from_email or "").strip() - to_email = str(to_email or "").strip() - amount = int(amount or 0) - if not from_email or not to_email: - raise HTTPException( - status_code=400, detail="from_email and to_email are required" - ) - if amount <= 0: - raise HTTPException(status_code=400, detail="amount must be positive") - db = DBManager() - result = db.transfer_points_by_email(from_email, to_email, amount) - result["operator_email"] = admin.get("email") - if not result.get("ok"): - raise HTTPException(status_code=400, detail=result) - return result - - -def get_ops_analytics_funnel(request: Request, days: int = 30) -> Dict[str, Any]: - _require_ops(request) - db = DBManager() - return db.get_app_analytics_funnel_summary(days=days) - - -def get_ops_truth_history( - request: Request, - city: str = "", - date_from: str = "", - date_to: str = "", - limit: int = 200, -) -> Dict[str, Any]: - _require_ops(request) - - truth_history = legacy_routes.TruthRecordRepository().load_all() - normalized_city = str(city or "").strip().lower() - normalized_from = str(date_from or "").strip() - normalized_to = str(date_to or "").strip() - max_limit = max(1, min(int(limit or 200), 1000)) - - rows = [] - for row_city, by_date in truth_history.items(): - if normalized_city and row_city != normalized_city: - continue - if not isinstance(by_date, dict): - continue - for target_date, payload in by_date.items(): - if normalized_from and str(target_date) < normalized_from: - continue - if normalized_to and str(target_date) > normalized_to: - continue - if not isinstance(payload, dict): - continue - rows.append( - { - "city": row_city, - "display_name": str( - (legacy_routes.CITY_REGISTRY.get(row_city) or {}).get("name") - or row_city - ), - "target_date": str(target_date), - "actual_high": payload.get("actual_high"), - "settlement_source": payload.get("settlement_source"), - "settlement_station_code": payload.get("settlement_station_code"), - "settlement_station_label": payload.get("settlement_station_label"), - "truth_version": payload.get("truth_version"), - "updated_by": payload.get("updated_by"), - "truth_updated_at": payload.get("truth_updated_at"), - "is_final": payload.get("is_final"), - } - ) - - rows.sort( - key=lambda item: (str(item["target_date"]), str(item["city"])), reverse=True - ) - filtered_count = len(rows) - rows = rows[:max_limit] - available_cities = [ - { - "city": city_id, - "name": str(info.get("name") or city_id), - } - for city_id, info in sorted( - legacy_routes.CITY_REGISTRY.items(), - key=lambda item: str(item[1].get("name") or item[0]), - ) - ] - return { - "items": rows, - "available_cities": available_cities, - "filters": { - "city": normalized_city or None, - "date_from": normalized_from or None, - "date_to": normalized_to or None, - "limit": max_limit, - }, - "filtered_count": filtered_count, - } - - -# ── Config ────────────────────────────────────────────────────────── - -_EDITABLE_CONFIG_KEYS: dict[str, str] = { - "POLYWEATHER_AUTH_REQUIRED": "是否强制要求 Supabase 登录访问 API", - "POLYWEATHER_PAYMENT_ENABLED": "是否启用支付功能", - "POLYWEATHER_PAYMENT_POINTS_ENABLED": "是否启用积分抵扣", - "POLYWEATHER_TELEGRAM_ALERT_PUSH_ENABLED": "是否启用 Telegram 告警推送", - "POLYWEATHER_GROUP_MEMBER_PRICE_USDC": "群成员月费 (USDC)", - "POLYWEATHER_PUBLIC_PRICE_USDC": "公开月费 (USDC)", - "POLYWEATHER_PAYMENT_POINTS_PER_USDC": "积分兑换汇率 (积分/USDC)", - "POLYWEATHER_PAYMENT_POINTS_MAX_DISCOUNT_USDC": "积分最高抵扣金额 (USDC)", - "POLYWEATHER_PAYMENT_DIRECT_RECEIVER_ADDRESS": "手动转账收款钱包地址", -} - -_SENSITIVE_CONFIG_KEYS: dict[str, dict[str, str]] = { - "POLYWEATHER_AMSC_SESSION_ID": { - "label": "AMSC AWOS sessionId", - "description": "中国跑道观测接口 sessionId，用于上海/北京/广州等 AMSC AWOS 数据源。", - }, -} - - -def get_ops_config(request: Request) -> dict[str, Any]: - _require_ops(request) - - configs: list[dict[str, Any]] = [] - for key, desc in _EDITABLE_CONFIG_KEYS.items(): - configs.append( - { - "key": key, - "value": os.getenv(key) or "", - "description": desc, - } - ) - return {"configs": configs} - - -def update_ops_config(request: Request, key: str, value: str) -> dict[str, Any]: - _require_ops(request) - - normalized_key = str(key or "").strip() - if normalized_key not in _EDITABLE_CONFIG_KEYS: - raise HTTPException( - status_code=400, detail=f"config key '{normalized_key}' is not editable" - ) - os.environ[normalized_key] = str(value) - return { - "key": normalized_key, - "value": value, - "ok": True, - } - - -def _sensitive_config_payload(key: str) -> dict[str, Any]: - definition = _SENSITIVE_CONFIG_KEYS.get(key) or {} - metadata = get_runtime_secret_status(key) - return { - "key": key, - "label": definition.get("label") or key, - "description": definition.get("description") or "", - "configured": bool(metadata.get("configured")), - "masked": str(metadata.get("masked") or ""), - "length": int(metadata.get("length") or 0), - "updated_at": str(metadata.get("updated_at") or ""), - "updated_by": str(metadata.get("updated_by") or ""), - "source": str(metadata.get("source") or "runtime_store"), - } - - -def get_ops_sensitive_config(request: Request) -> dict[str, Any]: - _require_ops(request) - return { - "configs": [ - _sensitive_config_payload(key) - for key in _SENSITIVE_CONFIG_KEYS - ] - } - - -def update_ops_sensitive_config( - request: Request, - key: str, - value: str, -) -> dict[str, Any]: - admin = _require_ops(request) or {} - normalized_key = str(key or "").strip() - if normalized_key not in _SENSITIVE_CONFIG_KEYS: - raise HTTPException( - status_code=400, - detail=f"sensitive config key '{normalized_key}' is not editable", - ) - secret_value = str(value or "").strip() - if not 12 <= len(secret_value) <= 256 or any(ch.isspace() for ch in secret_value): - raise HTTPException( - status_code=400, - detail="sessionId must be 12-256 non-whitespace characters", - ) - - db = DBManager() - try: - config = db.set_runtime_secret( - normalized_key, - secret_value, - updated_by=str(admin.get("email") or ""), - ) - except ValueError as exc: - raise HTTPException(status_code=400, detail=str(exc)) from exc - - os.environ[normalized_key] = secret_value - response_config = _sensitive_config_payload(normalized_key) - response_config.update( - { - "configured": bool(config.get("configured")), - "masked": str(config.get("masked") or ""), - "length": int(config.get("length") or 0), - "updated_at": str(config.get("updated_at") or ""), - "updated_by": str(config.get("updated_by") or ""), - "source": str(config.get("source") or "runtime_store"), - } - ) - health = ( - _check_amsc_awos_health(timeout=8) - if normalized_key == "POLYWEATHER_AMSC_SESSION_ID" - else None - ) - return {"ok": True, "config": response_config, "health": health} - - -def _build_amsc_awos_headers() -> dict[str, str]: - headers = { - "Accept": "application/json, text/plain, */*", - "Referer": os.getenv("AMSC_AWOS_REFERER", "https://www.amsc.net.cn/"), - "User-Agent": ( - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " - "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124 Safari/537.36" - ), - } - cookie = get_runtime_secret("POLYWEATHER_AMSC_COOKIE") - session_id = get_runtime_secret("POLYWEATHER_AMSC_SESSION_ID") - if session_id: - headers["sessionId"] = session_id - headers["app"] = "AMS" - elif cookie: - headers["Cookie"] = cookie - return headers - - -def _check_amsc_awos_health(timeout: int = 8) -> dict[str, Any]: - import time as _time - - from src.data_collection.amsc_awos_sources import _amsc_parse_wind_plate_payload - - amsc_base = str(os.getenv("AMSC_AWOS_BASE_URL") or "").strip() - if not amsc_base: - return {"ok": False, "error": "not configured"} - - credential_configured = bool( - get_runtime_secret("POLYWEATHER_AMSC_COOKIE") - or get_runtime_secret("POLYWEATHER_AMSC_SESSION_ID") - ) - try: - t0 = _time.perf_counter() - response = _requests.get( - f"{amsc_base}?cccc=ZSPD", - timeout=timeout, - verify=False, - headers=_build_amsc_awos_headers(), - ) - latency_ms = round((_time.perf_counter() - t0) * 1000) - try: - payload = response.json() if response.content else {} - except ValueError: - payload = {} - parsed = _amsc_parse_wind_plate_payload( - payload if isinstance(payload, dict) else {}, - city_key="shanghai", - icao="ZSPD", - ) - points = ( - ((parsed or {}).get("runway_obs") or {}).get("point_temperatures") - if isinstance(parsed, dict) - else [] - ) - point_count = len(points or []) - ok = bool(response.ok and parsed and point_count > 0) - result: dict[str, Any] = { - "ok": ok, - "status": response.status_code, - "latency_ms": latency_ms, - "credential_configured": credential_configured, - "points": point_count, - } - if isinstance(parsed, dict): - result["sample_city"] = "shanghai" - result["observation_time_local"] = parsed.get("observation_time_local") - if not ok: - result["error"] = "empty_or_unauthorized_response" - return result - except Exception as exc: - return { - "ok": False, - "credential_configured": credential_configured, - "error": str(exc)[:100], - } - - -# ── Subscriptions ─────────────────────────────────────────────────── - - -def _supabase_service_headers( - service_role_key: str, - *, - prefer: str | None = None, -) -> dict[str, str]: - headers = { - "apikey": service_role_key, - "Authorization": f"Bearer {service_role_key}", - "Content-Type": "application/json", - } - if prefer: - headers["Prefer"] = prefer - return headers - - -def _lookup_supabase_user_id_by_email( - supabase_url: str, - service_role_key: str, - email: str, -) -> str: - normalized_email = str(email or "").strip().lower() - if not normalized_email: - return "" - base = str(supabase_url or "").strip().rstrip("/") - headers = _supabase_service_headers(service_role_key) - - profile_resp = _requests.get( - f"{base}/rest/v1/profiles", - headers=headers, - params={ - "select": "id", - "email": f"eq.{normalized_email}", - "limit": "1", - }, - timeout=10, - ) - if profile_resp.ok: - profiles = profile_resp.json() if profile_resp.content else [] - if isinstance(profiles, list) and profiles: - user_id = str((profiles[0] or {}).get("id") or "").strip() - if user_id: - return user_id - - user_resp = _requests.get( - f"{base}/auth/v1/admin/users", - headers=headers, - params={"filter": f"email.eq.{normalized_email}"}, - timeout=10, - ) - users = user_resp.json().get("users", []) if user_resp.ok else [] - return str(users[0].get("id") or "").strip() if users else "" - - -def grant_ops_subscription( - request: Request, - email: str, - plan_code: str = "pro_monthly", - days: int = 30, - deduct_points: int = 0, -) -> dict[str, Any]: - _require_ops(request) - from datetime import datetime, timedelta - - supabase_url = str(os.getenv("SUPABASE_URL") or "").strip().rstrip("/") - service_role_key = str(os.getenv("SUPABASE_SERVICE_ROLE_KEY") or "").strip() - if not supabase_url or not service_role_key: - raise HTTPException(status_code=503, detail="Supabase not configured") - - allowed_plans = {"pro_monthly"} - if plan_code not in allowed_plans: - raise HTTPException( - status_code=400, detail=f"invalid plan_code, allowed: {allowed_plans}" - ) - - safe_days = max(1, min(365, int(days or 30))) - safe_deduct = max(0, int(deduct_points or 0)) - normalized_email = str(email or "").strip().lower() - if not normalized_email: - raise HTTPException(status_code=400, detail="email is required") - - user_id = _lookup_supabase_user_id_by_email( - supabase_url, - service_role_key, - normalized_email, - ) - if not user_id: - raise HTTPException( - status_code=404, detail=f"user not found: {normalized_email}" - ) - - now = datetime.utcnow() - starts_at = now.isoformat() + "Z" - expires_at = (now + timedelta(days=safe_days)).isoformat() + "Z" - - payload = { - "user_id": user_id, - "email": normalized_email, - "plan_code": plan_code, - "starts_at": starts_at, - "expires_at": expires_at, - "source": "ops_manual_grant", - "created_at": now.isoformat() + "Z", - } - - resp = _requests.post( - f"{supabase_url}/rest/v1/subscriptions", - headers=_supabase_service_headers(service_role_key, prefer="return=minimal"), - json=payload, - timeout=10, - ) - if not resp.ok: - raise HTTPException( - status_code=500, detail=f"Supabase insert failed: {resp.text[:200]}" - ) - legacy_routes.SUPABASE_ENTITLEMENT.invalidate_subscription_cache(user_id) - - result: dict[str, Any] = { - "ok": True, - "user_id": user_id, - "plan_code": plan_code, - "days": safe_days, - "expires_at": expires_at, - } - - # Optionally deduct points from the user (manual Pro grant with points payment) - if safe_deduct > 0: - db = DBManager() - deduct_result = db.deduct_points_by_supabase_email( - normalized_email, safe_deduct - ) - result["points_deducted"] = safe_deduct - result["points_result"] = deduct_result - - return result - - -def extend_ops_subscription( - request: Request, - email: str, - additional_days: int = 30, -) -> dict[str, Any]: - _require_ops(request) - from datetime import datetime, timedelta - - supabase_url = str(os.getenv("SUPABASE_URL") or "").strip().rstrip("/") - service_role_key = str(os.getenv("SUPABASE_SERVICE_ROLE_KEY") or "").strip() - if not supabase_url or not service_role_key: - raise HTTPException(status_code=503, detail="Supabase not configured") - - safe_days = max(1, min(365, int(additional_days or 30))) - normalized_email = str(email or "").strip().lower() - if not normalized_email: - raise HTTPException(status_code=400, detail="email is required") - - headers = _supabase_service_headers(service_role_key) - - user_id = _lookup_supabase_user_id_by_email( - supabase_url, - service_role_key, - normalized_email, - ) - if not user_id: - raise HTTPException( - status_code=404, detail=f"user not found: {normalized_email}" - ) - - # Find latest active subscription - subs_resp = _requests.get( - f"{supabase_url}/rest/v1/subscriptions", - headers=headers, - params={ - "select": "id,expires_at", - "user_id": f"eq.{user_id}", - "status": "eq.active", - "order": "expires_at.desc", - "limit": "1", - }, - timeout=10, - ) - subs = subs_resp.json() if subs_resp.ok else [] - if not subs: - raise HTTPException( - status_code=404, detail=f"no subscription found for {normalized_email}" - ) - - sub = subs[0] - current_expiry = sub.get("expires_at", "") - try: - dt = datetime.fromisoformat(current_expiry.replace("Z", "+00:00")) - new_expiry = (dt + timedelta(days=safe_days)).isoformat() - except Exception: - new_expiry = (datetime.utcnow() + timedelta(days=safe_days)).isoformat() + "Z" - - patch_resp = _requests.patch( - f"{supabase_url}/rest/v1/subscriptions?id=eq.{sub['id']}", - headers=_supabase_service_headers(service_role_key, prefer="return=minimal"), - json={"expires_at": new_expiry}, - timeout=10, - ) - if patch_resp.ok: - legacy_routes.SUPABASE_ENTITLEMENT.invalidate_subscription_cache(user_id) - return { - "ok": True, - "email": normalized_email, - "additional_days": safe_days, - "new_expires_at": new_expiry, - } - raise HTTPException( - status_code=500, detail=f"Supabase update failed: {patch_resp.text[:200]}" - ) - - -def get_ops_user_subscriptions( - request: Request, - email: str, -) -> dict[str, Any]: - """Return ALL subscription rows for a user (by email), regardless of status.""" - _require_ops(request) - - supabase_url = str(os.getenv("SUPABASE_URL") or "").strip().rstrip("/") - service_role_key = str(os.getenv("SUPABASE_SERVICE_ROLE_KEY") or "").strip() - if not supabase_url or not service_role_key: - raise HTTPException(status_code=503, detail="Supabase not configured") - - normalized_email = str(email or "").strip().lower() - if not normalized_email: - raise HTTPException(status_code=400, detail="email is required") - - headers = _supabase_service_headers(service_role_key) - - user_id = _lookup_supabase_user_id_by_email( - supabase_url, - service_role_key, - normalized_email, - ) - if not user_id: - raise HTTPException( - status_code=404, detail=f"user not found: {normalized_email}" - ) - - # Fetch all subscription rows for this user (no status filter) - subs_resp = _requests.get( - f"{supabase_url}/rest/v1/subscriptions", - headers=headers, - params={ - "select": "id,user_id,status,plan_code,source,starts_at,expires_at,created_at,updated_at", - "user_id": f"eq.{user_id}", - "order": "created_at.desc", - "limit": "50", - }, - timeout=10, - ) - rows = subs_resp.json() if subs_resp.ok and subs_resp.content else [] - if not isinstance(rows, list): - rows = [] - - return { - "email": normalized_email, - "user_id": user_id, - "subscriptions": rows, - "count": len(rows), - } - - -# ── Logs ──────────────────────────────────────────────────────────── - - -def get_ops_logs( - request: Request, - level: str = "", - lines: int = 100, -) -> dict[str, Any]: - _require_ops(request) - import os - import subprocess - - safe_lines = max(10, min(1000, int(lines or 100))) - log_text = "" - try: - # Read from Docker logs - result = subprocess.run( - ["docker", "logs", "--tail", str(safe_lines), "polyweather_bot"], - capture_output=True, - text=True, - timeout=10, - ) - log_text = result.stdout or result.stderr or "" - except Exception: - pass - - # Fallback to local log file if docker logs returns empty - if not log_text.strip(): - log_file = "data/logs/polyweather.log" - if os.path.exists(log_file): - try: - with open(log_file, "r", encoding="utf-8", errors="ignore") as f: - all_lines = f.readlines() - log_text = "".join(all_lines[-safe_lines:]) - except Exception: - pass - - log_lines = log_text.strip().split("\n") if log_text.strip() else [] - - if level: - level_upper = level.upper() - log_lines = [line for line in log_lines if level_upper in line.upper()] - - return { - "lines": log_lines[-safe_lines:], - "total": len(log_lines), - } - - -def _safe_source_text(value: Any) -> str: - return str(value or "").strip() - - -def _source_observed_at(source: dict[str, Any]) -> Any: - for key in ( - "observed_at", - "obs_time", - "report_time", - "observation_time", - "observation_time_local", - "time", - "timestamp", - ): - value = source.get(key) - if _safe_source_text(value): - return value - return None - - -def _source_code(source: dict[str, Any], fallback: str = "") -> str: - for key in ("source_code", "source", "provider_code", "network_provider"): - value = _safe_source_text(source.get(key)) - if value: - return canonical_observation_source_code(value) - return canonical_observation_source_code(fallback) - - -def _source_label(source: dict[str, Any], code: str, fallback: str = "") -> str: - for key in ("source_label", "station_label", "station_name", "label", "provider_label"): - value = _safe_source_text(source.get(key)) - if value: - return value - return fallback or code.upper() - - -def _source_health_entry( - *, - city: str, - role: str, - source: dict[str, Any], - fallback_code: str = "", - fallback_label: str = "", -) -> dict[str, Any] | None: - if not isinstance(source, dict) or not source: - return None - - code = _source_code(source, fallback_code) - label = _source_label(source, code, fallback_label) - observed_at = _source_observed_at(source) - freshness = source.get("freshness") if isinstance(source.get("freshness"), dict) else None - if freshness: - status = _safe_source_text(freshness.get("freshness_status")) or "unknown" - age_sec = freshness.get("age_sec") - observed_at = freshness.get("observed_at") or freshness.get("observed_at_local") or observed_at - expected_next = freshness.get("expected_next_update_at") - reason = freshness.get("freshness_reason") - else: - age_min = source.get("obs_age_min") - try: - age_min_int = int(age_min) if age_min is not None else None - except Exception: - age_min_int = None - freshness = build_observation_freshness( - source_code=code, - source_label=label, - observed_at=observed_at, - age_min=age_min_int, - ) - status = str(freshness.get("freshness_status") or "unknown") - age_sec = freshness.get("age_sec") - expected_next = freshness.get("expected_next_update_at") - reason = freshness.get("freshness_reason") - - return { - "city": city, - "role": role, - "source_code": code, - "source_label": label, - "station_code": source.get("station_code") or source.get("icao"), - "station_label": source.get("station_label") or source.get("station_name"), - "status": status, - "reason": reason, - "age_sec": age_sec, - "age_min": round(float(age_sec) / 60, 1) if isinstance(age_sec, (int, float)) else None, - "observed_at": observed_at, - "expected_next_update_at": expected_next, - "temp": source.get("temp"), - } - - -def _expected_city_source_codes(city: str, payload: dict[str, Any]) -> list[str]: - city_key = str(city or "").strip().lower().replace(" ", "") - expected: list[str] = [] - if city_key in {"ankara", "istanbul"}: - expected.append("mgm") - if city_key == "amsterdam": - expected.append("knmi") - if city_key in {"telaviv", "telavivyafo"}: - expected.append("ims") - provider = canonical_observation_source_code(payload.get("official_network_source")) - if provider and provider not in {"metar", "none"}: - expected.append(provider) - return list(dict.fromkeys(expected)) - - -def _collect_city_source_health(city: str, entry: dict[str, Any] | None) -> dict[str, Any]: - payload = (entry or {}).get("payload") if isinstance(entry, dict) else {} - if not isinstance(payload, dict): - payload = {} - - sources: list[dict[str, Any]] = [] - seen: set[tuple[str, str]] = set() - - def add(role: str, source: Any, fallback_code: str = "", fallback_label: str = "") -> None: - item = _source_health_entry( - city=city, - role=role, - source=source if isinstance(source, dict) else {}, - fallback_code=fallback_code, - fallback_label=fallback_label, - ) - if not item: - return - key = (str(item.get("role")), str(item.get("source_code"))) - if key in seen: - return - seen.add(key) - sources.append(item) - - add("settlement", payload.get("current"), fallback_label="Settlement") - add("airport_metar", payload.get("airport_current"), fallback_code="metar", fallback_label="METAR") - add("airport_primary", payload.get("airport_primary"), fallback_label="Airport station") - - official = payload.get("official") if isinstance(payload.get("official"), dict) else {} - add("official_airport_primary", official.get("airport_primary"), fallback_label="Official airport station") - add("official_airport_primary", official.get("airport_primary_current"), fallback_label="Official airport station") - - mgm = payload.get("mgm") if isinstance(payload.get("mgm"), dict) else {} - if mgm: - mgm_current = mgm.get("current") if isinstance(mgm.get("current"), dict) else {} - add( - "official_network", - { - **mgm_current, - "source_code": "mgm", - "source_label": "MGM", - "obs_time": mgm.get("obs_time") or mgm_current.get("time"), - }, - fallback_code="mgm", - fallback_label="MGM", - ) - - for nearby in payload.get("official_nearby") or payload.get("mgm_nearby") or []: - if isinstance(nearby, dict): - add("nearby_official", nearby, fallback_label="Nearby official") - - expected_codes = _expected_city_source_codes(city, payload) - present_codes = {str(item.get("source_code") or "") for item in sources} - for code in expected_codes: - if code and code not in present_codes: - sources.append( - { - "city": city, - "role": "expected_source", - "source_code": code, - "source_label": code.upper(), - "status": "missing", - "reason": "expected_source_not_present_in_cached_detail", - "age_sec": None, - "age_min": None, - "observed_at": None, - "expected_next_update_at": None, - "temp": None, - } - ) - - priority = {"stale": 4, "missing": 4, "delayed": 3, "unknown": 2, "expected_wait": 1, "fresh": 0} - worst = max(sources, key=lambda item: priority.get(str(item.get("status") or ""), 2), default=None) - full_age_sec = ( - round(max(0.0, __import__("time").time() - float(entry.get("updated_at_ts") or 0.0)), 1) - if entry - else None - ) - return { - "city": city, - "cache_exists": bool(entry), - "cache_updated_at": entry.get("updated_at") if entry else None, - "cache_age_sec": full_age_sec, - "source_count": len(sources), - "worst_status": str(worst.get("status") if worst else "missing"), - "sources": sources, - } - - -def get_ops_source_health( - request: Request, - cities: str = "", - limit: int = 80, -) -> dict[str, Any]: - _require_ops(request) - requested = legacy_routes._normalize_city_list(cities) if cities else [] - if requested: - selected = requested - else: - all_cities = getattr(legacy_routes, "CITIES", {}) - selected = list(all_cities.keys()) if isinstance(all_cities, dict) else [] - safe_limit = max(1, min(int(limit or 80), 200)) - rows = [] - for city in selected[:safe_limit]: - entry = legacy_routes._CACHE_DB.get_city_cache("full", city) - if not entry: - entry = legacy_routes._CACHE_DB.get_city_cache("panel", city) - rows.append(_collect_city_source_health(city, entry)) - - status_counts: dict[str, int] = {} - for row in rows: - for source in row.get("sources") or []: - status = str(source.get("status") or "unknown") - status_counts[status] = status_counts.get(status, 0) + 1 - - return { - "checked_at": __import__("datetime").datetime.utcnow().isoformat() + "Z", - "cities": rows, - "status_counts": status_counts, - "total_cities": len(rows), - } - - -def get_ops_observation_collector_status( - request: Request, - limit: int = 200, -) -> dict[str, Any]: - _require_ops(request) - safe_limit = max(1, min(int(limit or 200), 500)) - return ObservationCollectorStatusRepository().load_snapshot(limit=safe_limit) - - -def get_ops_health_check(request: Request) -> dict[str, Any]: - _require_ops(request) - import os - import requests as _r - import time as _time - import urllib3 as _urllib3 - - _urllib3.disable_warnings(_urllib3.exceptions.InsecureRequestWarning) - - results: dict[str, dict] = {} - timeout = 8 - - # Supabase - supabase_url = str(os.getenv("SUPABASE_URL") or "").strip().rstrip("/") - supabase_key = str(os.getenv("SUPABASE_SERVICE_ROLE_KEY") or "").strip() - if supabase_url and supabase_key: - try: - r = _r.get( - f"{supabase_url}/rest/v1/", - headers={ - "apikey": supabase_key, - "Authorization": f"Bearer {supabase_key}", - }, - timeout=timeout, - ) - results["supabase"] = { - "ok": r.ok, - "status": r.status_code, - "latency_ms": round(r.elapsed.total_seconds() * 1000), - } - except Exception as e: - results["supabase"] = {"ok": False, "error": str(e)[:100]} - else: - results["supabase"] = {"ok": False, "error": "not configured"} - - # Open-Meteo - try: - t0 = _time.perf_counter() - r = _r.get( - "https://api.open-meteo.com/v1/forecast?latitude=52.52&longitude=13.41&daily=temperature_2m_max&timezone=auto&forecast_days=1", - timeout=timeout, - ) - results["open_meteo"] = { - "ok": r.ok, - "status": r.status_code, - "latency_ms": round((_time.perf_counter() - t0) * 1000), - } - except Exception as e: - results["open_meteo"] = {"ok": False, "error": str(e)[:100]} - - # METAR (aviationweather) - try: - t0 = _time.perf_counter() - r = _r.get( - "https://aviationweather.gov/api/data/metar?ids=KJFK&format=json", - timeout=timeout, - ) - results["metar"] = { - "ok": r.ok, - "status": r.status_code, - "latency_ms": round((_time.perf_counter() - t0) * 1000), - } - except Exception as e: - results["metar"] = {"ok": False, "error": str(e)[:100]} - - # KNMI - knmi_key = str(os.getenv("KNMI_API_KEY") or "").strip() - if knmi_key: - try: - t0 = _time.perf_counter() - r = _r.get( - "https://api.dataplatform.knmi.nl/open-data/v1/datasets/10-minute-in-situ-meteorological-observations/versions/1.0/files?maxKeys=1", - headers={"Authorization": knmi_key}, - timeout=timeout, - ) - results["knmi"] = { - "ok": r.ok, - "status": r.status_code, - "latency_ms": round((_time.perf_counter() - t0) * 1000), - } - except Exception as e: - results["knmi"] = {"ok": False, "error": str(e)[:100]} - else: - results["knmi"] = {"ok": False, "error": "not configured"} - - # MADIS (NOAA) - try: - t0 = _time.perf_counter() - r = _r.head( - "https://madis-data.ncep.noaa.gov/madisPublic1/data/LDAD/hfmetar/netCDF/", - timeout=timeout, - allow_redirects=True, - ) - results["madis"] = { - "ok": r.ok, - "status": r.status_code, - "latency_ms": round((_time.perf_counter() - t0) * 1000), - } - except Exception as e: - results["madis"] = {"ok": False, "error": str(e)[:100]} - - # Telegram Bot - bot_token = str(os.getenv("TELEGRAM_BOT_TOKEN") or "").strip() - if bot_token: - try: - t0 = _time.perf_counter() - r = _r.get( - f"https://api.telegram.org/bot{bot_token}/getMe", timeout=timeout - ) - results["telegram"] = { - "ok": r.ok, - "status": r.status_code, - "latency_ms": round((_time.perf_counter() - t0) * 1000), - } - except Exception as e: - results["telegram"] = {"ok": False, "error": str(e)[:100]} - else: - results["telegram"] = {"ok": False, "error": "not configured"} - - # JMA (Japan Meteorological Agency) - try: - t0 = _time.perf_counter() - r = _r.get("https://www.jma.go.jp/bosai/forecast/", timeout=timeout) - results["jma"] = { - "ok": r.ok, - "status": r.status_code, - "latency_ms": round((_time.perf_counter() - t0) * 1000), - } - except Exception as e: - results["jma"] = {"ok": False, "error": str(e)[:100]} - - # MGM (Turkish State Meteorological Service) - try: - t0 = _time.perf_counter() - r = _r.get( - "https://servis.mgm.gov.tr/web/sondurumlar?istno=17130&_=1", - timeout=timeout, - headers={"Origin": "https://www.mgm.gov.tr"}, - ) - results["mgm"] = { - "ok": r.ok, - "status": r.status_code, - "latency_ms": round((_time.perf_counter() - t0) * 1000), - } - except Exception as e: - results["mgm"] = {"ok": False, "error": str(e)[:100]} - - # FMI (Finnish Meteorological Institute) - try: - t0 = _time.perf_counter() - r = _r.get( - "https://opendata.fmi.fi/wfs?service=WFS&version=2.0.0&request=GetCapabilities", - timeout=timeout, - ) - results["fmi"] = { - "ok": r.ok, - "status": r.status_code, - "latency_ms": round((_time.perf_counter() - t0) * 1000), - } - except Exception as e: - results["fmi"] = {"ok": False, "error": str(e)[:100]} - - # KMA (Korea Meteorological Administration) - try: - t0 = _time.perf_counter() - r = _r.get( - "https://www.weather.go.kr/wgis-nuri/js/info/sfc.geojson", timeout=timeout - ) - results["kma"] = { - "ok": r.ok, - "status": r.status_code, - "latency_ms": round((_time.perf_counter() - t0) * 1000), - } - except Exception as e: - results["kma"] = {"ok": False, "error": str(e)[:100]} - - # HKO (Hong Kong Observatory) - try: - t0 = _time.perf_counter() - r = _r.get( - "https://data.weather.gov.hk/weatherAPI/hko_data/regional-weather/latest_1min_temperature.csv", - timeout=timeout, - ) - results["hko"] = { - "ok": r.ok, - "status": r.status_code, - "latency_ms": round((_time.perf_counter() - t0) * 1000), - } - except Exception as e: - results["hko"] = {"ok": False, "error": str(e)[:100]} - - # Singapore MSS (data.gov.sg) - try: - t0 = _time.perf_counter() - r = _r.get( - "https://api.data.gov.sg/v1/environment/air-temperature", timeout=timeout - ) - results["singapore_mss"] = { - "ok": r.ok, - "status": r.status_code, - "latency_ms": round((_time.perf_counter() - t0) * 1000), - } - except Exception as e: - results["singapore_mss"] = {"ok": False, "error": str(e)[:100]} - - # CWA (Taiwan Central Weather Administration) - cwa_key = str( - os.getenv("CWA_API_KEY") - or os.getenv("CWA_OPEN_DATA_AUTH") - or os.getenv("CWA_OPEN_DATA_API_KEY") - or "" - ).strip() - if cwa_key: - try: - t0 = _time.perf_counter() - r = _r.get( - f"https://opendata.cwa.gov.tw/api/v1/rest/datastore/O-A0003-001?Authorization={cwa_key}&limit=1", - timeout=timeout, - ) - results["cwa"] = { - "ok": r.ok, - "status": r.status_code, - "latency_ms": round((_time.perf_counter() - t0) * 1000), - } - except Exception as e: - results["cwa"] = {"ok": False, "error": str(e)[:100]} - else: - results["cwa"] = {"ok": False, "error": "not configured"} - - - - # AMOS (Korea runway sensors) - try: - t0 = _time.perf_counter() - r = _r.get( - "https://global.amo.go.kr/amosobsnew/AmosRealTimeImage.do", timeout=timeout - ) - results["amos"] = { - "ok": r.ok, - "status": r.status_code, - "latency_ms": round((_time.perf_counter() - t0) * 1000), - } - except Exception as e: - results["amos"] = {"ok": False, "error": str(e)[:100]} - - # AMSC AWOS (China mainland airports) - results["amsc_awos"] = _check_amsc_awos_health(timeout=timeout) - - # NOAA WRH (US settlement verification) - try: - t0 = _time.perf_counter() - r = _r.get("https://www.weather.gov/wrh/timeseries?site=KJFK", timeout=timeout) - results["noaa_wrh"] = { - "ok": r.ok, - "status": r.status_code, - "latency_ms": round((_time.perf_counter() - t0) * 1000), - } - except Exception as e: - results["noaa_wrh"] = {"ok": False, "error": str(e)[:100]} - - all_ok = all(v.get("ok") for v in results.values()) - return { - "ok": all_ok, - "checked_at": __import__("datetime").datetime.utcnow().isoformat() + "Z", - "services": results, - } - - -def _evaluate_deb_records( - records: List[Dict[str, Any]], - *, - start_date: Optional[str] = None, - end_date: Optional[str] = None, -) -> Dict[str, Any]: - from src.analysis.settlement_rounding import apply_city_settlement - - hits = 0 - total = 0 - errors: List[float] = [] - signed_errors: List[float] = [] - cities = set() - dates = set() - for row in records: - target_date = str(row.get("target_date") or "").strip() - if start_date and target_date < start_date: - continue - if end_date and target_date > end_date: - continue - city = str(row.get("city") or "").strip().lower() - prediction = _sf(row.get("deb_prediction")) - actual = _sf(row.get("actual_high")) - if not city or not target_date or prediction is None or actual is None: - continue - try: - pred_bucket = apply_city_settlement(city, prediction) - actual_bucket = apply_city_settlement(city, actual) - except Exception: - continue - if pred_bucket is None or actual_bucket is None: - continue - total += 1 - if pred_bucket == actual_bucket: - hits += 1 - errors.append(abs(prediction - actual)) - signed_errors.append(prediction - actual) - cities.add(city) - dates.add(target_date) - - return { - "start_date": start_date, - "end_date": end_date, - "samples": total, - "hits": hits, - "hit_rate": _round_metric((hits / total * 100) if total else None, 1), - "mae": _round_metric((sum(errors) / len(errors)) if errors else None, 2), - "bias": _round_metric((sum(signed_errors) / len(signed_errors)) if signed_errors else None, 2), - "city_count": len(cities), - "date_count": len(dates), - } - - -def _build_city_deb_rows( - city_id: str, - city_rows: Dict[str, Dict[str, Any]], - today_str: str, -) -> List[Dict[str, Any]]: - rows: List[Dict[str, Any]] = [] - for target_date, record in sorted((city_rows or {}).items()): - if target_date >= today_str or not isinstance(record, dict): - continue - rows.append( - { - "city": city_id, - "target_date": target_date, - "actual_high": record.get("actual_high"), - "deb_prediction": record.get("deb_prediction"), - } - ) - return rows - - -def _deb_recent_bias_direction(bias: Optional[float]) -> str: - if bias is None: - return "unknown" - if bias <= -0.5: - return "under" - if bias >= 0.5: - return "over" - return "neutral" - - -def _build_deb_recent_trust_strategy( - recent_7d: Dict[str, Any], - recent_14d: Dict[str, Any], -) -> Dict[str, Any]: - window_key = "recent_14d" if int(recent_14d.get("samples") or 0) >= int(recent_7d.get("samples") or 0) else "recent_7d" - metrics = recent_14d if window_key == "recent_14d" else recent_7d - samples = int(metrics.get("samples") or 0) - hit_rate = _sf(metrics.get("hit_rate")) - mae = _sf(metrics.get("mae")) - bias = _sf(metrics.get("bias")) - - if samples < 3 or hit_rate is None or mae is None: - trust_tier = "insufficient" - recommendation = "insufficient" - reason = f"{window_key}: only {samples} settled DEB samples." - elif hit_rate >= 67.0 and mae <= 1.25: - trust_tier = "high" - recommendation = "primary" - reason = f"{window_key}: {hit_rate:.1f}% hit rate, MAE {mae:.2f}°." - elif hit_rate >= 34.0 and mae <= 1.75: - trust_tier = "medium" - recommendation = "supporting" - reason = f"{window_key}: {hit_rate:.1f}% hit rate, MAE {mae:.2f}°." - else: - trust_tier = "low" - recommendation = "context_only" - reason = f"{window_key}: {hit_rate:.1f}% hit rate, MAE {mae:.2f}°." - - return { - "trust_tier": trust_tier, - "recommendation": recommendation, - "bias_direction": _deb_recent_bias_direction(bias), - "reason": reason, - } - - -def _build_city_deb_recent_strategy( - city_id: str, - city_rows: Dict[str, Dict[str, Any]], - today_str: str, -) -> Optional[Dict[str, Any]]: - today_date = datetime.strptime(today_str, "%Y-%m-%d").date() - recent_7_start = (today_date - timedelta(days=7)).isoformat() - recent_14_start = (today_date - timedelta(days=14)).isoformat() - end_date = (today_date - timedelta(days=1)).isoformat() - rows = _build_city_deb_rows(city_id, city_rows, today_str) - if not rows: - return None - recent_7d = _evaluate_deb_records(rows, start_date=recent_7_start, end_date=end_date) - recent_14d = _evaluate_deb_records(rows, start_date=recent_14_start, end_date=end_date) - return { - "recent_7d": recent_7d, - "recent_14d": recent_14d, - **_build_deb_recent_trust_strategy(recent_7d, recent_14d), - } - - -def _build_city_deb_accuracy(city_id: str, city_rows: Dict[str, Dict[str, Any]], today_str: str) -> Optional[Dict[str, Any]]: - rows = _build_city_deb_rows(city_id, city_rows, today_str) - metrics = _evaluate_deb_records(rows) - if not metrics["samples"]: - return None - total = int(metrics["samples"]) - hits = int(metrics["hits"]) - mae = float(metrics["mae"] or 0.0) - hit_rate = float(metrics["hit_rate"] or 0.0) - return { - "hit_rate": hit_rate, - "mae": mae, - "total_days": total, - "hits": hits, - "details_str": f"过去{total}天 WU命中 {hits}/{total} ({hit_rate:.0f}%) | MAE: {mae:.1f}°", - } - - -def _build_city_mu_accuracy(city_id: str, city_rows: Dict[str, Dict[str, Any]], today_str: str) -> Optional[Dict[str, Any]]: - from src.analysis.settlement_rounding import apply_city_settlement - - errors: List[float] = [] - hits = 0 - total = 0 - brier_scores: List[float] = [] - for target_date, record in sorted((city_rows or {}).items()): - if target_date >= today_str or not isinstance(record, dict): - continue - actual = _sf(record.get("actual_high")) - mu_value = _sf(record.get("mu")) - if actual is None or mu_value is None: - continue - total += 1 - errors.append(abs(mu_value - actual)) - if apply_city_settlement(city_id, mu_value) == apply_city_settlement(city_id, actual): - hits += 1 - prob_snapshot = record.get("prob_snapshot") or [] - if isinstance(prob_snapshot, list): - actual_bucket = apply_city_settlement(city_id, actual) - score = 0.0 - used = False - for entry in prob_snapshot: - if not isinstance(entry, dict): - continue - predicted_p = _sf(entry.get("p")) or 0.0 - outcome = 1.0 if entry.get("v") == actual_bucket else 0.0 - score += (predicted_p - outcome) ** 2 - used = True - if used: - brier_scores.append(score) - - if not total: - return None - mae = sum(errors) / len(errors) - hit_rate = hits / total * 100 - brier = (sum(brier_scores) / len(brier_scores)) if brier_scores else None - details_parts = [ - f"μ准确率: 过去{total}天", - f"WU命中 {hits}/{total} ({hit_rate:.0f}%)", - f"MAE: {mae:.1f}°", - ] - if brier is not None: - details_parts.append(f"Brier: {brier:.3f}") - return { - "mae": mae, - "hit_rate": hit_rate, - "brier_score": brier, - "total_days": total, - "hits": hits, - "details_str": " | ".join(details_parts), - } - - -def _build_deb_historical_summary(accuracy_data: List[Dict[str, Any]]) -> Dict[str, Any]: - deb_rows = [row for row in accuracy_data if row.get("deb")] - if not deb_rows: - return { - "city_count": 0, - "avg_hit_rate": None, - "weighted_hit_rate": None, - "avg_mae": None, - "avg_days_per_city": 0, - "sample_days": 0, - "hits": 0, - } - sample_days = sum(int(row["deb"].get("total_days") or 0) for row in deb_rows) - hits = sum(int(row["deb"].get("hits") or 0) for row in deb_rows) - return { - "city_count": len(deb_rows), - "avg_hit_rate": _round_metric( - sum(float(row["deb"].get("hit_rate") or 0.0) for row in deb_rows) / len(deb_rows), - 1, - ), - "weighted_hit_rate": _round_metric((hits / sample_days * 100) if sample_days else None, 1), - "avg_mae": _round_metric( - sum(float(row["deb"].get("mae") or 0.0) for row in deb_rows) / len(deb_rows), - 2, - ), - "avg_days_per_city": round(sample_days / len(deb_rows)) if deb_rows else 0, - "sample_days": sample_days, - "hits": hits, - } - - -def _build_deb_usable_recent_summary( - accuracy_data: List[Dict[str, Any]], -) -> Dict[str, Any]: - usable_rows = [] - recommendations = {"primary": 0, "supporting": 0} - for row in accuracy_data: - recent = row.get("deb_recent") if isinstance(row.get("deb_recent"), dict) else None - if not recent: - continue - recommendation = str(recent.get("recommendation") or "").strip().lower() - if recommendation not in {"primary", "supporting"}: - continue - usable_rows.append(row) - recommendations[recommendation] += 1 - - def build_window(window_key: str) -> Dict[str, Any]: - samples = 0 - hits = 0 - weighted_mae = 0.0 - city_count = 0 - for row in usable_rows: - recent = row.get("deb_recent") if isinstance(row.get("deb_recent"), dict) else {} - metrics = recent.get(window_key) if isinstance(recent.get(window_key), dict) else {} - row_samples = int(metrics.get("samples") or 0) - if row_samples <= 0: - continue - row_hits = int(metrics.get("hits") or 0) - row_mae = _sf(metrics.get("mae")) - samples += row_samples - hits += row_hits - city_count += 1 - if row_mae is not None: - weighted_mae += row_mae * row_samples - return { - "window": window_key, - "city_count": city_count, - "samples": samples, - "hits": hits, - "hit_rate": _round_metric((hits / samples * 100) if samples else None, 1), - "avg_mae": _round_metric((weighted_mae / samples) if samples else None, 2), - "recommendations": dict(recommendations), - } - - recent_7d = build_window("recent_7d") - if int(recent_7d.get("samples") or 0) > 0: - return recent_7d - return build_window("recent_14d") - - -def _flatten_training_history(history: Dict[str, Dict[str, Dict[str, Any]]], today_str: str) -> List[Dict[str, Any]]: - rows: List[Dict[str, Any]] = [] - for city, city_rows in (history or {}).items(): - if not isinstance(city_rows, dict): - continue - for target_date, record in city_rows.items(): - if not isinstance(record, dict): - continue - target_text = str(target_date or "").strip() - if not target_text or target_text >= today_str: - continue - rows.append( - { - "city": str(city or "").strip().lower(), - "target_date": target_text, - "actual_high": record.get("actual_high"), - "deb_prediction": record.get("deb_prediction"), - "mu": record.get("mu"), - "prob_snapshot": record.get("prob_snapshot"), - } - ) - rows.sort(key=lambda row: (row["target_date"], row["city"])) - return rows - - -def _build_training_accuracy_payload( - history: Dict[str, Dict[str, Dict[str, Any]]], - city_registry: Dict[str, Dict[str, Any]], - *, - today_str: Optional[str] = None, -) -> Dict[str, Any]: - from src.analysis.deb_evaluation import backtest_deb_versions - - today = today_str or datetime.now(timezone.utc).strftime("%Y-%m-%d") - accuracy_data: List[Dict[str, Any]] = [] - for city_id, info in (city_registry or {}).items(): - city_rows = history.get(city_id) or history.get(str(city_id).strip().lower()) or {} - deb_payload = _build_city_deb_accuracy(city_id, city_rows, today) - deb_recent = _build_city_deb_recent_strategy(city_id, city_rows, today) if deb_payload else None - mu_payload = _build_city_mu_accuracy(city_id, city_rows, today) - if deb_payload or mu_payload: - accuracy_data.append( - { - "city_id": city_id, - "name": (info or {}).get("name") or city_id, - "deb": deb_payload, - "deb_recent": deb_recent, - "mu": mu_payload, - } - ) - - accuracy_data.sort( - key=lambda row: max( - row["deb"]["total_days"] if row.get("deb") else 0, - row["mu"]["total_days"] if row.get("mu") else 0, - ), - reverse=True, - ) - - all_rows = _flatten_training_history(history, today) - today_date = datetime.strptime(today, "%Y-%m-%d").date() - recent_7_start = (today_date - timedelta(days=7)).isoformat() - recent_14_start = (today_date - timedelta(days=14)).isoformat() - end_date = (today_date - timedelta(days=1)).isoformat() - version_rows = all_rows[-_DEB_VERSION_BACKTEST_SAMPLE_LIMIT:] - versions = backtest_deb_versions( - version_rows, - min_train_samples=2, - ).get("versions", {}) - - return { - "accuracy": accuracy_data, - "deb_summary": { - "historical": _build_deb_historical_summary(accuracy_data), - "usable_recent": _build_deb_usable_recent_summary(accuracy_data), - "recent_7d": _evaluate_deb_records( - all_rows, - start_date=recent_7_start, - end_date=end_date, - ), - "recent_14d": _evaluate_deb_records( - all_rows, - start_date=recent_14_start, - end_date=end_date, - ), - "versions": versions, - }, - } - - -def get_ops_training_accuracy(request: Request) -> Dict[str, Any]: - from src.analysis.deb_algorithm import load_history - from src.data_collection.city_registry import CITY_REGISTRY - - history_file = os.path.join( - os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), - "data", - "daily_records.json", - ) - history = load_history(history_file) - return _build_training_accuracy_payload(history, CITY_REGISTRY) - - -def get_ops_telegram_audit(request: Request) -> Dict[str, Any]: - _require_ops(request) - import concurrent.futures - import os - import sqlite3 - import requests - from src.database.db_manager import DBManager - from src.utils.telegram_chat_ids import get_telegram_chat_ids_from_env - - db = DBManager() - - # 1. Fetch all distinct telegram users from database - with db._get_connection() as conn: - conn.row_factory = sqlite3.Row - users_rows = conn.execute("SELECT telegram_id, username FROM users").fetchall() - bindings_rows = conn.execute( - "SELECT telegram_id, supabase_user_id, supabase_email FROM supabase_bindings" - ).fetchall() - - user_info = {} - for r in users_rows: - tid = int(r["telegram_id"]) - user_info[tid] = { - "telegram_id": tid, - "username": r["username"] or f"ID: {tid}", - "supabase_user_id": None, - "supabase_email": None, - "is_bound": False, - } - - for r in bindings_rows: - tid = int(r["telegram_id"]) - if tid not in user_info: - user_info[tid] = { - "telegram_id": tid, - "username": f"ID: {tid}", - "supabase_user_id": r["supabase_user_id"], - "supabase_email": r["supabase_email"], - "is_bound": True, - } - else: - user_info[tid]["supabase_user_id"] = r["supabase_user_id"] - user_info[tid]["supabase_email"] = r["supabase_email"] - user_info[tid]["is_bound"] = True - - # 2. Get Telegram Bot settings - bot_token = str(os.getenv("TELEGRAM_BOT_TOKEN") or "").strip() - chat_ids = get_telegram_chat_ids_from_env() - - # Add other group IDs if configured - for env_name in [ - "POLYWEATHER_TELEGRAM_GROUP_ID", - "POLYWEATHER_TELEGRAM_TOPICS_GROUP_ID", - ]: - val = str(os.getenv(env_name) or "").strip() - if val and val not in chat_ids: - chat_ids.append(val) - - if not bot_token or not chat_ids: - return { - "error": "Telegram Bot Token or Chat IDs not configured", - "anomalies": [], - } - - # 3. Check membership status for all users in parallel - results = [] - - def check_user_chat(tg_id, chat_id): - try: - resp = requests.get( - f"https://api.telegram.org/bot{bot_token}/getChatMember", - params={"chat_id": chat_id, "user_id": tg_id}, - timeout=5, - ) - if resp.status_code == 200: - data = resp.json() - if data.get("ok"): - res_status = data["result"].get("status") - return chat_id, res_status - return chat_id, None - except Exception: - return chat_id, None - - with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor: - futures = {} - for tg_id in user_info.keys(): - for c_id in chat_ids: - f = executor.submit(check_user_chat, tg_id, c_id) - futures[f] = (tg_id, c_id) - - for f in concurrent.futures.as_completed(futures): - tg_id, c_id = futures[f] - try: - _, status = f.result() - if status in {"creator", "administrator", "member"}: - results.append((tg_id, c_id, status)) - except Exception: - pass - - # 4. Filter and categorize members - anomalies = [] - valid_members = [] - - active_subs, _, used_active_window_query = _list_active_subscriptions_with_windows( - limit=5000 - ) - if not used_active_window_query: - active_subs = legacy_routes.SUPABASE_ENTITLEMENT.list_active_subscriptions( - limit=5000 - ) - active_subs_map = {} - for sub in active_subs: - uid = str(sub.get("user_id") or "").strip().lower() - if uid: - active_subs_map[uid] = sub - - for tg_id, chat_id, status in results: - info = user_info[tg_id] - - if not info["is_bound"]: - anomalies.append( - { - "telegram_id": tg_id, - "username": info["username"], - "chat_id": chat_id, - "status": status, - "anomaly_type": "unbound", - "reason": "未绑定网页账号", - "email": None, - "expires_at": None, - } - ) - else: - uid = str(info["supabase_user_id"]).strip().lower() - sub = active_subs_map.get(uid) - is_paid = False - plan_code = "" - expires_at = None - - if sub: - plan_code = str(sub.get("plan_code") or "").strip().lower() - source = str(sub.get("source") or "").strip().lower() - is_paid = "trial" not in plan_code and "trial" not in source - expires_at = sub.get("expires_at") - - if not sub: - anomalies.append( - { - "telegram_id": tg_id, - "username": info["username"], - "chat_id": chat_id, - "status": status, - "anomaly_type": "expired", - "reason": "没有有效的会员订阅", - "email": info["supabase_email"], - "expires_at": None, - } - ) - elif not is_paid: - anomalies.append( - { - "telegram_id": tg_id, - "username": info["username"], - "chat_id": chat_id, - "status": status, - "anomaly_type": "trial_only", - "reason": f"仅拥有试用会员 ({plan_code})", - "email": info["supabase_email"], - "expires_at": expires_at, - } - ) - else: - valid_members.append( - { - "telegram_id": tg_id, - "username": info["username"], - "chat_id": chat_id, - "status": status, - "email": info["supabase_email"], - "plan_code": plan_code, - "expires_at": expires_at, - } - ) - - return { - "anomalies": anomalies, - "valid_count": len(valid_members), - "anomaly_count": len(anomalies), - } +# --------------------------------------------------------------------------- +# Users / Points / Feedback / Analytics +# --------------------------------------------------------------------------- +from web.services.ops.users import ( # noqa: F401 + get_ops_analytics_funnel, + get_ops_weekly_leaderboard, + grant_ops_feedback_reward, + grant_ops_points, + list_ops_feedback, + search_ops_users, + transfer_ops_points, + update_ops_feedback_status, +) + +# --------------------------------------------------------------------------- +# Payments / Billing / Memberships +# --------------------------------------------------------------------------- +from web.services.ops.payments import ( # noqa: F401 + get_ops_billing_risk, + get_ops_memberships_growth, + get_ops_memberships_overview, + list_ops_memberships, + list_ops_payment_incidents, + list_ops_payments, + resolve_ops_payment_incident, +) + +# --------------------------------------------------------------------------- +# Health / Source Health / Training / Truth +# --------------------------------------------------------------------------- +from web.services.ops.health import ( # noqa: F401 + _build_training_accuracy_payload, + get_ops_health_check, + get_ops_observation_collector_status, + get_ops_source_health, + get_ops_training_accuracy, + get_ops_truth_history, +) + +# --------------------------------------------------------------------------- +# Config / Subscriptions / Logs / Telegram +# --------------------------------------------------------------------------- +from web.services.ops.config import ( # noqa: F401 + _lookup_supabase_user_id_by_email, + _supabase_rest_rows, + extend_ops_subscription, + get_ops_config, + get_ops_logs, + get_ops_sensitive_config, + get_ops_telegram_audit, + get_ops_user_subscriptions, + grant_ops_subscription, + update_ops_config, + update_ops_sensitive_config, +)