gavindiaz/PolyWeather
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Optimize terminal detail loading and Redis replay

Browse Source
This commit is contained in:
2569718930@qq.com
2026-05-31 04:32:48 +08:00
parent 2a6d8748f4
commit 46effcd45f
16 changed files with 948 additions and 80 deletions
Download Patch File Download Diff File Expand all files Collapse all files
web/redis_realtime_event_store.py
+129 -25
View File
@@ -20,6 +20,8 @@ DEFAULT_COUNTER_KEY = "counter:city_observation_revision"
DEFAULT_MAXLEN = 50000
DEFAULT_SOCKET_TIMEOUT_SECONDS = 15.0
DEFAULT_SOCKET_CONNECT_TIMEOUT_SECONDS = 5.0
DEFAULT_REPLAY_CHUNK_SIZE = 512
DEFAULT_REPLAY_SCAN_MAX = 5000
APPEND_EVENT_SCRIPT = """
local revision = redis.call('INCR', KEYS[2])
@@ -69,6 +71,14 @@ def _float_env(name: str, default: float) -> float:
return default
def _int_env(name: str, default: int, *, min_value: int, max_value: int) -> int:
try:
value = int(os.getenv(name) or default)
except (TypeError, ValueError):
value = default
return max(min_value, min(max_value, int(value)))
class RedisRealtimeEventStore:
"""Persist replayable observation patch events in a Redis Stream."""
@@ -87,6 +97,18 @@ class RedisRealtimeEventStore:
self.stream_key = stream_key or os.getenv("POLYWEATHER_REDIS_STREAM_KEY") or DEFAULT_STREAM_KEY
self.counter_key = counter_key or os.getenv("POLYWEATHER_REDIS_COUNTER_KEY") or DEFAULT_COUNTER_KEY
self.maxlen = max(1, int(maxlen or os.getenv("POLYWEATHER_REDIS_STREAM_MAXLEN") or DEFAULT_MAXLEN))
self.replay_chunk_size = _int_env(
"POLYWEATHER_REDIS_REPLAY_CHUNK_SIZE",
DEFAULT_REPLAY_CHUNK_SIZE,
min_value=50,
max_value=2000,
)
self.replay_scan_max = _int_env(
"POLYWEATHER_REDIS_REPLAY_SCAN_MAX",
DEFAULT_REPLAY_SCAN_MAX,
min_value=500,
max_value=50000,
)
self.producer_id = producer_id or os.getenv("POLYWEATHER_INSTANCE_ID") or socket.gethostname()
self._client = redis_client or self._build_client(redis_url)
self._subscriber_lock = threading.Lock()
@@ -157,7 +179,9 @@ class RedisRealtimeEventStore:
revision = _int_or_zero(_decode(value))
if revision:
return revision
return max((event["revision"] for event in self._all_events()), default=0)
rows = self._client.xrevrange(self.stream_key, max="+", min="-", count=1)
events = self._rows_to_events(rows)
return max((event["revision"] for event in events), default=0)
def status(self) -> Dict[str, Any]:
out: Dict[str, Any] = {
@@ -180,9 +204,7 @@ class RedisRealtimeEventStore:
xlen = getattr(self._client, "xlen", None)
if callable(xlen):
out["stream_len"] = int(xlen(self.stream_key))
events = self._all_events()
if events:
out["oldest_revision"] = min(int(event["revision"]) for event in events)
out["oldest_revision"] = self._oldest_revision()
except Exception as exc:
out["error"] = str(exc)
return out
@@ -197,16 +219,17 @@ class RedisRealtimeEventStore:
city_set = _normalize_city_set(cities)
since = max(0, int(since_revision or 0))
bounded_limit = max(1, min(MAX_REPLAY_LIMIT, int(limit or 1)))
replay: List[Dict[str, Any]] = []
for event in self._all_events():
if int(event.get("revision") or 0) <= since:
continue
if city_set and str(event.get("city") or "").strip().lower() not in city_set:
continue
replay.append(event)
if len(replay) >= bounded_limit:
break
return replay
if since <= 0:
return self._replay_events_forward(
city_set=city_set,
since=since,
limit=bounded_limit,
)[0]
return self._replay_events_reverse(
city_set=city_set,
since=since,
limit=bounded_limit,
)[0]
def replay_requires_resync(
self,
@@ -218,21 +241,29 @@ class RedisRealtimeEventStore:
) -> bool:
city_set = _normalize_city_set(cities)
since = max(0, int(since_revision or 0))
matching_events = [
event
for event in self._all_events()
if not city_set or str(event.get("city") or "").strip().lower() in city_set
]
if not matching_events:
return False
min_revision = min(int(event["revision"]) for event in matching_events)
if since > 0 and since < min_revision - 1:
oldest_revision = self._oldest_revision()
if since > 0 and oldest_revision and since < oldest_revision - 1:
return True
bounded_limit = max(1, int(limit or 1))
if int(replay_count or 0) < bounded_limit:
return False
return sum(1 for event in matching_events if int(event["revision"]) > since) > bounded_limit
probe_limit = min(MAX_REPLAY_LIMIT + 1, bounded_limit + 1)
if since <= 0:
probe_events, _, _ = self._replay_events_forward(
city_set=city_set,
since=since,
limit=probe_limit,
)
return len(probe_events) > bounded_limit
probe_events, hit_boundary, scanned = self._replay_events_reverse(
city_set=city_set,
since=since,
limit=probe_limit,
)
if len(probe_events) > bounded_limit:
return True
return not hit_boundary and scanned >= self.replay_scan_max
def start_live_subscription(self, callback: Callable[[Dict[str, Any]], None]) -> None:
with self._subscriber_lock:
@@ -273,8 +304,81 @@ class RedisRealtimeEventStore:
def _all_events(self) -> List[Dict[str, Any]]:
rows = self._client.xrange(self.stream_key, min="-", max="+")
return self._rows_to_events(rows)
def _oldest_revision(self) -> Optional[int]:
rows = self._client.xrange(self.stream_key, min="-", max="+", count=1)
events = self._rows_to_events(rows)
if not events:
return None
return int(events[0].get("revision") or 0) or None
def _rows_to_events(self, rows: Any) -> List[Dict[str, Any]]:
return [self._entry_to_event(entry_id, fields) for entry_id, fields in rows or []]
@staticmethod
def _matches_city(event: Dict[str, Any], city_set: Set[str]) -> bool:
return not city_set or str(event.get("city") or "").strip().lower() in city_set
def _replay_events_forward(
self,
*,
city_set: Set[str],
since: int,
limit: int,
) -> tuple[List[Dict[str, Any]], bool, int]:
replay: List[Dict[str, Any]] = []
scanned = 0
min_id = "-"
hit_boundary = False
while len(replay) < limit and scanned < self.replay_scan_max:
count = min(self.replay_chunk_size, self.replay_scan_max - scanned)
rows = self._client.xrange(self.stream_key, min=min_id, max="+", count=count)
if not rows:
hit_boundary = True
break
scanned += len(rows)
for entry_id, fields in rows:
event = self._entry_to_event(entry_id, fields)
if int(event.get("revision") or 0) <= since:
continue
if self._matches_city(event, city_set):
replay.append(event)
if len(replay) >= limit:
break
min_id = f"({_decode(rows[-1][0])}"
return replay, hit_boundary, scanned
def _replay_events_reverse(
self,
*,
city_set: Set[str],
since: int,
limit: int,
) -> tuple[List[Dict[str, Any]], bool, int]:
replay_desc: List[Dict[str, Any]] = []
scanned = 0
max_id = "+"
hit_boundary = False
while len(replay_desc) < limit and scanned < self.replay_scan_max:
count = min(self.replay_chunk_size, self.replay_scan_max - scanned)
rows = self._client.xrevrange(self.stream_key, max=max_id, min="-", count=count)
if not rows:
hit_boundary = True
break
scanned += len(rows)
for entry_id, fields in rows:
event = self._entry_to_event(entry_id, fields)
if int(event.get("revision") or 0) <= since:
hit_boundary = True
return list(reversed(replay_desc)), hit_boundary, scanned
if self._matches_city(event, city_set):
replay_desc.append(event)
if len(replay_desc) >= limit:
break
max_id = f"({_decode(rows[-1][0])}"
return list(reversed(replay_desc)), hit_boundary, scanned
@staticmethod
def _entry_to_event(entry_id: Any, fields: Dict[Any, Any]) -> Dict[str, Any]:
normalized = {_decode(key): _decode(value) for key, value in dict(fields or {}).items()}
web/routers/city.py
+22
View File
@@ -5,6 +5,7 @@ from typing import Any, Dict, List, Optional
from fastapi import APIRouter, BackgroundTasks, Query, Request
from web.services.city_api import (
get_city_detail_batch_payload,
get_city_detail_aggregate_payload,
get_city_detail_payload,
get_city_summary_payload,
@@ -104,6 +105,27 @@ async def cities_model_range(
return {"cities": rows}
@router.get("/api/cities/detail-batch")
async def city_detail_batch(
request: Request,
cities: str = "",
force_refresh: bool = False,
market_slug: Optional[str] = None,
target_date: Optional[str] = None,
resolution: Optional[str] = "10m",
limit: int = 12,
):
return await get_city_detail_batch_payload(
request,
cities=cities,
force_refresh=force_refresh,
market_slug=market_slug,
target_date=target_date,
resolution=resolution,
limit=limit,
)
@router.get("/api/city/{name}")
async def city_detail(
request: Request,
web/services/analysis_utils.py
+14 -1
View File
@@ -99,8 +99,21 @@ def add_signal(
def parse_utc_datetime(value: Any) -> Optional[datetime]:
raw = str(value or "").strip()
if not raw or "T" not in raw:
if not raw:
return None
if "T" not in raw:
try:
epoch = float(raw)
except Exception:
return None
if epoch <= 1_000_000_000:
return None
if epoch > 10_000_000_000:
epoch = epoch / 1000.0
try:
return datetime.fromtimestamp(epoch, tz=timezone.utc)
except Exception:
return None
try:
dt = datetime.fromisoformat(raw.replace("Z", "+00:00"))
except Exception:
web/services/city_api.py
+93 -1
View File
@@ -3,9 +3,10 @@
from __future__ import annotations
import os
import asyncio
import threading
import time
from typing import Any, Dict, Optional
from typing import Any, Dict, List, Optional, Tuple
from fastapi import HTTPException, Request
from fastapi.concurrency import run_in_threadpool
@@ -253,4 +254,95 @@ async def get_city_detail_aggregate_payload(
)
def _parse_batch_city_names(raw_cities: str, *, limit: int) -> List[str]:
seen = set()
out: List[str] = []
for item in str(raw_cities or "").split(","):
raw = item.strip()
if not raw:
continue
city = legacy_routes._normalize_city_or_404(raw)
if city in seen:
continue
seen.add(city)
out.append(city)
if len(out) >= limit:
break
return out
def _build_city_detail_batch_item(
city: str,
*,
force_refresh: bool,
market_slug: Optional[str],
target_date: Optional[str],
resolution: Optional[str],
) -> Tuple[str, Dict[str, Any]]:
if force_refresh:
data = legacy_routes._refresh_city_full_cache(city, True)
else:
cached_entry = legacy_routes._CACHE_DB.get_city_cache("full", city)
if cached_entry and legacy_routes._city_cache_is_fresh(
cached_entry,
legacy_routes.CITY_FULL_CACHE_TTL_SEC,
):
data = legacy_routes._overlay_latest_wunderground_current(
city,
cached_entry.get("payload") or {},
)
else:
data = legacy_routes._refresh_city_full_cache(city, False)
detail = legacy_routes._build_city_detail_payload(
data,
market_slug,
target_date,
resolution,
)
return city, detail
async def get_city_detail_batch_payload(
request: Request,
*,
cities: str,
force_refresh: bool = False,
market_slug: Optional[str] = None,
target_date: Optional[str] = None,
resolution: Optional[str] = "10m",
limit: int = 12,
) -> Dict[str, Any]:
legacy_routes._assert_entitlement(request)
city_names = _parse_batch_city_names(cities, limit=max(1, min(24, int(limit or 12))))
if not city_names:
return {"cities": [], "details": {}, "errors": {}}
tasks = [
run_in_threadpool(
_build_city_detail_batch_item,
city,
force_refresh=force_refresh,
market_slug=market_slug,
target_date=target_date,
resolution=resolution,
)
for city in city_names
]
results = await asyncio.gather(*tasks, return_exceptions=True)
details: Dict[str, Any] = {}
errors: Dict[str, str] = {}
for city, result in zip(city_names, results):
if isinstance(result, Exception):
errors[city] = str(result)
continue
result_city, payload = result
details[result_city] = payload
return {
"cities": city_names,
"details": details,
"errors": errors,
}
web/services/ops_api.py
+158 -19
View File
@@ -380,6 +380,51 @@ def get_ops_memberships_overview(
}
def _normalize_payment_incident(item: Dict[str, Any]) -> Dict[str, Any]:
payload = item.get("payload") if isinstance(item, dict) else {}
payload = payload if isinstance(payload, dict) else {}
confirm_failure = (
payload.get("confirm_failure")
if isinstance(payload.get("confirm_failure"), dict)
else {}
)
reason = str(
payload.get("reason")
or confirm_failure.get("reason")
or payload.get("error")
or "unknown"
).strip().lower()
detail = str(
payload.get("detail")
or confirm_failure.get("detail")
or payload.get("message")
or payload.get("error")
or ""
).strip()
resolved_at = str(payload.get("resolved_at") or "").strip()
return {
**item,
"payload": payload,
"reason": reason or "unknown",
"detail": detail,
"intent_id": str(
payload.get("intent_id")
or payload.get("payment_intent_id")
or confirm_failure.get("intent_id")
or ""
).strip(),
"user_id": str(payload.get("user_id") or "").strip(),
"tx_hash": str(
payload.get("tx_hash")
or confirm_failure.get("tx_hash")
or ""
).strip(),
"resolved": bool(resolved_at),
"resolved_at": resolved_at,
"resolved_by": str(payload.get("resolved_by") or "").strip(),
}
def list_ops_payment_incidents(
request: Request,
limit: int = 50,
@@ -395,16 +440,15 @@ def list_ops_payment_incidents(
normalized_reason = str(reason or "").strip().lower()
filtered = []
for item in incidents:
payload = item.get("payload") if isinstance(item, dict) else {}
payload = payload if isinstance(payload, dict) else {}
item_reason = str(payload.get("reason") or "").strip().lower()
resolved_at = str(payload.get("resolved_at") or "").strip()
normalized_item = _normalize_payment_incident(item)
item_reason = str(normalized_item.get("reason") or "").strip().lower()
resolved = bool(normalized_item.get("resolved"))
if normalized_reason and item_reason != normalized_reason:
continue
if not include_resolved and resolved_at:
if not include_resolved and resolved:
continue
filtered.append(item)
return {"incidents": filtered}
filtered.append(normalized_item)
return {"incidents": filtered, "total": len(filtered)}
def resolve_ops_payment_incident(request: Request, event_id: int) -> Dict[str, Any]:
@@ -498,7 +542,28 @@ def get_ops_billing_risk(
{
"select": "id,user_id,email,telegram_user_id,claimed_at,created_at",
"order": "created_at.desc",
"limit": str(min(safe_limit, 80)),
"limit": str(max(safe_limit * 10, 500)),
},
)
subscription_rows = collect(
"subscriptions",
{
"select": (
"id,user_id,plan_code,source,status,starts_at,expires_at,"
"created_at,updated_at"
),
"or": "(source.eq.signup_trial,plan_code.eq.signup_trial_3d,status.eq.active)",
"order": "created_at.desc",
"limit": str(max(safe_limit * 20, 1000)),
},
)
entitlement_trial_events = collect(
"entitlement_events",
{
"select": "id,user_id,action,payload,created_at",
"action": "in.(signup_trial_claimed,signup_trial_granted)",
"order": "created_at.desc",
"limit": str(max(safe_limit * 10, 500)),
},
)
@@ -683,22 +748,58 @@ def get_ops_billing_risk(
if str(row.get("event_type") or "").strip().lower()
in {"signup_success", "signup_completed"}
]
def normalize_user_key(value: Any) -> str:
return str(value or "").strip().lower()
trial_actor_keys = {
_app_analytics_actor_key(row)
for row in events
if str(row.get("event_type") or "").strip().lower() == "trial_created"
}
trial_gaps: List[Dict[str, Any]] = []
for row in signup_rows[:300]:
actor_key = _app_analytics_actor_key(row)
if actor_key in trial_actor_keys:
continue
subscription_user_keys = {
normalize_user_key(row.get("user_id"))
for row in subscription_rows
if normalize_user_key(row.get("user_id"))
}
trial_subscription_user_keys = {
normalize_user_key(row.get("user_id"))
for row in subscription_rows
if normalize_user_key(row.get("user_id"))
and (
str(row.get("plan_code") or "").strip().lower() == "signup_trial_3d"
or str(row.get("source") or "").strip().lower() == "signup_trial"
)
}
trial_claim_user_keys = {
normalize_user_key(row.get("user_id"))
for row in trial_claims
if normalize_user_key(row.get("user_id"))
}
trial_event_user_keys: set[str] = set()
for row in entitlement_trial_events:
event_user_id = normalize_user_key(row.get("user_id"))
payload = row.get("payload") if isinstance(row.get("payload"), dict) else {}
payload_user_id = normalize_user_key(payload.get("user_id"))
if event_user_id:
trial_event_user_keys.add(event_user_id)
if payload_user_id:
trial_event_user_keys.add(payload_user_id)
backend_trial_user_keys = (
trial_subscription_user_keys | trial_claim_user_keys | trial_event_user_keys
)
trial_gaps: List[Dict[str, Any]] = []
for claim in trial_claims:
claim_user_id = normalize_user_key(claim.get("user_id"))
if not claim_user_id or claim_user_id in trial_subscription_user_keys:
continue
gap = {
"event_id": row.get("id"),
"actor_key": actor_key,
"user_id": row.get("user_id") or payload.get("user_id"),
"created_at": row.get("created_at"),
"claim_id": claim.get("id"),
"user_id": claim.get("user_id"),
"email": claim.get("email"),
"created_at": claim.get("created_at") or claim.get("claimed_at"),
"reason": "trial_claim_without_subscription",
}
trial_gaps.append(gap)
if len(trial_gaps) <= 20:
@@ -706,8 +807,46 @@ def get_ops_billing_risk(
_risk_issue(
category="signup_trial",
severity="high",
title="注册成功后未记录试用开通",
detail="该用户进入 signup_success，但同窗口内没有 trial_created 事件。",
title="试用 claim 已写入但订阅缺失",
detail="trial_claims 已记录该用户领取试用，但 subscriptions 中没有 signup_trial_3d 记录。",
user_id=gap.get("user_id"),
created_at=gap.get("created_at"),
reference=str(gap.get("claim_id") or ""),
payload=gap,
)
)
for row in signup_rows[:300]:
actor_key = _app_analytics_actor_key(row)
if actor_key in trial_actor_keys:
continue
payload = row.get("payload") if isinstance(row.get("payload"), dict) else {}
signup_user_id = normalize_user_key(row.get("user_id") or payload.get("user_id"))
if not signup_user_id:
continue
if (
signup_user_id in backend_trial_user_keys
or signup_user_id in subscription_user_keys
):
continue
gap = {
"event_id": row.get("id"),
"actor_key": actor_key,
"user_id": signup_user_id,
"created_at": row.get("created_at"),
"reason": "signup_without_backend_trial_evidence",
}
trial_gaps.append(gap)
if len(trial_gaps) <= 20:
issues.append(
_risk_issue(
category="signup_trial",
severity="high",
title="注册成功后未发现后端试用记录",
detail=(
"该用户进入 signup_success，但没有 trial_created、trial_claims、"
"signup_trial subscription 或其他有效订阅证据。"
),
user_id=gap.get("user_id"),
created_at=gap.get("created_at"),
reference=str(gap.get("event_id") or ""),