集成 Turnstile 人机验证 + Cloudflare R2 事件归档

2026-06-16 03:40:47 +08:00
parent 9d61d9443d
commit 382bc99f0c
8 changed files with 786 additions and 0 deletions
@@ -0,0 +1,152 @@
+"""Archive realtime observation patch events to Cloudflare R2 as JSONL.
+
+This script is intentionally read-only for Redis/SQLite. It uploads a cold
+archive object and does not delete live replay data.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sqlite3
+import sys
+from datetime import date, datetime, timedelta, timezone
+from pathlib import Path
+from typing import Any, Iterable
+
+ROOT = Path(__file__).resolve().parents[1]
+
+
+def _ensure_repo_path() -> None:
+    if str(ROOT) not in sys.path:
+        sys.path.insert(0, str(ROOT))
+
+
+def _default_sqlite_db_path() -> str:
+    _ensure_repo_path()
+    from src.database.db_manager import DBManager
+
+    return DBManager().db_path
+
+
+def _parse_date(value: str) -> date:
+    return datetime.strptime(value, "%Y-%m-%d").date()
+
+
+def _date_window(day: date) -> tuple[datetime, datetime]:
+    start = datetime(day.year, day.month, day.day, tzinfo=timezone.utc)
+    return start, start + timedelta(days=1)
+
+
+def _decode(value: Any) -> str:
+    if isinstance(value, bytes):
+        return value.decode("utf-8")
+    return str(value or "")
+
+
+def _load_sqlite_events(db_path: str, archive_day: date) -> list[dict[str, Any]]:
+    start, end = _date_window(archive_day)
+    with sqlite3.connect(db_path) as conn:
+        conn.row_factory = sqlite3.Row
+        rows = conn.execute(
+            """
+            SELECT revision, schema_type, schema_version, city, source, obs_time,
+                   payload_json, created_at
+            FROM observation_patch_events
+            WHERE created_at >= ? AND created_at < ?
+            ORDER BY revision ASC
+            """,
+            (start.isoformat(), end.isoformat()),
+        ).fetchall()
+    return [
+        {
+            "revision": int(row["revision"]),
+            "schema_type": row["schema_type"],
+            "schema_version": int(row["schema_version"]),
+            "type": f"{row['schema_type']}.v{int(row['schema_version'])}",
+            "city": row["city"],
+            "source": row["source"],
+            "obs_time": row["obs_time"],
+            "created_at": row["created_at"],
+            "payload": json.loads(row["payload_json"]),
+        }
+        for row in rows
+    ]
+
+
+def _load_redis_events(redis_url: str, stream_key: str, archive_day: date) -> list[dict[str, Any]]:
+    import redis  # type: ignore
+
+    start, end = _date_window(archive_day)
+    min_id = f"{int(start.timestamp() * 1000)}-0"
+    max_id = f"{int(end.timestamp() * 1000) - 1}-999999"
+    client = redis.Redis.from_url(redis_url)
+    rows = client.xrange(stream_key, min=min_id, max=max_id)
+    events = []
+    for stream_id, fields in rows:
+        decoded = {_decode(key): _decode(value) for key, value in dict(fields).items()}
+        payload_json = decoded.get("payload_json") or "{}"
+        events.append(
+            {
+                "stream_id": _decode(stream_id),
+                "revision": int(decoded.get("revision") or 0),
+                "type": decoded.get("type") or "",
+                "schema_type": decoded.get("schema_type") or "",
+                "schema_version": int(decoded.get("schema_version") or 0),
+                "city": decoded.get("city") or "",
+                "source": decoded.get("source") or "",
+                "obs_time": decoded.get("obs_time") or "",
+                "created_at_ms": int(decoded.get("created_at_ms") or 0),
+                "ts": int(decoded.get("ts") or 0),
+                "producer_id": decoded.get("producer_id") or "",
+                "payload": json.loads(payload_json),
+            }
+        )
+    return events
+
+
+def _build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--source", choices=["redis", "sqlite"], default=os.getenv("POLYWEATHER_R2_ARCHIVE_SOURCE", "redis"))
+    parser.add_argument("--date", default=(datetime.now(timezone.utc).date() - timedelta(days=1)).isoformat())
+    parser.add_argument("--sqlite-db", default=os.getenv("POLYWEATHER_SQLITE_DB_PATH", ""))
+    parser.add_argument("--redis-url", default=os.getenv("POLYWEATHER_REDIS_URL", "redis://127.0.0.1:6379/0"))
+    parser.add_argument("--redis-stream-key", default=os.getenv("POLYWEATHER_REDIS_STREAM_KEY", "stream:city_observation"))
+    parser.add_argument("--dry-run", action="store_true")
+    return parser
+
+
+def main(argv: Iterable[str] | None = None) -> int:
+    args = _build_parser().parse_args(list(argv) if argv is not None else None)
+    _ensure_repo_path()
+    from web.r2_archive import (
+        R2ArchiveConfig,
+        R2ObjectStore,
+        build_realtime_archive_key,
+        events_to_jsonl,
+    )
+
+    archive_day = _parse_date(args.date)
+    if args.source == "sqlite":
+        db_path = args.sqlite_db or _default_sqlite_db_path()
+        events = _load_sqlite_events(db_path, archive_day)
+    else:
+        events = _load_redis_events(args.redis_url, args.redis_stream_key, archive_day)
+
+    body = events_to_jsonl(events)
+    key = build_realtime_archive_key(args.source, archive_day.isoformat())
+    print(json.dumps({"source": args.source, "date": archive_day.isoformat(), "events": len(events), "key": key}, indent=2))
+    if args.dry_run:
+        return 0
+    config = R2ArchiveConfig.from_env()
+    if config is None:
+        print("R2 env is not configured; set POLYWEATHER_R2_* variables", file=sys.stderr)
+        return 2
+    result = R2ObjectStore(config).put_object(key, body, content_type="application/x-ndjson")
+    print(json.dumps(result, indent=2))
+    return 0 if result.get("ok") else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())