集成 Turnstile 人机验证 + Cloudflare R2 事件归档
This commit is contained in:
@@ -0,0 +1,93 @@
|
|||||||
|
import fs from "node:fs";
|
||||||
|
import path from "node:path";
|
||||||
|
|
||||||
|
function assert(condition: unknown, message: string) {
|
||||||
|
if (!condition) throw new Error(message);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function runTests() {
|
||||||
|
const projectRoot = process.cwd();
|
||||||
|
const turnstileLibPath = path.join(projectRoot, "lib", "turnstile.ts");
|
||||||
|
const widgetPath = path.join(projectRoot, "components", "security", "TurnstileWidget.tsx");
|
||||||
|
const loginPath = path.join(projectRoot, "components", "auth", "LoginClient.tsx");
|
||||||
|
const feedbackModalPath = path.join(
|
||||||
|
projectRoot,
|
||||||
|
"components",
|
||||||
|
"dashboard",
|
||||||
|
"scan-terminal",
|
||||||
|
"UserFeedbackModal.tsx",
|
||||||
|
);
|
||||||
|
const feedbackRoutePath = path.join(projectRoot, "app", "api", "feedback", "route.ts");
|
||||||
|
const paymentIntentRoutePath = path.join(projectRoot, "app", "api", "payments", "intents", "route.ts");
|
||||||
|
const paymentSubmitRoutePath = path.join(
|
||||||
|
projectRoot,
|
||||||
|
"app",
|
||||||
|
"api",
|
||||||
|
"payments",
|
||||||
|
"intents",
|
||||||
|
"[intentId]",
|
||||||
|
"submit",
|
||||||
|
"route.ts",
|
||||||
|
);
|
||||||
|
const paymentFlowPath = path.join(projectRoot, "components", "account", "usePaymentFlow.ts");
|
||||||
|
|
||||||
|
assert(fs.existsSync(turnstileLibPath), "shared Turnstile verifier module must exist");
|
||||||
|
assert(fs.existsSync(widgetPath), "Turnstile widget component must exist");
|
||||||
|
|
||||||
|
const turnstileLib = fs.readFileSync(turnstileLibPath, "utf8");
|
||||||
|
const widgetSource = fs.readFileSync(widgetPath, "utf8");
|
||||||
|
const loginSource = fs.readFileSync(loginPath, "utf8");
|
||||||
|
const feedbackModalSource = fs.readFileSync(feedbackModalPath, "utf8");
|
||||||
|
const feedbackRouteSource = fs.readFileSync(feedbackRoutePath, "utf8");
|
||||||
|
const paymentIntentRouteSource = fs.readFileSync(paymentIntentRoutePath, "utf8");
|
||||||
|
const paymentSubmitRouteSource = fs.readFileSync(paymentSubmitRoutePath, "utf8");
|
||||||
|
const paymentFlowSource = fs.readFileSync(paymentFlowPath, "utf8");
|
||||||
|
|
||||||
|
assert(
|
||||||
|
turnstileLib.includes("POLYWEATHER_TURNSTILE_SECRET_KEY") &&
|
||||||
|
turnstileLib.includes("NEXT_PUBLIC_TURNSTILE_SITE_KEY") &&
|
||||||
|
turnstileLib.includes("https://challenges.cloudflare.com/turnstile/v0/siteverify") &&
|
||||||
|
turnstileLib.includes("requireTurnstileForRequest") &&
|
||||||
|
turnstileLib.includes("POLYWEATHER_TURNSTILE_BYPASS"),
|
||||||
|
"Turnstile verifier must be optional, use Cloudflare siteverify, and expose a request guard",
|
||||||
|
);
|
||||||
|
assert(
|
||||||
|
widgetSource.includes("NEXT_PUBLIC_TURNSTILE_SITE_KEY") &&
|
||||||
|
widgetSource.includes("challenges.cloudflare.com/turnstile/v0/api.js") &&
|
||||||
|
widgetSource.includes("window.turnstile.render") &&
|
||||||
|
widgetSource.includes("data-turnstile-action"),
|
||||||
|
"Turnstile widget must render Cloudflare Turnstile only when a site key is configured",
|
||||||
|
);
|
||||||
|
assert(
|
||||||
|
loginSource.includes("TurnstileWidget") &&
|
||||||
|
loginSource.includes("getTurnstileTokenForAction") &&
|
||||||
|
loginSource.includes("captchaToken") &&
|
||||||
|
loginSource.includes("signInWithPassword") &&
|
||||||
|
loginSource.includes("signUp"),
|
||||||
|
"login/signup must attach a Turnstile captchaToken to Supabase auth calls",
|
||||||
|
);
|
||||||
|
assert(
|
||||||
|
feedbackModalSource.includes("TurnstileWidget") &&
|
||||||
|
feedbackModalSource.includes("getTurnstileTokenForAction") &&
|
||||||
|
feedbackModalSource.includes("turnstile_token"),
|
||||||
|
"feedback modal must include a Turnstile token in feedback submissions",
|
||||||
|
);
|
||||||
|
assert(
|
||||||
|
feedbackRouteSource.includes('requireTurnstileForRequest(req, "feedback_submit"') &&
|
||||||
|
feedbackRouteSource.indexOf('requireTurnstileForRequest(req, "feedback_submit"') <
|
||||||
|
feedbackRouteSource.indexOf("fetch(`${API_BASE}/api/feedback`"),
|
||||||
|
"feedback POST proxy must verify Turnstile before forwarding user feedback",
|
||||||
|
);
|
||||||
|
assert(
|
||||||
|
paymentIntentRouteSource.includes('requireTurnstileForRequest(req, "payment_intent_create"') &&
|
||||||
|
paymentSubmitRouteSource.includes('requireTurnstileForRequest(req, "payment_tx_submit"'),
|
||||||
|
"payment mutation proxies must verify Turnstile before forwarding create and submit requests",
|
||||||
|
);
|
||||||
|
assert(
|
||||||
|
paymentFlowSource.includes("getTurnstileTokenForAction") &&
|
||||||
|
paymentFlowSource.includes("turnstile_token") &&
|
||||||
|
paymentFlowSource.includes("payment_intent_create") &&
|
||||||
|
paymentFlowSource.includes("payment_tx_submit"),
|
||||||
|
"payment flow must include Turnstile tokens in intent creation and tx submit payloads",
|
||||||
|
);
|
||||||
|
}
|
||||||
@@ -0,0 +1,128 @@
|
|||||||
|
"use client";
|
||||||
|
|
||||||
|
import { useEffect, useRef, useState } from "react";
|
||||||
|
import { getConfiguredTurnstileSiteKey, isTurnstileEnabled } from "@/lib/turnstile-client";
|
||||||
|
|
||||||
|
const TURNSTILE_SCRIPT_ID = "polyweather-turnstile-script";
|
||||||
|
const TURNSTILE_SCRIPT_SRC = "https://challenges.cloudflare.com/turnstile/v0/api.js?render=explicit";
|
||||||
|
const TURNSTILE_SITE_KEY_ENV = "NEXT_PUBLIC_TURNSTILE_SITE_KEY";
|
||||||
|
|
||||||
|
type TurnstileRenderOptions = {
|
||||||
|
sitekey: string;
|
||||||
|
action: string;
|
||||||
|
callback: (token: string) => void;
|
||||||
|
"expired-callback": () => void;
|
||||||
|
"error-callback": () => void;
|
||||||
|
theme?: "light" | "dark" | "auto";
|
||||||
|
size?: "normal" | "compact" | "flexible";
|
||||||
|
};
|
||||||
|
|
||||||
|
declare global {
|
||||||
|
interface Window {
|
||||||
|
turnstile?: {
|
||||||
|
render: (container: HTMLElement, options: TurnstileRenderOptions) => string;
|
||||||
|
reset: (widgetId?: string) => void;
|
||||||
|
remove: (widgetId?: string) => void;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function ensureTurnstileScript(onReady: () => void, onError: () => void) {
|
||||||
|
if (typeof window === "undefined") return;
|
||||||
|
if (window.turnstile) {
|
||||||
|
onReady();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const existing = document.getElementById(TURNSTILE_SCRIPT_ID) as HTMLScriptElement | null;
|
||||||
|
if (existing) {
|
||||||
|
existing.addEventListener("load", onReady, { once: true });
|
||||||
|
existing.addEventListener("error", onError, { once: true });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const script = document.createElement("script");
|
||||||
|
script.id = TURNSTILE_SCRIPT_ID;
|
||||||
|
script.src = TURNSTILE_SCRIPT_SRC;
|
||||||
|
script.async = true;
|
||||||
|
script.defer = true;
|
||||||
|
script.addEventListener("load", onReady, { once: true });
|
||||||
|
script.addEventListener("error", onError, { once: true });
|
||||||
|
document.head.appendChild(script);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function TurnstileWidget({
|
||||||
|
action,
|
||||||
|
onToken,
|
||||||
|
onError,
|
||||||
|
resetKey,
|
||||||
|
theme = "light",
|
||||||
|
size = "normal",
|
||||||
|
}: {
|
||||||
|
action: string;
|
||||||
|
onToken: (token: string) => void;
|
||||||
|
onError?: () => void;
|
||||||
|
resetKey?: string | number;
|
||||||
|
theme?: "light" | "dark" | "auto";
|
||||||
|
size?: "normal" | "compact" | "flexible";
|
||||||
|
}) {
|
||||||
|
const containerRef = useRef<HTMLDivElement | null>(null);
|
||||||
|
const widgetIdRef = useRef<string>("");
|
||||||
|
const [scriptReady, setScriptReady] = useState(false);
|
||||||
|
const siteKey = getConfiguredTurnstileSiteKey();
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
if (!isTurnstileEnabled()) return;
|
||||||
|
ensureTurnstileScript(
|
||||||
|
() => setScriptReady(true),
|
||||||
|
() => {
|
||||||
|
onToken("");
|
||||||
|
onError?.();
|
||||||
|
},
|
||||||
|
);
|
||||||
|
}, [onError, onToken]);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
if (!siteKey || !scriptReady || !containerRef.current || !window.turnstile) return;
|
||||||
|
if (widgetIdRef.current) {
|
||||||
|
window.turnstile.remove(widgetIdRef.current);
|
||||||
|
widgetIdRef.current = "";
|
||||||
|
}
|
||||||
|
widgetIdRef.current = window.turnstile.render(containerRef.current, {
|
||||||
|
sitekey: siteKey,
|
||||||
|
action,
|
||||||
|
theme,
|
||||||
|
size,
|
||||||
|
callback: onToken,
|
||||||
|
"expired-callback": () => onToken(""),
|
||||||
|
"error-callback": () => {
|
||||||
|
onToken("");
|
||||||
|
onError?.();
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
return () => {
|
||||||
|
if (widgetIdRef.current && window.turnstile) {
|
||||||
|
window.turnstile.remove(widgetIdRef.current);
|
||||||
|
widgetIdRef.current = "";
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}, [action, onError, onToken, scriptReady, siteKey, size, theme]);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
if (!resetKey || !widgetIdRef.current || !window.turnstile) return;
|
||||||
|
onToken("");
|
||||||
|
window.turnstile.reset(widgetIdRef.current);
|
||||||
|
}, [onToken, resetKey]);
|
||||||
|
|
||||||
|
if (!siteKey) return null;
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div
|
||||||
|
ref={containerRef}
|
||||||
|
data-turnstile-action={action}
|
||||||
|
data-turnstile-site-key-env={TURNSTILE_SITE_KEY_ENV}
|
||||||
|
className="min-h-[65px] w-full overflow-hidden"
|
||||||
|
/>
|
||||||
|
);
|
||||||
|
}
|
||||||
@@ -0,0 +1,23 @@
|
|||||||
|
export function getConfiguredTurnstileSiteKey() {
|
||||||
|
return String(process.env.NEXT_PUBLIC_TURNSTILE_SITE_KEY || "").trim();
|
||||||
|
}
|
||||||
|
|
||||||
|
export function isTurnstileEnabled() {
|
||||||
|
return Boolean(getConfiguredTurnstileSiteKey());
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getTurnstileTokenForAction(
|
||||||
|
token: string | null | undefined,
|
||||||
|
action: string,
|
||||||
|
options?: { message?: string },
|
||||||
|
) {
|
||||||
|
if (!isTurnstileEnabled()) return undefined;
|
||||||
|
const normalized = String(token || "").trim();
|
||||||
|
if (!normalized) {
|
||||||
|
throw new Error(
|
||||||
|
options?.message ||
|
||||||
|
`Complete the security check before continuing (${action}).`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return normalized;
|
||||||
|
}
|
||||||
@@ -0,0 +1,121 @@
|
|||||||
|
import { NextRequest, NextResponse } from "next/server";
|
||||||
|
|
||||||
|
const TURNSTILE_VERIFY_URL = "https://challenges.cloudflare.com/turnstile/v0/siteverify";
|
||||||
|
const NO_STORE = "no-store, max-age=0";
|
||||||
|
|
||||||
|
type TurnstileVerification = {
|
||||||
|
ok: boolean;
|
||||||
|
skipped?: boolean;
|
||||||
|
error?: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
function turnstileSecretKey() {
|
||||||
|
return String(process.env.POLYWEATHER_TURNSTILE_SECRET_KEY || "").trim();
|
||||||
|
}
|
||||||
|
|
||||||
|
function turnstileSiteKey() {
|
||||||
|
return String(process.env.NEXT_PUBLIC_TURNSTILE_SITE_KEY || "").trim();
|
||||||
|
}
|
||||||
|
|
||||||
|
export function isTurnstileServerEnabled() {
|
||||||
|
if (process.env.POLYWEATHER_TURNSTILE_BYPASS === "true") return false;
|
||||||
|
return Boolean(turnstileSecretKey() && turnstileSiteKey());
|
||||||
|
}
|
||||||
|
|
||||||
|
function enforceTurnstileAction() {
|
||||||
|
return process.env.POLYWEATHER_TURNSTILE_ENFORCE_ACTION === "true";
|
||||||
|
}
|
||||||
|
|
||||||
|
export function stripTurnstileToken<T>(body: T): T {
|
||||||
|
if (!body || typeof body !== "object" || Array.isArray(body)) return body;
|
||||||
|
const copy = { ...(body as Record<string, unknown>) };
|
||||||
|
delete copy.turnstile_token;
|
||||||
|
delete copy.turnstileToken;
|
||||||
|
return copy as T;
|
||||||
|
}
|
||||||
|
|
||||||
|
function tokenFromBody(body: unknown) {
|
||||||
|
if (!body || typeof body !== "object") return "";
|
||||||
|
const record = body as Record<string, unknown>;
|
||||||
|
return String(record.turnstile_token || record.turnstileToken || "").trim();
|
||||||
|
}
|
||||||
|
|
||||||
|
function remoteIpFromRequest(req: NextRequest) {
|
||||||
|
const forwardedFor = req.headers.get("x-forwarded-for") || "";
|
||||||
|
return (
|
||||||
|
req.headers.get("cf-connecting-ip") ||
|
||||||
|
forwardedFor.split(",")[0]?.trim() ||
|
||||||
|
req.headers.get("x-real-ip") ||
|
||||||
|
""
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function verifyTurnstileToken(
|
||||||
|
token: string,
|
||||||
|
req: NextRequest,
|
||||||
|
action: string,
|
||||||
|
): Promise<TurnstileVerification> {
|
||||||
|
if (!isTurnstileServerEnabled()) {
|
||||||
|
return { ok: true, skipped: true };
|
||||||
|
}
|
||||||
|
|
||||||
|
const normalizedToken = String(token || "").trim();
|
||||||
|
if (!normalizedToken) {
|
||||||
|
return { ok: false, error: "missing_turnstile_token" };
|
||||||
|
}
|
||||||
|
|
||||||
|
const body = new URLSearchParams();
|
||||||
|
body.set("secret", turnstileSecretKey());
|
||||||
|
body.set("response", normalizedToken);
|
||||||
|
const remoteIp = remoteIpFromRequest(req);
|
||||||
|
if (remoteIp) body.set("remoteip", remoteIp);
|
||||||
|
|
||||||
|
try {
|
||||||
|
const response = await fetch(TURNSTILE_VERIFY_URL, {
|
||||||
|
method: "POST",
|
||||||
|
headers: {
|
||||||
|
"Content-Type": "application/x-www-form-urlencoded",
|
||||||
|
},
|
||||||
|
body,
|
||||||
|
cache: "no-store",
|
||||||
|
});
|
||||||
|
const data = (await response.json().catch(() => null)) as {
|
||||||
|
success?: unknown;
|
||||||
|
action?: unknown;
|
||||||
|
"error-codes"?: unknown;
|
||||||
|
} | null;
|
||||||
|
if (!response.ok || data?.success !== true) {
|
||||||
|
return { ok: false, error: "turnstile_siteverify_failed" };
|
||||||
|
}
|
||||||
|
const returnedAction = String(data.action || "").trim();
|
||||||
|
if (enforceTurnstileAction() && returnedAction && returnedAction !== action) {
|
||||||
|
return { ok: false, error: "turnstile_action_mismatch" };
|
||||||
|
}
|
||||||
|
return { ok: true };
|
||||||
|
} catch {
|
||||||
|
return { ok: false, error: "turnstile_siteverify_unavailable" };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function requireTurnstileForRequest(
|
||||||
|
req: NextRequest,
|
||||||
|
action: string,
|
||||||
|
body?: unknown,
|
||||||
|
) {
|
||||||
|
const token = tokenFromBody(body);
|
||||||
|
const result = await verifyTurnstileToken(token, req, action);
|
||||||
|
if (result.ok) return null;
|
||||||
|
return NextResponse.json(
|
||||||
|
{
|
||||||
|
error: "Security verification failed. Please refresh the challenge and retry.",
|
||||||
|
code: result.error || "turnstile_failed",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
status: 403,
|
||||||
|
headers: {
|
||||||
|
"Cache-Control": NO_STORE,
|
||||||
|
"Cloudflare-CDN-Cache-Control": NO_STORE,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
);
|
||||||
|
}
|
||||||
@@ -0,0 +1,152 @@
|
|||||||
|
"""Archive realtime observation patch events to Cloudflare R2 as JSONL.
|
||||||
|
|
||||||
|
This script is intentionally read-only for Redis/SQLite. It uploads a cold
|
||||||
|
archive object and does not delete live replay data.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import sqlite3
|
||||||
|
import sys
|
||||||
|
from datetime import date, datetime, timedelta, timezone
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any, Iterable
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
|
||||||
|
|
||||||
|
def _ensure_repo_path() -> None:
|
||||||
|
if str(ROOT) not in sys.path:
|
||||||
|
sys.path.insert(0, str(ROOT))
|
||||||
|
|
||||||
|
|
||||||
|
def _default_sqlite_db_path() -> str:
|
||||||
|
_ensure_repo_path()
|
||||||
|
from src.database.db_manager import DBManager
|
||||||
|
|
||||||
|
return DBManager().db_path
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_date(value: str) -> date:
|
||||||
|
return datetime.strptime(value, "%Y-%m-%d").date()
|
||||||
|
|
||||||
|
|
||||||
|
def _date_window(day: date) -> tuple[datetime, datetime]:
|
||||||
|
start = datetime(day.year, day.month, day.day, tzinfo=timezone.utc)
|
||||||
|
return start, start + timedelta(days=1)
|
||||||
|
|
||||||
|
|
||||||
|
def _decode(value: Any) -> str:
|
||||||
|
if isinstance(value, bytes):
|
||||||
|
return value.decode("utf-8")
|
||||||
|
return str(value or "")
|
||||||
|
|
||||||
|
|
||||||
|
def _load_sqlite_events(db_path: str, archive_day: date) -> list[dict[str, Any]]:
|
||||||
|
start, end = _date_window(archive_day)
|
||||||
|
with sqlite3.connect(db_path) as conn:
|
||||||
|
conn.row_factory = sqlite3.Row
|
||||||
|
rows = conn.execute(
|
||||||
|
"""
|
||||||
|
SELECT revision, schema_type, schema_version, city, source, obs_time,
|
||||||
|
payload_json, created_at
|
||||||
|
FROM observation_patch_events
|
||||||
|
WHERE created_at >= ? AND created_at < ?
|
||||||
|
ORDER BY revision ASC
|
||||||
|
""",
|
||||||
|
(start.isoformat(), end.isoformat()),
|
||||||
|
).fetchall()
|
||||||
|
return [
|
||||||
|
{
|
||||||
|
"revision": int(row["revision"]),
|
||||||
|
"schema_type": row["schema_type"],
|
||||||
|
"schema_version": int(row["schema_version"]),
|
||||||
|
"type": f"{row['schema_type']}.v{int(row['schema_version'])}",
|
||||||
|
"city": row["city"],
|
||||||
|
"source": row["source"],
|
||||||
|
"obs_time": row["obs_time"],
|
||||||
|
"created_at": row["created_at"],
|
||||||
|
"payload": json.loads(row["payload_json"]),
|
||||||
|
}
|
||||||
|
for row in rows
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def _load_redis_events(redis_url: str, stream_key: str, archive_day: date) -> list[dict[str, Any]]:
|
||||||
|
import redis # type: ignore
|
||||||
|
|
||||||
|
start, end = _date_window(archive_day)
|
||||||
|
min_id = f"{int(start.timestamp() * 1000)}-0"
|
||||||
|
max_id = f"{int(end.timestamp() * 1000) - 1}-999999"
|
||||||
|
client = redis.Redis.from_url(redis_url)
|
||||||
|
rows = client.xrange(stream_key, min=min_id, max=max_id)
|
||||||
|
events = []
|
||||||
|
for stream_id, fields in rows:
|
||||||
|
decoded = {_decode(key): _decode(value) for key, value in dict(fields).items()}
|
||||||
|
payload_json = decoded.get("payload_json") or "{}"
|
||||||
|
events.append(
|
||||||
|
{
|
||||||
|
"stream_id": _decode(stream_id),
|
||||||
|
"revision": int(decoded.get("revision") or 0),
|
||||||
|
"type": decoded.get("type") or "",
|
||||||
|
"schema_type": decoded.get("schema_type") or "",
|
||||||
|
"schema_version": int(decoded.get("schema_version") or 0),
|
||||||
|
"city": decoded.get("city") or "",
|
||||||
|
"source": decoded.get("source") or "",
|
||||||
|
"obs_time": decoded.get("obs_time") or "",
|
||||||
|
"created_at_ms": int(decoded.get("created_at_ms") or 0),
|
||||||
|
"ts": int(decoded.get("ts") or 0),
|
||||||
|
"producer_id": decoded.get("producer_id") or "",
|
||||||
|
"payload": json.loads(payload_json),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
return events
|
||||||
|
|
||||||
|
|
||||||
|
def _build_parser() -> argparse.ArgumentParser:
|
||||||
|
parser = argparse.ArgumentParser(description=__doc__)
|
||||||
|
parser.add_argument("--source", choices=["redis", "sqlite"], default=os.getenv("POLYWEATHER_R2_ARCHIVE_SOURCE", "redis"))
|
||||||
|
parser.add_argument("--date", default=(datetime.now(timezone.utc).date() - timedelta(days=1)).isoformat())
|
||||||
|
parser.add_argument("--sqlite-db", default=os.getenv("POLYWEATHER_SQLITE_DB_PATH", ""))
|
||||||
|
parser.add_argument("--redis-url", default=os.getenv("POLYWEATHER_REDIS_URL", "redis://127.0.0.1:6379/0"))
|
||||||
|
parser.add_argument("--redis-stream-key", default=os.getenv("POLYWEATHER_REDIS_STREAM_KEY", "stream:city_observation"))
|
||||||
|
parser.add_argument("--dry-run", action="store_true")
|
||||||
|
return parser
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv: Iterable[str] | None = None) -> int:
|
||||||
|
args = _build_parser().parse_args(list(argv) if argv is not None else None)
|
||||||
|
_ensure_repo_path()
|
||||||
|
from web.r2_archive import (
|
||||||
|
R2ArchiveConfig,
|
||||||
|
R2ObjectStore,
|
||||||
|
build_realtime_archive_key,
|
||||||
|
events_to_jsonl,
|
||||||
|
)
|
||||||
|
|
||||||
|
archive_day = _parse_date(args.date)
|
||||||
|
if args.source == "sqlite":
|
||||||
|
db_path = args.sqlite_db or _default_sqlite_db_path()
|
||||||
|
events = _load_sqlite_events(db_path, archive_day)
|
||||||
|
else:
|
||||||
|
events = _load_redis_events(args.redis_url, args.redis_stream_key, archive_day)
|
||||||
|
|
||||||
|
body = events_to_jsonl(events)
|
||||||
|
key = build_realtime_archive_key(args.source, archive_day.isoformat())
|
||||||
|
print(json.dumps({"source": args.source, "date": archive_day.isoformat(), "events": len(events), "key": key}, indent=2))
|
||||||
|
if args.dry_run:
|
||||||
|
return 0
|
||||||
|
config = R2ArchiveConfig.from_env()
|
||||||
|
if config is None:
|
||||||
|
print("R2 env is not configured; set POLYWEATHER_R2_* variables", file=sys.stderr)
|
||||||
|
return 2
|
||||||
|
result = R2ObjectStore(config).put_object(key, body, content_type="application/x-ndjson")
|
||||||
|
print(json.dumps(result, indent=2))
|
||||||
|
return 0 if result.get("ok") else 1
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
@@ -0,0 +1,17 @@
|
|||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
|
||||||
|
|
||||||
|
def test_frontend_cache_validator_checks_cloudflare_edge_hits():
|
||||||
|
script = (ROOT / "scripts" / "validate_frontend_cache.sh").read_text(encoding="utf-8")
|
||||||
|
|
||||||
|
assert "REQUIRE_CF_CACHE" in script
|
||||||
|
assert "CF-Cache-Status" in script
|
||||||
|
assert "cf_cache_status" in script
|
||||||
|
assert "HIT" in script
|
||||||
|
assert "MISS" in script
|
||||||
|
assert "REVALIDATED" in script
|
||||||
|
assert 'check_cloudflare_cache_hit "/api/cities" "cities edge cache"' in script
|
||||||
|
assert 'check_cloudflare_cache_hit "/api/scan/terminal?limit=1" "scan terminal edge cache"' in script
|
||||||
@@ -0,0 +1,87 @@
|
|||||||
|
import json
|
||||||
|
|
||||||
|
from web.r2_archive import (
|
||||||
|
R2ArchiveConfig,
|
||||||
|
R2ObjectStore,
|
||||||
|
build_realtime_archive_key,
|
||||||
|
events_to_jsonl,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_r2_config_is_disabled_until_all_required_env_is_present(monkeypatch):
|
||||||
|
for key in (
|
||||||
|
"POLYWEATHER_R2_ACCOUNT_ID",
|
||||||
|
"POLYWEATHER_R2_BUCKET",
|
||||||
|
"POLYWEATHER_R2_ACCESS_KEY_ID",
|
||||||
|
"POLYWEATHER_R2_SECRET_ACCESS_KEY",
|
||||||
|
):
|
||||||
|
monkeypatch.delenv(key, raising=False)
|
||||||
|
|
||||||
|
assert R2ArchiveConfig.from_env() is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_r2_put_object_signs_cloudflare_r2_request():
|
||||||
|
captured = {}
|
||||||
|
|
||||||
|
def fake_transport(request, timeout):
|
||||||
|
captured["url"] = request.full_url
|
||||||
|
captured["method"] = request.get_method()
|
||||||
|
captured["headers"] = dict(request.header_items())
|
||||||
|
captured["body"] = request.data
|
||||||
|
|
||||||
|
class Response:
|
||||||
|
status = 200
|
||||||
|
|
||||||
|
def read(self):
|
||||||
|
return b""
|
||||||
|
|
||||||
|
def __enter__(self):
|
||||||
|
return self
|
||||||
|
|
||||||
|
def __exit__(self, *_args):
|
||||||
|
return False
|
||||||
|
|
||||||
|
return Response()
|
||||||
|
|
||||||
|
config = R2ArchiveConfig(
|
||||||
|
account_id="acct123",
|
||||||
|
bucket="polyweather-archive",
|
||||||
|
access_key_id="access",
|
||||||
|
secret_access_key="secret",
|
||||||
|
)
|
||||||
|
store = R2ObjectStore(config, transport=fake_transport)
|
||||||
|
|
||||||
|
store.put_object(
|
||||||
|
"sse-events/2026/06/16/test.jsonl",
|
||||||
|
b'{"ok":true}\n',
|
||||||
|
content_type="application/x-ndjson",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert captured["method"] == "PUT"
|
||||||
|
assert captured["url"] == (
|
||||||
|
"https://polyweather-archive.acct123.r2.cloudflarestorage.com/"
|
||||||
|
"sse-events/2026/06/16/test.jsonl"
|
||||||
|
)
|
||||||
|
assert captured["body"] == b'{"ok":true}\n'
|
||||||
|
assert captured["headers"]["Content-type"] == "application/x-ndjson"
|
||||||
|
assert captured["headers"]["Host"] == "polyweather-archive.acct123.r2.cloudflarestorage.com"
|
||||||
|
assert captured["headers"]["Authorization"].startswith("AWS4-HMAC-SHA256 ")
|
||||||
|
assert "Credential=access/" in captured["headers"]["Authorization"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_realtime_archive_key_and_jsonl_are_stable():
|
||||||
|
key = build_realtime_archive_key("redis", "2026-06-16", "city_observation_patch")
|
||||||
|
|
||||||
|
assert key == "sse-events/2026/06/16/city_observation_patch.redis.jsonl"
|
||||||
|
|
||||||
|
body = events_to_jsonl(
|
||||||
|
[
|
||||||
|
{"revision": 2, "city": "taipei", "payload": {"temp_c": 31.2}},
|
||||||
|
{"revision": 3, "city": "seoul", "payload": {"temp_c": 24.5}},
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
lines = body.decode("utf-8").splitlines()
|
||||||
|
assert len(lines) == 2
|
||||||
|
assert json.loads(lines[0])["revision"] == 2
|
||||||
|
assert json.loads(lines[1])["city"] == "seoul"
|
||||||
@@ -0,0 +1,165 @@
|
|||||||
|
"""Cloudflare R2 archive helpers for cold PolyWeather event snapshots."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
import hmac
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from typing import Any, Callable, Iterable, Optional
|
||||||
|
from urllib.parse import quote, urlparse
|
||||||
|
from urllib.request import Request, urlopen
|
||||||
|
|
||||||
|
|
||||||
|
R2_REGION = "auto"
|
||||||
|
R2_SERVICE = "s3"
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class R2ArchiveConfig:
|
||||||
|
account_id: str
|
||||||
|
bucket: str
|
||||||
|
access_key_id: str
|
||||||
|
secret_access_key: str
|
||||||
|
endpoint_url: str = ""
|
||||||
|
region: str = R2_REGION
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def from_env(cls, env: Optional[dict[str, str]] = None) -> Optional["R2ArchiveConfig"]:
|
||||||
|
source = env if env is not None else os.environ
|
||||||
|
account_id = str(source.get("POLYWEATHER_R2_ACCOUNT_ID") or "").strip()
|
||||||
|
bucket = str(source.get("POLYWEATHER_R2_BUCKET") or "").strip()
|
||||||
|
access_key_id = str(source.get("POLYWEATHER_R2_ACCESS_KEY_ID") or "").strip()
|
||||||
|
secret_access_key = str(source.get("POLYWEATHER_R2_SECRET_ACCESS_KEY") or "").strip()
|
||||||
|
if not all([account_id, bucket, access_key_id, secret_access_key]):
|
||||||
|
return None
|
||||||
|
return cls(
|
||||||
|
account_id=account_id,
|
||||||
|
bucket=bucket,
|
||||||
|
access_key_id=access_key_id,
|
||||||
|
secret_access_key=secret_access_key,
|
||||||
|
endpoint_url=str(source.get("POLYWEATHER_R2_ENDPOINT_URL") or "").strip(),
|
||||||
|
region=str(source.get("POLYWEATHER_R2_REGION") or R2_REGION).strip() or R2_REGION,
|
||||||
|
)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def base_url(self) -> str:
|
||||||
|
if self.endpoint_url:
|
||||||
|
return self.endpoint_url.rstrip("/")
|
||||||
|
return f"https://{self.bucket}.{self.account_id}.r2.cloudflarestorage.com"
|
||||||
|
|
||||||
|
|
||||||
|
Transport = Callable[[Request, float], Any]
|
||||||
|
|
||||||
|
|
||||||
|
def _default_transport(request: Request, timeout: float) -> Any:
|
||||||
|
return urlopen(request, timeout=timeout)
|
||||||
|
|
||||||
|
|
||||||
|
def _sha256_hex(data: bytes) -> str:
|
||||||
|
return hashlib.sha256(data).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def _sign(key: bytes, message: str) -> bytes:
|
||||||
|
return hmac.new(key, message.encode("utf-8"), hashlib.sha256).digest()
|
||||||
|
|
||||||
|
|
||||||
|
def _signature_key(secret: str, date_stamp: str, region: str) -> bytes:
|
||||||
|
k_date = _sign(("AWS4" + secret).encode("utf-8"), date_stamp)
|
||||||
|
k_region = _sign(k_date, region)
|
||||||
|
k_service = _sign(k_region, R2_SERVICE)
|
||||||
|
return _sign(k_service, "aws4_request")
|
||||||
|
|
||||||
|
|
||||||
|
class R2ObjectStore:
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
config: R2ArchiveConfig,
|
||||||
|
*,
|
||||||
|
transport: Transport = _default_transport,
|
||||||
|
timeout_sec: float = 30.0,
|
||||||
|
) -> None:
|
||||||
|
self.config = config
|
||||||
|
self.transport = transport
|
||||||
|
self.timeout_sec = max(1.0, float(timeout_sec))
|
||||||
|
|
||||||
|
def put_object(self, key: str, body: bytes, *, content_type: str = "application/octet-stream") -> dict[str, Any]:
|
||||||
|
object_key = quote(str(key).lstrip("/"), safe="/")
|
||||||
|
url = f"{self.config.base_url}/{object_key}"
|
||||||
|
parsed = urlparse(url)
|
||||||
|
host = parsed.netloc
|
||||||
|
payload_hash = _sha256_hex(body)
|
||||||
|
now = datetime.now(timezone.utc)
|
||||||
|
amz_date = now.strftime("%Y%m%dT%H%M%SZ")
|
||||||
|
date_stamp = now.strftime("%Y%m%d")
|
||||||
|
canonical_uri = parsed.path or "/"
|
||||||
|
canonical_headers = (
|
||||||
|
f"content-type:{content_type}\n"
|
||||||
|
f"host:{host}\n"
|
||||||
|
f"x-amz-content-sha256:{payload_hash}\n"
|
||||||
|
f"x-amz-date:{amz_date}\n"
|
||||||
|
)
|
||||||
|
signed_headers = "content-type;host;x-amz-content-sha256;x-amz-date"
|
||||||
|
canonical_request = "\n".join(
|
||||||
|
[
|
||||||
|
"PUT",
|
||||||
|
canonical_uri,
|
||||||
|
"",
|
||||||
|
canonical_headers,
|
||||||
|
signed_headers,
|
||||||
|
payload_hash,
|
||||||
|
]
|
||||||
|
)
|
||||||
|
credential_scope = f"{date_stamp}/{self.config.region}/{R2_SERVICE}/aws4_request"
|
||||||
|
string_to_sign = "\n".join(
|
||||||
|
[
|
||||||
|
"AWS4-HMAC-SHA256",
|
||||||
|
amz_date,
|
||||||
|
credential_scope,
|
||||||
|
_sha256_hex(canonical_request.encode("utf-8")),
|
||||||
|
]
|
||||||
|
)
|
||||||
|
signature = hmac.new(
|
||||||
|
_signature_key(self.config.secret_access_key, date_stamp, self.config.region),
|
||||||
|
string_to_sign.encode("utf-8"),
|
||||||
|
hashlib.sha256,
|
||||||
|
).hexdigest()
|
||||||
|
authorization = (
|
||||||
|
"AWS4-HMAC-SHA256 "
|
||||||
|
f"Credential={self.config.access_key_id}/{credential_scope}, "
|
||||||
|
f"SignedHeaders={signed_headers}, "
|
||||||
|
f"Signature={signature}"
|
||||||
|
)
|
||||||
|
request = Request(url, data=body, method="PUT")
|
||||||
|
request.add_header("Content-Type", content_type)
|
||||||
|
request.add_header("Host", host)
|
||||||
|
request.add_header("X-Amz-Content-Sha256", payload_hash)
|
||||||
|
request.add_header("X-Amz-Date", amz_date)
|
||||||
|
request.add_header("Authorization", authorization)
|
||||||
|
|
||||||
|
with self.transport(request, self.timeout_sec) as response:
|
||||||
|
response.read()
|
||||||
|
status = int(getattr(response, "status", 200) or 200)
|
||||||
|
return {"ok": 200 <= status < 300, "status": status, "key": key}
|
||||||
|
|
||||||
|
|
||||||
|
def build_realtime_archive_key(source: str, archive_date: str, stream_name: str = "city_observation_patch") -> str:
|
||||||
|
parts = str(archive_date).split("-")
|
||||||
|
if len(parts) != 3:
|
||||||
|
raise ValueError("archive_date must be YYYY-MM-DD")
|
||||||
|
year, month, day = parts
|
||||||
|
safe_source = quote(str(source or "unknown").strip().lower() or "unknown", safe="")
|
||||||
|
safe_stream = quote(str(stream_name or "events").strip().lower() or "events", safe="")
|
||||||
|
return f"sse-events/{year}/{month}/{day}/{safe_stream}.{safe_source}.jsonl"
|
||||||
|
|
||||||
|
|
||||||
|
def events_to_jsonl(events: Iterable[dict[str, Any]]) -> bytes:
|
||||||
|
lines = [
|
||||||
|
json.dumps(event, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
|
||||||
|
for event in events
|
||||||
|
]
|
||||||
|
if not lines:
|
||||||
|
return b""
|
||||||
|
return ("\n".join(lines) + "\n").encode("utf-8")
|
||||||
Reference in New Issue
Block a user