From df3c750efec1ce19151b894a72b8a56c9ffd074f Mon Sep 17 00:00:00 2001 From: jaxperro Date: Fri, 10 Jul 2026 01:32:22 -0400 Subject: [PATCH] live app goes push-mode: own Alchemy webhook + health check + watchdog 60s poll's ~39s avg detection lag is slippage the copy edge pays; push is ~3s (paper-measured). fly.live.toml gains http_service + /health check (self-heal); start.sh live role switches on ALCHEMY_SIGNING_KEY like paper (poll fallback intact; push mode keeps the 60s heartbeat + 5min backstop poll). sync_webhook.py syncs BOTH webhooks (alchemy_webhook_id_live). watchdog.yml probes both apps; a disarmed live app paging is expected. Co-Authored-By: Claude Fable 5 --- .github/workflows/watchdog.yml | 23 ++++++++---- fly.live.toml | 30 +++++++++++++--- host/start.sh | 12 +++++-- live/sync_webhook.py | 65 +++++++++++++++++++--------------- 4 files changed, 90 insertions(+), 40 deletions(-) diff --git a/.github/workflows/watchdog.yml b/.github/workflows/watchdog.yml index 0428ee13..5c8f32a4 100644 --- a/.github/workflows/watchdog.yml +++ b/.github/workflows/watchdog.yml @@ -1,9 +1,12 @@ -# Dead-man's switch for the copybot worker (the notify half; the self-heal -# half is fly.toml's http /health check, which restarts a dark machine). +# Dead-man's switch for the copybot workers (the notify half; the self-heal +# half is each app's http /health check, which restarts a dark machine). # GH cron is coarse (fires ~every 30-150 min in practice) — fine here: the # failure mode this guards is MULTI-HOUR silence (the 2026-07-07 Fly trial # expiry left the bot dark 3.5h with nothing watching). Runs on GitHub's -# infra, so it works when the Mac is asleep AND the Fly box is dead. +# infra, so it works when the Mac is asleep AND the Fly boxes are dead. +# 2026-07-10: also probes the REAL-MONEY app (push mode gave it HTTP). +# NB: a DISARMED live app idles without HTTP — expected to page; disarming +# is a deliberate act, treat the page as confirmation. name: watchdog on: schedule: @@ -13,13 +16,21 @@ on: jobs: health: runs-on: ubuntu-latest - timeout-minutes: 10 + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + include: + - app: wwf-copybot + label: copybot (paper) + - app: wwf-copybot-live + label: copybot LIVE (real money) steps: - name: probe /health (3 tries, 60s apart) id: probe run: | for i in 1 2 3; do - if curl -fsS --max-time 10 https://wwf-copybot.fly.dev/health; then + if curl -fsS --max-time 10 https://${{ matrix.app }}.fly.dev/health; then echo; echo "healthy on try $i"; exit 0 fi echo "try $i failed"; [ $i -lt 3 ] && sleep 60 @@ -29,7 +40,7 @@ jobs: if: failure() run: | curl -fsS -X POST -H 'Content-Type: application/json' \ - -d '{"content":"🚨 **copybot DOWN** — wwf-copybot.fly.dev/health failed 3 probes over 2 min. Check `flyctl status --app wwf-copybot` / `flyctl logs`. (Fly auto-restart should be trying; if this repeats, it is crash-looping or the account/billing broke — see 2026-07-07 trial-expiry incident.)"}' \ + -d '{"content":"🚨 **${{ matrix.label }} DOWN** — ${{ matrix.app }}.fly.dev/health failed 3 probes over 2 min. Check `flyctl status --app ${{ matrix.app }}` / `flyctl logs`. (Fly auto-restart should be trying; if this repeats, it is crash-looping or the account/billing broke — see 2026-07-07 trial-expiry incident. If you just DISARMED the live app, this page is expected.)"}' \ "$DISCORD_WEBHOOK_URL" env: DISCORD_WEBHOOK_URL: ${{ secrets.DISCORD_WEBHOOK_URL }} diff --git a/fly.live.toml b/fly.live.toml index 7884c280..5c37db8b 100644 --- a/fly.live.toml +++ b/fly.live.toml @@ -2,10 +2,9 @@ # paper worker (wwf-copybot) so the two books never share a process, state # file, or failure mode (LIVE_ROLLOUT Phase 3.3). Region rules: see fly.toml. # -# Poll mode only (60s; the Alchemy push webhook stays on the paper app), no -# inbound services, no health checks — the unarmed machine idles in -# geocheck --idle until the USER sets LIVE_PRIVATE_KEY + LIVE_FUNDER_ADDRESS -# + LIVE_CONFIRM (the typed confirmation phrase) via flyctl secrets set. +# The unarmed machine idles in geocheck --idle until the USER sets +# LIVE_PRIVATE_KEY + LIVE_FUNDER_ADDRESS + LIVE_CONFIRM (the typed +# confirmation phrase) via flyctl secrets set. app = "wwf-copybot-live" primary_region = "arn" @@ -15,6 +14,29 @@ primary_region = "arn" [env] COPYBOT_ROLE = "live" +# public HTTPS endpoint for the live app's OWN Alchemy push webhook +# (POST /alchemy — added 2026-07-10; the 60s poll's ~39s avg detection lag +# is slippage the copy edge pays, push is ~3s). Its webhook + signing key +# are SEPARATE from the paper app's; the two books still share nothing. +[http_service] + internal_port = 8080 + force_https = true + auto_stop_machines = "off" + auto_start_machines = true + min_machines_running = 1 + +# self-heal half of the watchdog (notify half: watchdog.yml → Discord). +# NB: an UNARMED live app idles in geocheck --idle (no HTTP server) — a +# failing health check while disarmed is expected, not a fault. +[checks] + [checks.health] + type = "http" + port = 8080 + path = "/health" + interval = "30s" + timeout = "5s" + grace_period = "60s" + [[vm]] size = "shared-cpu-1x" memory = "512mb" diff --git a/host/start.sh b/host/start.sh index 012456d9..8a06401b 100755 --- a/host/start.sh +++ b/host/start.sh @@ -65,9 +65,17 @@ git config user.name "copybot[bot]" git config user.email "copybot@users.noreply.github.com" # live role, ARMED: own config (committed template + env secrets), own state -# file, own feed/fills (config paths), ALWAYS poll mode — the Alchemy push -# webhook stays pointed at the paper app (LIVE_ROLLOUT Phase 3.3). +# file, own feed/fills (config paths). ALCHEMY_SIGNING_KEY set -> PUSH mode +# (the live app's OWN webhook, wired 2026-07-10 — ~3s detection vs the 60s +# poll's ~39s avg; 60s heartbeat + 5min backstop poll are built in). Without +# the key -> classic poll, same as before. if [ "${COPYBOT_ROLE:-paper}" = "live" ]; then + if [ -n "${ALCHEMY_SIGNING_KEY:-}" ]; then + exec python3 copybot.py \ + --config config.live.example.json \ + --state copybot_state.live.json \ + --live + fi exec python3 copybot.py \ --config config.live.example.json \ --state copybot_state.live.json \ diff --git a/live/sync_webhook.py b/live/sync_webhook.py index ce766e29..2c9a1cd0 100644 --- a/live/sync_webhook.py +++ b/live/sync_webhook.py @@ -31,31 +31,7 @@ def call(method, path, token, body=None): return json.loads(urllib.request.urlopen(req, timeout=30, context=_SSL).read()) -def main(): - try: - cfg = json.load(open(os.path.join(HERE, "..", "config.json"))) - except Exception: - cfg = {} - token = cfg.get("alchemy_notify_token") - wh_id = cfg.get("alchemy_webhook_id") - if not token: - print("[webhook-sync] no alchemy_notify_token in ../config.json — skipped." - " (Copy the Auth token from dashboard.alchemy.com → Webhooks to enable" - " automatic address sync; until then update the address list there" - " manually — the 5-min backstop poll covers the gap at poll-speed lag.)") - return 0 - want = {w["wallet"].lower() - for w in json.load(open(os.path.join(HERE, "copybot.paper.json")))["wallets"]} - - if not wh_id: # find it by our railway URL - hooks = call("GET", "/team-webhooks", token).get("data", []) - ours = [h for h in hooks if "copybot-production" in (h.get("webhook_url") or "")] - if not ours: - print("[webhook-sync] ⚠ no webhook pointing at the copybot URL found — create" - " one first (see README) or set alchemy_webhook_id in config.json") - return 1 - wh_id = ours[0]["id"] - +def sync_one(token, wh_id, want, label): have, after = set(), None while True: # paginated q = f"/webhook-addresses?webhook_id={wh_id}&limit=100" + \ @@ -68,13 +44,46 @@ def main(): add, drop = sorted(want - have), sorted(have - want) if not add and not drop: - print(f"[webhook-sync] in sync — {len(want)} addresses on {wh_id}") - return 0 + print(f"[webhook-sync] {label} in sync — {len(want)} addresses on {wh_id}") + return call("PATCH", "/update-webhook-addresses", token, {"webhook_id": wh_id, "addresses_to_add": add, "addresses_to_remove": drop}) - print(f"[webhook-sync] {wh_id}: +{len(add)} −{len(drop)} → {len(want)} addresses" + print(f"[webhook-sync] {label} {wh_id}: +{len(add)} −{len(drop)} → {len(want)} addresses" + (f" (added {', '.join(a[:10] for a in add)})" if add else "") + (f" (removed {', '.join(a[:10] for a in drop)})" if drop else "")) + + +def main(): + try: + cfg = json.load(open(os.path.join(HERE, "..", "config.json"))) + except Exception: + cfg = {} + token = cfg.get("alchemy_notify_token") + if not token: + print("[webhook-sync] no alchemy_notify_token in ../config.json — skipped." + " (Copy the Auth token from dashboard.alchemy.com → Webhooks to enable" + " automatic address sync; until then update the address list there" + " manually — the 5-min backstop poll covers the gap at poll-speed lag.)") + return 0 + want = {w["wallet"].lower() + for w in json.load(open(os.path.join(HERE, "copybot.paper.json")))["wallets"]} + + wh_id = cfg.get("alchemy_webhook_id") + if not wh_id: # find it by our railway URL + hooks = call("GET", "/team-webhooks", token).get("data", []) + ours = [h for h in hooks if "copybot-production" in (h.get("webhook_url") or "")] + if not ours: + print("[webhook-sync] ⚠ no webhook pointing at the copybot URL found — create" + " one first (see README) or set alchemy_webhook_id in config.json") + return 1 + wh_id = ours[0]["id"] + sync_one(token, wh_id, want, "paper") + + # the live app's own webhook (2026-07-10) — same follow set, separate + # webhook + signing key so the two books share nothing + wh_live = cfg.get("alchemy_webhook_id_live") + if wh_live: + sync_one(token, wh_live, want, "live") return 0