live app goes push-mode: own Alchemy webhook + health check + watchdog

60s poll's ~39s avg detection lag is slippage the copy edge pays; push is
~3s (paper-measured). fly.live.toml gains http_service + /health check
(self-heal); start.sh live role switches on ALCHEMY_SIGNING_KEY like paper
(poll fallback intact; push mode keeps the 60s heartbeat + 5min backstop
poll). sync_webhook.py syncs BOTH webhooks (alchemy_webhook_id_live).
watchdog.yml probes both apps; a disarmed live app paging is expected.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

.github/workflows/watchdog.yml

@@ -1,9 +1,12 @@
-# Dead-man's switch for the copybot worker (the notify half; the self-heal
-# half is fly.toml's http /health check, which restarts a dark machine).
+# Dead-man's switch for the copybot workers (the notify half; the self-heal
+# half is each app's http /health check, which restarts a dark machine).
 # GH cron is coarse (fires ~every 30-150 min in practice) — fine here: the
 # failure mode this guards is MULTI-HOUR silence (the 2026-07-07 Fly trial
 # expiry left the bot dark 3.5h with nothing watching). Runs on GitHub's
-# infra, so it works when the Mac is asleep AND the Fly box is dead.
+# infra, so it works when the Mac is asleep AND the Fly boxes are dead.
+# 2026-07-10: also probes the REAL-MONEY app (push mode gave it HTTP).
+# NB: a DISARMED live app idles without HTTP — expected to page; disarming
+# is a deliberate act, treat the page as confirmation.
 name: watchdog
 on:
   schedule:
@@ -13,13 +16,21 @@ on:
 jobs:
   health:
     runs-on: ubuntu-latest
-    timeout-minutes: 10
+    timeout-minutes: 15
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - app: wwf-copybot
+            label: copybot (paper)
+          - app: wwf-copybot-live
+            label: copybot LIVE (real money)
     steps:
       - name: probe /health (3 tries, 60s apart)
         id: probe
         run: |
           for i in 1 2 3; do
-            if curl -fsS --max-time 10 https://wwf-copybot.fly.dev/health; then
+            if curl -fsS --max-time 10 https://${{ matrix.app }}.fly.dev/health; then
               echo; echo "healthy on try $i"; exit 0
             fi
             echo "try $i failed"; [ $i -lt 3 ] && sleep 60
@@ -29,7 +40,7 @@ jobs:
         if: failure()
         run: |
           curl -fsS -X POST -H 'Content-Type: application/json' \
-            -d '{"content":"🚨 **copybot DOWN** — wwf-copybot.fly.dev/health failed 3 probes over 2 min. Check `flyctl status --app wwf-copybot` / `flyctl logs`. (Fly auto-restart should be trying; if this repeats, it is crash-looping or the account/billing broke — see 2026-07-07 trial-expiry incident.)"}' \
+            -d '{"content":"🚨 **${{ matrix.label }} DOWN** — ${{ matrix.app }}.fly.dev/health failed 3 probes over 2 min. Check `flyctl status --app ${{ matrix.app }}` / `flyctl logs`. (Fly auto-restart should be trying; if this repeats, it is crash-looping or the account/billing broke — see 2026-07-07 trial-expiry incident. If you just DISARMED the live app, this page is expected.)"}' \
             "$DISCORD_WEBHOOK_URL"
         env:
           DISCORD_WEBHOOK_URL: ${{ secrets.DISCORD_WEBHOOK_URL }}

fly.live.toml

@@ -2,10 +2,9 @@
 # paper worker (wwf-copybot) so the two books never share a process, state
 # file, or failure mode (LIVE_ROLLOUT Phase 3.3). Region rules: see fly.toml.
 #
-# Poll mode only (60s; the Alchemy push webhook stays on the paper app), no
-# inbound services, no health checks — the unarmed machine idles in
-# geocheck --idle until the USER sets LIVE_PRIVATE_KEY + LIVE_FUNDER_ADDRESS
-# + LIVE_CONFIRM (the typed confirmation phrase) via flyctl secrets set.
+# The unarmed machine idles in geocheck --idle until the USER sets
+# LIVE_PRIVATE_KEY + LIVE_FUNDER_ADDRESS + LIVE_CONFIRM (the typed
+# confirmation phrase) via flyctl secrets set.
 app = "wwf-copybot-live"
 primary_region = "arn"
 
@@ -15,6 +14,29 @@ primary_region = "arn"
 [env]
   COPYBOT_ROLE = "live"
 
+# public HTTPS endpoint for the live app's OWN Alchemy push webhook
+# (POST /alchemy — added 2026-07-10; the 60s poll's ~39s avg detection lag
+# is slippage the copy edge pays, push is ~3s). Its webhook + signing key
+# are SEPARATE from the paper app's; the two books still share nothing.
+[http_service]
+  internal_port = 8080
+  force_https = true
+  auto_stop_machines = "off"
+  auto_start_machines = true
+  min_machines_running = 1
+
+# self-heal half of the watchdog (notify half: watchdog.yml → Discord).
+# NB: an UNARMED live app idles in geocheck --idle (no HTTP server) — a
+# failing health check while disarmed is expected, not a fault.
+[checks]
+  [checks.health]
+    type = "http"
+    port = 8080
+    path = "/health"
+    interval = "30s"
+    timeout = "5s"
+    grace_period = "60s"
+
 [[vm]]
   size = "shared-cpu-1x"
   memory = "512mb"

host/start.sh

@@ -65,9 +65,17 @@ git config user.name  "copybot[bot]"
 git config user.email "copybot@users.noreply.github.com"
 
 # live role, ARMED: own config (committed template + env secrets), own state
-# file, own feed/fills (config paths), ALWAYS poll mode — the Alchemy push
-# webhook stays pointed at the paper app (LIVE_ROLLOUT Phase 3.3).
+# file, own feed/fills (config paths). ALCHEMY_SIGNING_KEY set -> PUSH mode
+# (the live app's OWN webhook, wired 2026-07-10 — ~3s detection vs the 60s
+# poll's ~39s avg; 60s heartbeat + 5min backstop poll are built in). Without
+# the key -> classic poll, same as before.
 if [ "${COPYBOT_ROLE:-paper}" = "live" ]; then
+  if [ -n "${ALCHEMY_SIGNING_KEY:-}" ]; then
+    exec python3 copybot.py \
+      --config config.live.example.json \
+      --state  copybot_state.live.json \
+      --live
+  fi
   exec python3 copybot.py \
     --config config.live.example.json \
     --state  copybot_state.live.json \

live/sync_webhook.py

@@ -31,31 +31,7 @@ def call(method, path, token, body=None):
     return json.loads(urllib.request.urlopen(req, timeout=30, context=_SSL).read())
 
 
-def main():
-    try:
-        cfg = json.load(open(os.path.join(HERE, "..", "config.json")))
-    except Exception:
-        cfg = {}
-    token = cfg.get("alchemy_notify_token")
-    wh_id = cfg.get("alchemy_webhook_id")
-    if not token:
-        print("[webhook-sync] no alchemy_notify_token in ../config.json — skipped."
-              " (Copy the Auth token from dashboard.alchemy.com → Webhooks to enable"
-              " automatic address sync; until then update the address list there"
-              " manually — the 5-min backstop poll covers the gap at poll-speed lag.)")
-        return 0
-    want = {w["wallet"].lower()
-            for w in json.load(open(os.path.join(HERE, "copybot.paper.json")))["wallets"]}
-
-    if not wh_id:  # find it by our railway URL
-        hooks = call("GET", "/team-webhooks", token).get("data", [])
-        ours = [h for h in hooks if "copybot-production" in (h.get("webhook_url") or "")]
-        if not ours:
-            print("[webhook-sync] ⚠ no webhook pointing at the copybot URL found — create"
-                  " one first (see README) or set alchemy_webhook_id in config.json")
-            return 1
-        wh_id = ours[0]["id"]
-
+def sync_one(token, wh_id, want, label):
     have, after = set(), None
     while True:  # paginated
         q = f"/webhook-addresses?webhook_id={wh_id}&limit=100" + \
@@ -68,13 +44,46 @@ def main():
 
     add, drop = sorted(want - have), sorted(have - want)
     if not add and not drop:
-        print(f"[webhook-sync] in sync — {len(want)} addresses on {wh_id}")
-        return 0
+        print(f"[webhook-sync] {label} in sync — {len(want)} addresses on {wh_id}")
+        return
     call("PATCH", "/update-webhook-addresses", token,
          {"webhook_id": wh_id, "addresses_to_add": add, "addresses_to_remove": drop})
-    print(f"[webhook-sync] {wh_id}: +{len(add)} −{len(drop)} → {len(want)} addresses"
+    print(f"[webhook-sync] {label} {wh_id}: +{len(add)} −{len(drop)} → {len(want)} addresses"
           + (f" (added {', '.join(a[:10] for a in add)})" if add else "")
           + (f" (removed {', '.join(a[:10] for a in drop)})" if drop else ""))
+
+
+def main():
+    try:
+        cfg = json.load(open(os.path.join(HERE, "..", "config.json")))
+    except Exception:
+        cfg = {}
+    token = cfg.get("alchemy_notify_token")
+    if not token:
+        print("[webhook-sync] no alchemy_notify_token in ../config.json — skipped."
+              " (Copy the Auth token from dashboard.alchemy.com → Webhooks to enable"
+              " automatic address sync; until then update the address list there"
+              " manually — the 5-min backstop poll covers the gap at poll-speed lag.)")
+        return 0
+    want = {w["wallet"].lower()
+            for w in json.load(open(os.path.join(HERE, "copybot.paper.json")))["wallets"]}
+
+    wh_id = cfg.get("alchemy_webhook_id")
+    if not wh_id:  # find it by our railway URL
+        hooks = call("GET", "/team-webhooks", token).get("data", [])
+        ours = [h for h in hooks if "copybot-production" in (h.get("webhook_url") or "")]
+        if not ours:
+            print("[webhook-sync] ⚠ no webhook pointing at the copybot URL found — create"
+                  " one first (see README) or set alchemy_webhook_id in config.json")
+            return 1
+        wh_id = ours[0]["id"]
+    sync_one(token, wh_id, want, "paper")
+
+    # the live app's own webhook (2026-07-10) — same follow set, separate
+    # webhook + signing key so the two books share nothing
+    wh_live = cfg.get("alchemy_webhook_id_live")
+    if wh_live:
+        sync_one(token, wh_live, want, "live")
     return 0