Add Sentry, Pino logs, and OTel spans on WS pipeline (M2).

Capture REST/WS/SSE errors in Sentry, emit structured pino logs with correlation IDs, and trace the book-tick pipeline to Sentry via OpenTelemetry.

Co-authored-by: Cursor <cursoragent@cursor.com>

src/interfaces/http/rate-limit.ts

@@ -0,0 +1,86 @@
+import { Ratelimit } from "@upstash/ratelimit";
+import { Redis } from "@upstash/redis";
+import type { NextFunction, Request, Response } from "express";
+import { isUpstashConfigured } from "../cache/upstash.js";
+
+type RouteTier = "read" | "stream" | "write" | "health";
+
+const LIMITS: Record<
+  Exclude<RouteTier, "health">,
+  { requests: number; window: `${number} s` | `${number} m` }
+> = {
+  read: { requests: 60, window: "1 m" },
+  stream: { requests: 10, window: "1 m" },
+  write: { requests: 30, window: "1 m" },
+};
+
+const limiters = new Map<Exclude<RouteTier, "health">, Ratelimit>();
+
+function getLimiter(tier: Exclude<RouteTier, "health">): Ratelimit | null {
+  if (!isUpstashConfigured()) {
+    return null;
+  }
+  let limiter = limiters.get(tier);
+  if (limiter) {
+    return limiter;
+  }
+  const url = process.env.UPSTASH_REDIS_REST_URL!.trim();
+  const token = process.env.UPSTASH_REDIS_REST_TOKEN!.trim();
+  const redis = new Redis({ url, token });
+  const cfg = LIMITS[tier];
+  limiter = new Ratelimit({
+    redis,
+    limiter: Ratelimit.slidingWindow(cfg.requests, cfg.window),
+    prefix: `arbpulse:${tier}`,
+  });
+  limiters.set(tier, limiter);
+  return limiter;
+}
+
+function clientKey(req: Request): string {
+  const forwarded = req.header("x-forwarded-for");
+  if (forwarded) {
+    return forwarded.split(",")[0]?.trim() || "unknown";
+  }
+  return req.ip || req.socket.remoteAddress || "unknown";
+}
+
+function tierForPath(path: string, method: string): RouteTier {
+  if (path === "/health") return "health";
+  if (path === "/stream") return "stream";
+  if (method === "GET" || method === "HEAD") return "read";
+  return "write";
+}
+
+export function createRateLimitMiddleware() {
+  return async (
+    req: Request,
+    res: Response,
+    next: NextFunction,
+  ): Promise<void> => {
+    const tier = tierForPath(req.path, req.method);
+    if (tier === "health") {
+      next();
+      return;
+    }
+
+    const limiter = getLimiter(tier);
+    if (!limiter) {
+      next();
+      return;
+    }
+
+    const { success, reset } = await limiter.limit(`${tier}:${clientKey(req)}`);
+    if (success) {
+      next();
+      return;
+    }
+
+    const retryAfterSec = Math.max(1, Math.ceil((reset - Date.now()) / 1000));
+    res.setHeader("Retry-After", String(retryAfterSec));
+    res.status(429).json({
+      success: false,
+      error: "Too many requests",
+    });
+  };
+}

src/interfaces/http/routes.ts

@@ -1,6 +1,20 @@
-import { Router, type Request, type Response } from "express";
+import {
+  Router,
+  type NextFunction,
+  type Request,
+  type Response,
+} from "express";
 import type { ApplicationService } from "../../composition/application-service.js";
+import {
+  bindRequestCorrelation,
+  runWithCorrelation,
+} from "../../infrastructure/logging/correlation.js";
+import {
+  getCachedSnapshot,
+  setCachedSnapshot,
+} from "../../infrastructure/cache/upstash.js";
 import type { SseHub } from "../sse/sse.js";
+import { createRateLimitMiddleware } from "./rate-limit.js";
 import {
   ConfigPatchSchema,
   DemoControlBodySchema,
@@ -13,12 +27,28 @@ import { parseBody } from "./validate.js";
 export function createRouter(app: ApplicationService, sse: SseHub): Router {
   const router = Router();
 
+  router.use((req: Request, res: Response, next: NextFunction) => {
+    const ctx = bindRequestCorrelation(req.header("x-request-id"));
+    res.setHeader("x-request-id", ctx.correlationId);
+    runWithCorrelation(ctx, () => next());
+  });
+
+  router.use(createRateLimitMiddleware());
+
   router.get("/health", (_req: Request, res: Response) => {
     res.json({ success: true, data: { status: "ok", ts: Date.now() } });
   });
 
-  router.get("/state", (_req: Request, res: Response) => {
-    res.json({ success: true, data: app.getSnapshot() });
+  router.get("/state", async (_req: Request, res: Response) => {
+    const cached =
+      await getCachedSnapshot<ReturnType<ApplicationService["getSnapshot"]>>();
+    if (cached) {
+      res.json({ success: true, data: cached });
+      return;
+    }
+    const snapshot = app.getSnapshot();
+    void setCachedSnapshot(snapshot);
+    res.json({ success: true, data: snapshot });
   });
 
   router.get("/stream", (req: Request, res: Response) => {

src/interfaces/sse/sse.ts

@@ -1,6 +1,9 @@
 import type { Request, Response } from "express";
+import * as Sentry from "@sentry/node";
 import { config } from "../../infrastructure/config/config.js";
+import { setCachedSnapshot } from "../../infrastructure/cache/upstash.js";
 import { createLogger } from "../../infrastructure/logging/logger.js";
+import { withSpan } from "../../instrumentation/otel.js";
 import type { ApplicationService } from "../../composition/application-service.js";
 
 const log = createLogger("sse");
@@ -22,26 +25,58 @@ export class SseHub {
   }
 
   handle(req: Request, res: Response): void {
-    res.writeHead(200, {
-      "Content-Type": "text/event-stream",
-      "Cache-Control": "no-cache, no-transform",
-      Connection: "keep-alive",
-      "X-Accel-Buffering": "no",
-    });
-    res.write(`retry: 2000\n\n`);
-    this.send(res, this.app.getSnapshot());
-    this.clients.add(res);
-    log.info(`client connected (${this.clients.size} total)`);
+    try {
+      res.writeHead(200, {
+        "Content-Type": "text/event-stream",
+        "Cache-Control": "no-cache, no-transform",
+        Connection: "keep-alive",
+        "X-Accel-Buffering": "no",
+      });
+      res.write(`retry: 2000\n\n`);
+      this.send(res, this.app.getSnapshot());
+      this.clients.add(res);
+      log.info(`client connected`, { clientCount: this.clients.size });
 
-    req.on("close", () => {
-      this.clients.delete(res);
-    });
+      req.on("close", () => {
+        this.clients.delete(res);
+      });
+
+      res.on("error", (err) => {
+        const error = err instanceof Error ? err : new Error(String(err));
+        log.error("sse client error", { error: error.message });
+        Sentry.captureException(error, { tags: { component: "sse" } });
+        this.clients.delete(res);
+      });
+    } catch (err) {
+      const error = err instanceof Error ? err : new Error(String(err));
+      log.error("sse handle failed", { error: error.message });
+      Sentry.captureException(error, { tags: { component: "sse" } });
+      if (!res.headersSent) {
+        res.status(500).json({ success: false, error: "SSE setup failed" });
+      }
+    }
   }
 
   private broadcast(): void {
     if (this.clients.size === 0) return;
-    const snapshot = this.app.getSnapshot();
-    for (const res of this.clients) this.send(res, snapshot);
+    void withSpan(
+      "sse.broadcast",
+      { clientCount: this.clients.size },
+      async () => {
+        const snapshot = this.app.getSnapshot();
+        void setCachedSnapshot(snapshot);
+        for (const res of this.clients) {
+          try {
+            this.send(res, snapshot);
+          } catch (err) {
+            const error = err instanceof Error ? err : new Error(String(err));
+            log.warn("sse send failed", { error: error.message });
+            Sentry.captureException(error, { tags: { component: "sse" } });
+            this.clients.delete(res);
+          }
+        }
+      },
+    );
   }
 
   private send(res: Response, data: unknown): void {