Expand chronological signal audit

README.md

@@ -47,9 +47,12 @@ clean live comparison.
 
 Run `npm run evaluate:signals` to test the price-signal rules against one month
 of hourly Polymarket history. The evaluator forms signals only from prior
-one-hour, one-day, and one-week prices, marks them 12 hours later, applies a
+one-hour, one-day, and one-week prices, marks them 6, 12, 24, and 72 hours later,
-conservative half-cent cost estimate, and reports a chronological 70/30 split.
+applies a conservative half-cent cost estimate, and reports a chronological
-Set `EVAL_MARKETS` or `EVAL_CONCURRENCY` to change the default 80-market run.
+70/30 split plus three consecutive time segments. Results are also clustered by
+market so repeated observations from one contract cannot masquerade as broad
+evidence. Set `EVAL_MARKETS`, `EVAL_CONCURRENCY`, `EVAL_HORIZONS`, or
+`EVAL_COST_CENTS` to change the audit.
 The first 80-market audit found that reversal signals lost 4.34% on average in
 both chronological partitions, while crypto and longshot samples were also
 negative overall. Engine v37 therefore blocks reversal entries outside the fixed
@@ -57,6 +60,13 @@ negative overall. Engine v37 therefore blocks reversal entries outside the fixed
 It does not boost any rule from this audit because no positive rule was robust
 across the chronological split.
 
+A corrected 200-market audit paged through 197 markets with usable history and
+1,912 twelve-hour outcomes. Reversals remained negative in every chronological
+segment and averaged -4.13%. Sports trends were negative in train and test and
+averaged -3.53% at 72 hours. Politics trends were the sole cohort with positive
+row-level returns in all three 72-hour segments, but its market-cluster interval
+still crossed zero; that supports a longer hold test, not a larger entry bet.
+
 Paper accounts created with a password are also saved through the backend, so a
 user can log in from another device and see the same paper portfolio, activity,
 and value history. Passwordless paper accounts remain local-only.

scripts/evaluate-signals.mjs

@@ -1,7 +1,10 @@
 const GAMMA = "https://gamma-api.polymarket.com";
 const CLOB = "https://clob.polymarket.com";
-const MARKET_LIMIT = Math.max(10, Math.min(200, Number(process.env.EVAL_MARKETS || 80)));
+const MARKET_LIMIT = Math.max(10, Math.min(500, Number(process.env.EVAL_MARKETS || 80)));
 const CONCURRENCY = Math.max(1, Math.min(12, Number(process.env.EVAL_CONCURRENCY || 6)));
+const HORIZONS = [...new Set(String(process.env.EVAL_HORIZONS || "6,12,24,72").split(",")
+  .map(Number).filter((value) => Number.isFinite(value) && value >= 1 && value <= 168))].sort((a, b) => a - b);
+const COST_CENTS = Math.max(0, Math.min(5, Number(process.env.EVAL_COST_CENTS || 0.5)));
 const HOUR = 3600;
 
 const CATEGORY_RULES = [
@@ -102,21 +105,26 @@ function evaluateMarket(market, points) {
     if (bucket === previousBucket || current.p < 0.08 || current.p > 0.92) continue;
     const signal = signalAt(points, index);
     if (!signal) continue;
-    const future = atOrAfter(points, current.t + 12 * HOUR);
-    if (!future || future.t - (current.t + 12 * HOUR) > 3 * HOUR) continue;
     const entry = signal.side === "YES" ? current.p : 1 - current.p;
-    const exit = signal.side === "YES" ? future.p : 1 - future.p;
     const fadeEntry = signal.side === "YES" ? 1 - current.p : current.p;
-    const fadeExit = signal.side === "YES" ? 1 - future.p : future.p;
     if (entry <= 0.02 || entry >= 0.98) continue;
+    let captured = false;
+    for (const horizonHours of HORIZONS) {
+      const future = atOrAfter(points, current.t + horizonHours * HOUR);
+      if (!future || future.t - (current.t + horizonHours * HOUR) > 3 * HOUR) continue;
+      const exit = signal.side === "YES" ? future.p : 1 - future.p;
+      const fadeExit = signal.side === "YES" ? 1 - future.p : future.p;
       const grossReturn = exit / entry - 1;
-    const netReturn = grossReturn - 0.005 / entry;
+      const netReturn = grossReturn - (COST_CENTS / 100) / entry;
-    const fadeNetReturn = fadeEntry > 0.02 && fadeEntry < 0.98 ? fadeExit / fadeEntry - 1 - 0.005 / fadeEntry : null;
+      const fadeNetReturn = fadeEntry > 0.02 && fadeEntry < 0.98
+        ? fadeExit / fadeEntry - 1 - (COST_CENTS / 100) / fadeEntry : null;
       outcomes.push({ marketId: market.id, question: market.question, category: market.category,
-      type: signal.type, side: signal.side, band: priceBand(entry), entry, exit,
+        type: signal.type, side: signal.side, band: priceBand(entry), entry, exit, horizonHours,
         grossReturn, netReturn, fadeNetReturn, hourMove: signal.hourMove, dayMove: signal.dayMove, weekMove: signal.weekMove,
         observedAt: current.t, evaluatedAt: future.t });
-    previousBucket = bucket;
+      captured = true;
+    }
+    if (captured) previousBucket = bucket;
   }
   return outcomes;
 }
@@ -127,13 +135,26 @@ function median(values) {
 }
 
 function summarize(rows, field = "netReturn") {
-  if (!rows.length) return { count: 0, mean: 0, median: 0, winRate: 0, worst: 0, best: 0 };
+  if (!rows.length) return { count: 0, markets: 0, mean: 0, median: 0, winRate: 0, worst: 0, best: 0, marketMean: 0, lower90: 0, upper90: 0 };
   const returns = rows.map((row) => row[field]).filter(Number.isFinite);
-  if (!returns.length) return { count: 0, mean: 0, median: 0, winRate: 0, worst: 0, best: 0 };
+  if (!returns.length) return { count: 0, markets: 0, mean: 0, median: 0, winRate: 0, worst: 0, best: 0, marketMean: 0, lower90: 0, upper90: 0 };
-  return { count: rows.length,
+  const marketBuckets = new Map();
+  rows.forEach((row) => {
+    const value = row[field];
+    if (!Number.isFinite(value)) return;
+    const bucket = marketBuckets.get(row.marketId) || [];
+    bucket.push(value); marketBuckets.set(row.marketId, bucket);
+  });
+  const marketReturns = [...marketBuckets.values()].map((values) => values.reduce((sum, value) => sum + value, 0) / values.length);
+  const marketMean = marketReturns.reduce((sum, value) => sum + value, 0) / Math.max(1, marketReturns.length);
+  const variance = marketReturns.length > 1
+    ? marketReturns.reduce((sum, value) => sum + (value - marketMean) ** 2, 0) / (marketReturns.length - 1) : 0;
+  const margin90 = 1.645 * Math.sqrt(variance / Math.max(1, marketReturns.length));
+  return { count: returns.length,
     mean: returns.reduce((sum, value) => sum + value, 0) / returns.length,
     median: median(returns), winRate: returns.filter((value) => value > 0).length / returns.length,
-    worst: Math.min(...returns), best: Math.max(...returns) };
+    worst: Math.min(...returns), best: Math.max(...returns), markets: marketReturns.length,
+    marketMean, lower90: marketMean - margin90, upper90: marketMean + margin90 };
 }
 
 function grouped(rows, key) {
@@ -144,8 +165,17 @@ const RULES = [
   { name: "follow_all", field: "netReturn", test: () => true },
   { name: "follow_trend", field: "netReturn", test: (row) => row.type === "trend" },
   { name: "follow_trend_no", field: "netReturn", test: (row) => row.type === "trend" && row.side === "NO" },
+  { name: "follow_trend_yes", field: "netReturn", test: (row) => row.type === "trend" && row.side === "YES" },
+  { name: "follow_trend_mid", field: "netReturn", test: (row) => row.type === "trend" && row.band === "mid" },
   { name: "follow_trend_favorites", field: "netReturn", test: (row) => row.type === "trend" && ["favorite", "heavy-favorite"].includes(row.band) },
+  { name: "follow_trend_non_longshot", field: "netReturn", test: (row) => row.type === "trend" && row.band !== "longshot" },
   { name: "follow_strong_trend", field: "netReturn", test: (row) => row.type === "trend" && Math.abs(row.dayMove) >= 0.015 && Math.abs(row.weekMove) >= 0.03 },
+  { name: "follow_moderate_trend", field: "netReturn", test: (row) => row.type === "trend" && Math.abs(row.dayMove) <= 0.03 && Math.abs(row.weekMove) <= 0.10 },
+  { name: "follow_hour_confirmed_trend", field: "netReturn", test: (row) => row.type === "trend" && Math.sign(row.hourMove) === Math.sign(row.dayMove) },
+  ...["Politics", "Sports", "Crypto", "Economy", "Pop Culture", "Other"].map((category) => ({
+    name: `follow_trend_${category.toLowerCase().replace(/\s+/g, "_")}`, field: "netReturn",
+    test: (row) => row.type === "trend" && row.category === category,
+  })),
   { name: "follow_reversal", field: "netReturn", test: (row) => row.type === "reversal" },
   { name: "fade_trend", field: "fadeNetReturn", test: (row) => row.type === "trend" },
   { name: "fade_trend_yes_move", field: "fadeNetReturn", test: (row) => row.type === "trend" && row.side === "YES" },
@@ -156,9 +186,48 @@ function evaluateRules(rows) {
   return Object.fromEntries(RULES.map((rule) => [rule.name, summarize(rows.filter(rule.test), rule.field)]));
 }
 
-const params = new URLSearchParams({ active: "true", closed: "false", archived: "false", include_tag: "true",
+function chronologicalEvaluation(rows) {
-  limit: String(MARKET_LIMIT), order: "volume24hr", ascending: "false" });
+  const ordered = [...rows].sort((a, b) => a.observedAt - b.observedAt);
-const rawMarkets = await fetchJson(`${GAMMA}/markets?${params}`);
+  const splitTime = ordered[Math.floor(ordered.length * 0.70)]?.observedAt || 0;
+  const train = ordered.filter((row) => row.observedAt < splitTime), test = ordered.filter((row) => row.observedAt >= splitTime);
+  const cut1 = ordered[Math.floor(ordered.length / 3)]?.observedAt || 0;
+  const cut2 = ordered[Math.floor(ordered.length * 2 / 3)]?.observedAt || 0;
+  const thirds = [ordered.filter((row) => row.observedAt < cut1),
+    ordered.filter((row) => row.observedAt >= cut1 && row.observedAt < cut2),
+    ordered.filter((row) => row.observedAt >= cut2)];
+  const thirdRules = thirds.map(evaluateRules), pooled = evaluateRules(ordered);
+  const robustRules = Object.fromEntries(RULES.map((rule) => {
+    const segments = thirdRules.map((result) => result[rule.name]);
+    const enoughData = segments.every((segment) => segment.count >= 20 && segment.markets >= 5);
+    const allPositive = enoughData && segments.every((segment) => segment.mean > 0 && segment.marketMean > 0);
+    const allNegative = enoughData && segments.every((segment) => segment.mean < 0 && segment.marketMean < 0);
+    return [rule.name, { enoughData, allPositive, allNegative,
+      minimumSegmentMean: Math.min(...segments.map((segment) => segment.mean)),
+      maximumSegmentMean: Math.max(...segments.map((segment) => segment.mean)), pooled: pooled[rule.name] }];
+  }));
+  return { splitTime: splitTime ? new Date(splitTime * 1000).toISOString() : null,
+    trainCount: train.length, testCount: test.length, train: evaluateRules(train), test: evaluateRules(test),
+    thirds: thirdRules, robustRules };
+}
+
+async function fetchActiveMarkets(limit) {
+  const markets = [], seen = new Set(), pageSize = 100;
+  for (let offset = 0; offset < limit; offset += pageSize) {
+    const params = new URLSearchParams({ active: "true", closed: "false", archived: "false", include_tag: "true",
+      limit: String(Math.min(pageSize, limit - offset)), offset: String(offset), order: "volume24hr", ascending: "false" });
+    const page = await fetchJson(`${GAMMA}/markets?${params}`);
+    if (!Array.isArray(page) || !page.length) break;
+    for (const market of page) {
+      const id = String(market.id || "");
+      if (!id || seen.has(id)) continue;
+      seen.add(id); markets.push(market);
+    }
+    if (page.length < Math.min(pageSize, limit - offset)) break;
+  }
+  return markets.slice(0, limit);
+}
+
+const rawMarkets = await fetchActiveMarkets(MARKET_LIMIT);
 const markets = rawMarkets.map((raw) => ({ id: String(raw.id), question: raw.question || "", category: categoryOf(raw),
   tokenId: String(parseJson(raw.clobTokenIds)[0] || "") })).filter((market) => market.id && market.tokenId);
 const histories = await mapLimit(markets, CONCURRENCY, async (market) => {
@@ -169,17 +238,20 @@ const histories = await mapLimit(markets, CONCURRENCY, async (market) => {
 });
 const successful = histories.filter((result) => result && !result.error && result.points.length);
 const outcomes = successful.flatMap((result) => result.outcomes);
-const ordered = [...outcomes].sort((a, b) => a.observedAt - b.observedAt);
+const primaryHorizon = HORIZONS.includes(12) ? 12 : HORIZONS[0];
-const splitTime = ordered[Math.floor(ordered.length * 0.70)]?.observedAt || 0;
+const primaryOutcomes = outcomes.filter((row) => row.horizonHours === primaryHorizon);
-const train = ordered.filter((row) => row.observedAt < splitTime), test = ordered.filter((row) => row.observedAt >= splitTime);
 const report = {
   generatedAt: new Date().toISOString(), marketLimit: MARKET_LIMIT, marketsWithHistory: successful.length,
-  methodology: { horizonHours: 12, observationBucketHours: 6, historyInterval: "1m", fidelityMinutes: 60,
+  methodology: { horizonHours: HORIZONS, primaryHorizon, observationBucketHours: 6, historyInterval: "1m", fidelityMinutes: 60,
-    estimatedRoundTripCostCents: 0.5, note: "Current active-market selection and current category tags are a survivorship-biased proxy; signal inputs and future marks are time-ordered without lookahead." },
+    estimatedRoundTripCostCents: COST_CENTS, clusterUnit: "market",
-  overall: summarize(outcomes), byType: grouped(outcomes, "type"), byCategory: grouped(outcomes, "category"),
+    note: "Current active-market selection and current category tags are a survivorship-biased proxy; signal inputs and future marks are time-ordered without lookahead. Confidence intervals use per-market means to reduce repeated-observation distortion." },
-  byBand: grouped(outcomes, "band"), bySide: grouped(outcomes, "side"),
+  overall: summarize(primaryOutcomes), byType: grouped(primaryOutcomes, "type"), byCategory: grouped(primaryOutcomes, "category"),
-  chronologicalSplit: { splitTime: splitTime ? new Date(splitTime * 1000).toISOString() : null,
+  byBand: grouped(primaryOutcomes, "band"), bySide: grouped(primaryOutcomes, "side"),
-    trainCount: train.length, testCount: test.length, train: evaluateRules(train), test: evaluateRules(test) },
+  chronologicalSplit: chronologicalEvaluation(primaryOutcomes),
+  horizons: Object.fromEntries(HORIZONS.map((horizon) => {
+    const rows = outcomes.filter((row) => row.horizonHours === horizon);
+    return [horizon, { overall: summarize(rows), chronological: chronologicalEvaluation(rows) }];
+  })),
   failures: histories.filter((result) => result?.error).length,
 };
 console.log(JSON.stringify(report, null, 2));