From 38945b2045964ba0c99401427b868d048f28146d Mon Sep 17 00:00:00 2001 From: Theodore Song Date: Tue, 18 Aug 2026 09:50:46 -0400 Subject: [PATCH] Expand chronological signal audit --- README.md | 16 ++++- scripts/evaluate-signals.mjs | 130 +++++++++++++++++++++++++++-------- 2 files changed, 114 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index 14eb01c..9a4be80 100644 --- a/README.md +++ b/README.md @@ -47,9 +47,12 @@ clean live comparison. Run `npm run evaluate:signals` to test the price-signal rules against one month of hourly Polymarket history. The evaluator forms signals only from prior -one-hour, one-day, and one-week prices, marks them 12 hours later, applies a -conservative half-cent cost estimate, and reports a chronological 70/30 split. -Set `EVAL_MARKETS` or `EVAL_CONCURRENCY` to change the default 80-market run. +one-hour, one-day, and one-week prices, marks them 6, 12, 24, and 72 hours later, +applies a conservative half-cent cost estimate, and reports a chronological +70/30 split plus three consecutive time segments. Results are also clustered by +market so repeated observations from one contract cannot masquerade as broad +evidence. Set `EVAL_MARKETS`, `EVAL_CONCURRENCY`, `EVAL_HORIZONS`, or +`EVAL_COST_CENTS` to change the audit. The first 80-market audit found that reversal signals lost 4.34% on average in both chronological partitions, while crypto and longshot samples were also negative overall. Engine v37 therefore blocks reversal entries outside the fixed @@ -57,6 +60,13 @@ negative overall. Engine v37 therefore blocks reversal entries outside the fixed It does not boost any rule from this audit because no positive rule was robust across the chronological split. +A corrected 200-market audit paged through 197 markets with usable history and +1,912 twelve-hour outcomes. Reversals remained negative in every chronological +segment and averaged -4.13%. Sports trends were negative in train and test and +averaged -3.53% at 72 hours. Politics trends were the sole cohort with positive +row-level returns in all three 72-hour segments, but its market-cluster interval +still crossed zero; that supports a longer hold test, not a larger entry bet. + Paper accounts created with a password are also saved through the backend, so a user can log in from another device and see the same paper portfolio, activity, and value history. Passwordless paper accounts remain local-only. diff --git a/scripts/evaluate-signals.mjs b/scripts/evaluate-signals.mjs index c39d297..3a6533e 100644 --- a/scripts/evaluate-signals.mjs +++ b/scripts/evaluate-signals.mjs @@ -1,7 +1,10 @@ const GAMMA = "https://gamma-api.polymarket.com"; const CLOB = "https://clob.polymarket.com"; -const MARKET_LIMIT = Math.max(10, Math.min(200, Number(process.env.EVAL_MARKETS || 80))); +const MARKET_LIMIT = Math.max(10, Math.min(500, Number(process.env.EVAL_MARKETS || 80))); const CONCURRENCY = Math.max(1, Math.min(12, Number(process.env.EVAL_CONCURRENCY || 6))); +const HORIZONS = [...new Set(String(process.env.EVAL_HORIZONS || "6,12,24,72").split(",") + .map(Number).filter((value) => Number.isFinite(value) && value >= 1 && value <= 168))].sort((a, b) => a - b); +const COST_CENTS = Math.max(0, Math.min(5, Number(process.env.EVAL_COST_CENTS || 0.5))); const HOUR = 3600; const CATEGORY_RULES = [ @@ -102,21 +105,26 @@ function evaluateMarket(market, points) { if (bucket === previousBucket || current.p < 0.08 || current.p > 0.92) continue; const signal = signalAt(points, index); if (!signal) continue; - const future = atOrAfter(points, current.t + 12 * HOUR); - if (!future || future.t - (current.t + 12 * HOUR) > 3 * HOUR) continue; const entry = signal.side === "YES" ? current.p : 1 - current.p; - const exit = signal.side === "YES" ? future.p : 1 - future.p; const fadeEntry = signal.side === "YES" ? 1 - current.p : current.p; - const fadeExit = signal.side === "YES" ? 1 - future.p : future.p; if (entry <= 0.02 || entry >= 0.98) continue; - const grossReturn = exit / entry - 1; - const netReturn = grossReturn - 0.005 / entry; - const fadeNetReturn = fadeEntry > 0.02 && fadeEntry < 0.98 ? fadeExit / fadeEntry - 1 - 0.005 / fadeEntry : null; - outcomes.push({ marketId: market.id, question: market.question, category: market.category, - type: signal.type, side: signal.side, band: priceBand(entry), entry, exit, - grossReturn, netReturn, fadeNetReturn, hourMove: signal.hourMove, dayMove: signal.dayMove, weekMove: signal.weekMove, - observedAt: current.t, evaluatedAt: future.t }); - previousBucket = bucket; + let captured = false; + for (const horizonHours of HORIZONS) { + const future = atOrAfter(points, current.t + horizonHours * HOUR); + if (!future || future.t - (current.t + horizonHours * HOUR) > 3 * HOUR) continue; + const exit = signal.side === "YES" ? future.p : 1 - future.p; + const fadeExit = signal.side === "YES" ? 1 - future.p : future.p; + const grossReturn = exit / entry - 1; + const netReturn = grossReturn - (COST_CENTS / 100) / entry; + const fadeNetReturn = fadeEntry > 0.02 && fadeEntry < 0.98 + ? fadeExit / fadeEntry - 1 - (COST_CENTS / 100) / fadeEntry : null; + outcomes.push({ marketId: market.id, question: market.question, category: market.category, + type: signal.type, side: signal.side, band: priceBand(entry), entry, exit, horizonHours, + grossReturn, netReturn, fadeNetReturn, hourMove: signal.hourMove, dayMove: signal.dayMove, weekMove: signal.weekMove, + observedAt: current.t, evaluatedAt: future.t }); + captured = true; + } + if (captured) previousBucket = bucket; } return outcomes; } @@ -127,13 +135,26 @@ function median(values) { } function summarize(rows, field = "netReturn") { - if (!rows.length) return { count: 0, mean: 0, median: 0, winRate: 0, worst: 0, best: 0 }; + if (!rows.length) return { count: 0, markets: 0, mean: 0, median: 0, winRate: 0, worst: 0, best: 0, marketMean: 0, lower90: 0, upper90: 0 }; const returns = rows.map((row) => row[field]).filter(Number.isFinite); - if (!returns.length) return { count: 0, mean: 0, median: 0, winRate: 0, worst: 0, best: 0 }; - return { count: rows.length, + if (!returns.length) return { count: 0, markets: 0, mean: 0, median: 0, winRate: 0, worst: 0, best: 0, marketMean: 0, lower90: 0, upper90: 0 }; + const marketBuckets = new Map(); + rows.forEach((row) => { + const value = row[field]; + if (!Number.isFinite(value)) return; + const bucket = marketBuckets.get(row.marketId) || []; + bucket.push(value); marketBuckets.set(row.marketId, bucket); + }); + const marketReturns = [...marketBuckets.values()].map((values) => values.reduce((sum, value) => sum + value, 0) / values.length); + const marketMean = marketReturns.reduce((sum, value) => sum + value, 0) / Math.max(1, marketReturns.length); + const variance = marketReturns.length > 1 + ? marketReturns.reduce((sum, value) => sum + (value - marketMean) ** 2, 0) / (marketReturns.length - 1) : 0; + const margin90 = 1.645 * Math.sqrt(variance / Math.max(1, marketReturns.length)); + return { count: returns.length, mean: returns.reduce((sum, value) => sum + value, 0) / returns.length, median: median(returns), winRate: returns.filter((value) => value > 0).length / returns.length, - worst: Math.min(...returns), best: Math.max(...returns) }; + worst: Math.min(...returns), best: Math.max(...returns), markets: marketReturns.length, + marketMean, lower90: marketMean - margin90, upper90: marketMean + margin90 }; } function grouped(rows, key) { @@ -144,8 +165,17 @@ const RULES = [ { name: "follow_all", field: "netReturn", test: () => true }, { name: "follow_trend", field: "netReturn", test: (row) => row.type === "trend" }, { name: "follow_trend_no", field: "netReturn", test: (row) => row.type === "trend" && row.side === "NO" }, + { name: "follow_trend_yes", field: "netReturn", test: (row) => row.type === "trend" && row.side === "YES" }, + { name: "follow_trend_mid", field: "netReturn", test: (row) => row.type === "trend" && row.band === "mid" }, { name: "follow_trend_favorites", field: "netReturn", test: (row) => row.type === "trend" && ["favorite", "heavy-favorite"].includes(row.band) }, + { name: "follow_trend_non_longshot", field: "netReturn", test: (row) => row.type === "trend" && row.band !== "longshot" }, { name: "follow_strong_trend", field: "netReturn", test: (row) => row.type === "trend" && Math.abs(row.dayMove) >= 0.015 && Math.abs(row.weekMove) >= 0.03 }, + { name: "follow_moderate_trend", field: "netReturn", test: (row) => row.type === "trend" && Math.abs(row.dayMove) <= 0.03 && Math.abs(row.weekMove) <= 0.10 }, + { name: "follow_hour_confirmed_trend", field: "netReturn", test: (row) => row.type === "trend" && Math.sign(row.hourMove) === Math.sign(row.dayMove) }, + ...["Politics", "Sports", "Crypto", "Economy", "Pop Culture", "Other"].map((category) => ({ + name: `follow_trend_${category.toLowerCase().replace(/\s+/g, "_")}`, field: "netReturn", + test: (row) => row.type === "trend" && row.category === category, + })), { name: "follow_reversal", field: "netReturn", test: (row) => row.type === "reversal" }, { name: "fade_trend", field: "fadeNetReturn", test: (row) => row.type === "trend" }, { name: "fade_trend_yes_move", field: "fadeNetReturn", test: (row) => row.type === "trend" && row.side === "YES" }, @@ -156,9 +186,48 @@ function evaluateRules(rows) { return Object.fromEntries(RULES.map((rule) => [rule.name, summarize(rows.filter(rule.test), rule.field)])); } -const params = new URLSearchParams({ active: "true", closed: "false", archived: "false", include_tag: "true", - limit: String(MARKET_LIMIT), order: "volume24hr", ascending: "false" }); -const rawMarkets = await fetchJson(`${GAMMA}/markets?${params}`); +function chronologicalEvaluation(rows) { + const ordered = [...rows].sort((a, b) => a.observedAt - b.observedAt); + const splitTime = ordered[Math.floor(ordered.length * 0.70)]?.observedAt || 0; + const train = ordered.filter((row) => row.observedAt < splitTime), test = ordered.filter((row) => row.observedAt >= splitTime); + const cut1 = ordered[Math.floor(ordered.length / 3)]?.observedAt || 0; + const cut2 = ordered[Math.floor(ordered.length * 2 / 3)]?.observedAt || 0; + const thirds = [ordered.filter((row) => row.observedAt < cut1), + ordered.filter((row) => row.observedAt >= cut1 && row.observedAt < cut2), + ordered.filter((row) => row.observedAt >= cut2)]; + const thirdRules = thirds.map(evaluateRules), pooled = evaluateRules(ordered); + const robustRules = Object.fromEntries(RULES.map((rule) => { + const segments = thirdRules.map((result) => result[rule.name]); + const enoughData = segments.every((segment) => segment.count >= 20 && segment.markets >= 5); + const allPositive = enoughData && segments.every((segment) => segment.mean > 0 && segment.marketMean > 0); + const allNegative = enoughData && segments.every((segment) => segment.mean < 0 && segment.marketMean < 0); + return [rule.name, { enoughData, allPositive, allNegative, + minimumSegmentMean: Math.min(...segments.map((segment) => segment.mean)), + maximumSegmentMean: Math.max(...segments.map((segment) => segment.mean)), pooled: pooled[rule.name] }]; + })); + return { splitTime: splitTime ? new Date(splitTime * 1000).toISOString() : null, + trainCount: train.length, testCount: test.length, train: evaluateRules(train), test: evaluateRules(test), + thirds: thirdRules, robustRules }; +} + +async function fetchActiveMarkets(limit) { + const markets = [], seen = new Set(), pageSize = 100; + for (let offset = 0; offset < limit; offset += pageSize) { + const params = new URLSearchParams({ active: "true", closed: "false", archived: "false", include_tag: "true", + limit: String(Math.min(pageSize, limit - offset)), offset: String(offset), order: "volume24hr", ascending: "false" }); + const page = await fetchJson(`${GAMMA}/markets?${params}`); + if (!Array.isArray(page) || !page.length) break; + for (const market of page) { + const id = String(market.id || ""); + if (!id || seen.has(id)) continue; + seen.add(id); markets.push(market); + } + if (page.length < Math.min(pageSize, limit - offset)) break; + } + return markets.slice(0, limit); +} + +const rawMarkets = await fetchActiveMarkets(MARKET_LIMIT); const markets = rawMarkets.map((raw) => ({ id: String(raw.id), question: raw.question || "", category: categoryOf(raw), tokenId: String(parseJson(raw.clobTokenIds)[0] || "") })).filter((market) => market.id && market.tokenId); const histories = await mapLimit(markets, CONCURRENCY, async (market) => { @@ -169,17 +238,20 @@ const histories = await mapLimit(markets, CONCURRENCY, async (market) => { }); const successful = histories.filter((result) => result && !result.error && result.points.length); const outcomes = successful.flatMap((result) => result.outcomes); -const ordered = [...outcomes].sort((a, b) => a.observedAt - b.observedAt); -const splitTime = ordered[Math.floor(ordered.length * 0.70)]?.observedAt || 0; -const train = ordered.filter((row) => row.observedAt < splitTime), test = ordered.filter((row) => row.observedAt >= splitTime); +const primaryHorizon = HORIZONS.includes(12) ? 12 : HORIZONS[0]; +const primaryOutcomes = outcomes.filter((row) => row.horizonHours === primaryHorizon); const report = { generatedAt: new Date().toISOString(), marketLimit: MARKET_LIMIT, marketsWithHistory: successful.length, - methodology: { horizonHours: 12, observationBucketHours: 6, historyInterval: "1m", fidelityMinutes: 60, - estimatedRoundTripCostCents: 0.5, note: "Current active-market selection and current category tags are a survivorship-biased proxy; signal inputs and future marks are time-ordered without lookahead." }, - overall: summarize(outcomes), byType: grouped(outcomes, "type"), byCategory: grouped(outcomes, "category"), - byBand: grouped(outcomes, "band"), bySide: grouped(outcomes, "side"), - chronologicalSplit: { splitTime: splitTime ? new Date(splitTime * 1000).toISOString() : null, - trainCount: train.length, testCount: test.length, train: evaluateRules(train), test: evaluateRules(test) }, + methodology: { horizonHours: HORIZONS, primaryHorizon, observationBucketHours: 6, historyInterval: "1m", fidelityMinutes: 60, + estimatedRoundTripCostCents: COST_CENTS, clusterUnit: "market", + note: "Current active-market selection and current category tags are a survivorship-biased proxy; signal inputs and future marks are time-ordered without lookahead. Confidence intervals use per-market means to reduce repeated-observation distortion." }, + overall: summarize(primaryOutcomes), byType: grouped(primaryOutcomes, "type"), byCategory: grouped(primaryOutcomes, "category"), + byBand: grouped(primaryOutcomes, "band"), bySide: grouped(primaryOutcomes, "side"), + chronologicalSplit: chronologicalEvaluation(primaryOutcomes), + horizons: Object.fromEntries(HORIZONS.map((horizon) => { + const rows = outcomes.filter((row) => row.horizonHours === horizon); + return [horizon, { overall: summarize(rows), chronological: chronologicalEvaluation(rows) }]; + })), failures: histories.filter((result) => result?.error).length, }; console.log(JSON.stringify(report, null, 2));