diff --git a/README.md b/README.md index 1538289..946b50c 100644 --- a/README.md +++ b/README.md @@ -70,6 +70,21 @@ Engine v38 gives Politics trend positions that 72-hour observation window before ordinary signal exits. Stops, profit locks, settlement handling, and risk-budget reductions remain immediate. +Run `npm run evaluate:settlements` to evaluate fixed decisions made 1, 3, 7, +14, 30, and 90 days before known binary settlements. The audit uses one +observation per resolved market and horizon, includes losing contracts at zero, +applies the same half-cent cost assumption, clusters related contracts by event, +and requires positive event-clustered confidence bounds in train and test plus +positive results in three chronological segments before it calls a settlement +cohort robust. Environment variables beginning with +`SETTLEMENT_` control its market count, concurrency, horizons, and cost. + +The first event-clustered run loaded 498 of the 500 highest-volume resolved +markets. No side, price band, category, or 1-90 day holding rule passed the +required train/test confidence checks. In particular, older YES/underdog gains +reversed in the recent test segment. The engine therefore does not install a +static settlement-direction boost from this audit. + Paper accounts created with a password are also saved through the backend, so a user can log in from another device and see the same paper portfolio, activity, and value history. Passwordless paper accounts remain local-only. diff --git a/package.json b/package.json index 5b872b1..baa209f 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,7 @@ { "scripts": { - "evaluate:signals": "node scripts/evaluate-signals.mjs" + "evaluate:signals": "node scripts/evaluate-signals.mjs", + "evaluate:settlements": "node scripts/evaluate-settlements.mjs" }, "dependencies": { "@neondatabase/serverless": "^1.1.0", diff --git a/scripts/evaluate-settlements.mjs b/scripts/evaluate-settlements.mjs new file mode 100644 index 0000000..e9c4006 --- /dev/null +++ b/scripts/evaluate-settlements.mjs @@ -0,0 +1,211 @@ +const GAMMA = "https://gamma-api.polymarket.com"; +const CLOB = "https://clob.polymarket.com"; +const MARKET_LIMIT = Math.max(20, Math.min(500, Number(process.env.SETTLEMENT_MARKETS || 200))); +const CONCURRENCY = Math.max(1, Math.min(12, Number(process.env.SETTLEMENT_CONCURRENCY || 6))); +const HORIZON_DAYS = [...new Set(String(process.env.SETTLEMENT_HORIZONS || "1,3,7,14,30,90").split(",") + .map(Number).filter((value) => Number.isFinite(value) && value >= 1 && value <= 365))].sort((a, b) => a - b); +const COST_CENTS = Math.max(0, Math.min(5, Number(process.env.SETTLEMENT_COST_CENTS || 0.5))); +const DAY = 86400; + +function parseJson(value) { + if (Array.isArray(value)) return value; + try { return JSON.parse(value || "[]"); } catch { return []; } +} + +function toTimestamp(value) { + const parsed = Date.parse(String(value || "").replace(" ", "T").replace(/\+00$/, "Z")); + return Number.isFinite(parsed) ? parsed / 1000 : null; +} + +function categoryOf(raw) { + const text = `${raw.category || ""} ${raw.question || ""} ${(raw.events || []).flatMap((event) => event.tags || []) + .map((tag) => tag.slug || tag.label || "").join(" ")}`.toLowerCase(); + if (/\b(election|president|politic|senate|congress|parliament|minister|governor|government|nominee|primary)\b/.test(text)) return "Politics"; + if (/\b(bitcoin|crypto|ethereum|btc|eth|solana|xrp|token|stablecoin)\b/.test(text)) return "Crypto"; + if (/\b(nba|nfl|nhl|mlb|soccer|football|baseball|basketball|tennis|ufc|boxing|championship|match|game|tournament|league)\b/.test(text)) return "Sports"; + if (/\b(fed|inflation|gdp|recession|stock|company|economy|tariff|interest rate|unemployment|earnings)\b/.test(text)) return "Economy"; + if (/\b(movie|music|album|box office|television|celebrity|award|gaming|youtube|stream)\b/.test(text)) return "Pop Culture"; + return "Other"; +} + +async function fetchJson(url, options = {}, attempts = 3) { + let lastError; + for (let attempt = 0; attempt < attempts; attempt++) { + try { + const response = await fetch(url, { ...options, signal: AbortSignal.timeout(25000), + headers: { accept: "application/json", ...(options.headers || {}) } }); + if (response.ok) return response.json(); + lastError = new Error(`${response.status} ${response.statusText}`); + if (response.status !== 429 && response.status < 500) break; + } catch (error) { lastError = error; } + await new Promise((resolve) => setTimeout(resolve, 500 * (attempt + 1))); + } + throw lastError || new Error("request failed"); +} + +async function mapLimit(items, limit, task) { + const output = new Array(items.length); + let cursor = 0; + async function worker() { + while (cursor < items.length) { + const index = cursor++; + try { output[index] = await task(items[index], index); } + catch (error) { output[index] = { error: error.message }; } + } + } + await Promise.all(Array.from({ length: Math.min(limit, items.length) }, worker)); + return output; +} + +function atOrBefore(points, target) { + let lo = 0, hi = points.length - 1, answer = null; + while (lo <= hi) { + const mid = (lo + hi) >> 1; + if (points[mid].t <= target) { answer = points[mid]; lo = mid + 1; } + else hi = mid - 1; + } + return answer; +} + +function priceBand(price) { + if (price < 0.25) return "longshot"; + if (price < 0.55) return "mid"; + if (price < 0.78) return "favorite"; + return "heavy-favorite"; +} + +async function fetchResolvedMarkets(limit) { + const raw = [], seen = new Set(), pageSize = 100; + for (let offset = 0; raw.length < limit && offset < limit * 3; offset += pageSize) { + const params = new URLSearchParams({ closed: "true", order: "volumeNum", ascending: "false", + limit: String(pageSize), offset: String(offset) }); + const page = await fetchJson(`${GAMMA}/markets?${params}`); + if (!Array.isArray(page) || !page.length) break; + for (const market of page) { + const id = String(market.id || ""), outcomes = parseJson(market.outcomePrices).map(Number); + const tokens = parseJson(market.clobTokenIds), closedAt = toTimestamp(market.closedTime || market.endDate); + const resolved = outcomes.length === 2 && outcomes.every(Number.isFinite) + && ((outcomes[0] >= 0.99 && outcomes[1] <= 0.01) || (outcomes[1] >= 0.99 && outcomes[0] <= 0.01)); + if (!id || seen.has(id) || !resolved || tokens.length !== 2 || !closedAt) continue; + seen.add(id); raw.push({ id, question: market.question || "", category: categoryOf(market), + eventId: String(market.events?.[0]?.id || id), + tokenId: String(tokens[0]), finalYes: outcomes[0] >= 0.99 ? 1 : 0, closedAt, + volume: Number(market.volumeNum || market.volume || 0) }); + if (raw.length >= limit) break; + } + if (page.length < pageSize) break; + } + return raw; +} + +function evaluateMarket(market, points) { + const rows = []; + for (const horizonDays of HORIZON_DAYS) { + const target = market.closedAt - horizonDays * DAY, point = atOrBefore(points, target); + const maximumStaleness = Math.max(36 * 3600, horizonDays * DAY * 0.15); + if (!point || target - point.t > maximumStaleness || point.p <= 0.03 || point.p >= 0.97) continue; + const yesEntry = point.p, noEntry = 1 - point.p, favoriteSide = yesEntry >= noEntry ? "YES" : "NO"; + const winningSide = market.finalYes ? "YES" : "NO"; + for (const side of ["YES", "NO"]) { + const entry = side === "YES" ? yesEntry : noEntry, final = side === winningSide ? 1 : 0; + const netReturn = final / entry - 1 - (COST_CENTS / 100) / entry; + rows.push({ marketId: market.id, eventId: market.eventId, question: market.question, category: market.category, closedAt: market.closedAt, + horizonDays, side, favorite: side === favoriteSide, winner: side === winningSide, + entry, band: priceBand(entry), netReturn }); + } + } + return rows; +} + +function median(values) { + const sorted = [...values].sort((a, b) => a - b), mid = Math.floor(sorted.length / 2); + return sorted.length % 2 ? sorted[mid] : (sorted[mid - 1] + sorted[mid]) / 2; +} + +function summarize(rows) { + if (!rows.length) return { count: 0, events: 0, mean: 0, median: 0, winRate: 0, lower90: 0, upper90: 0, + eventMean: 0, eventLower90: 0, eventUpper90: 0, worst: 0, best: 0 }; + const values = rows.map((row) => row.netReturn), mean = values.reduce((sum, value) => sum + value, 0) / values.length; + const variance = values.length > 1 ? values.reduce((sum, value) => sum + (value - mean) ** 2, 0) / (values.length - 1) : 0; + const margin90 = 1.645 * Math.sqrt(variance / values.length); + const eventBuckets = new Map(); + rows.forEach((row) => { + const bucket = eventBuckets.get(row.eventId) || []; + bucket.push(row.netReturn); eventBuckets.set(row.eventId, bucket); + }); + const eventReturns = [...eventBuckets.values()].map((bucket) => bucket.reduce((sum, value) => sum + value, 0) / bucket.length); + const eventMean = eventReturns.reduce((sum, value) => sum + value, 0) / eventReturns.length; + const eventVariance = eventReturns.length > 1 + ? eventReturns.reduce((sum, value) => sum + (value - eventMean) ** 2, 0) / (eventReturns.length - 1) : 0; + const eventMargin90 = 1.645 * Math.sqrt(eventVariance / eventReturns.length); + return { count: values.length, mean, median: median(values), winRate: rows.filter((row) => row.winner).length / rows.length, + lower90: mean - margin90, upper90: mean + margin90, events: eventReturns.length, + eventMean, eventLower90: eventMean - eventMargin90, eventUpper90: eventMean + eventMargin90, + worst: Math.min(...values), best: Math.max(...values) }; +} + +const RULES = [ + { name: "buy_favorite", test: (row) => row.favorite }, + { name: "buy_heavy_favorite", test: (row) => row.favorite && row.entry >= 0.78 }, + { name: "buy_60_78_favorite", test: (row) => row.favorite && row.entry >= 0.60 && row.entry < 0.78 }, + { name: "buy_55_60_favorite", test: (row) => row.favorite && row.entry >= 0.55 && row.entry < 0.60 }, + { name: "buy_underdog", test: (row) => !row.favorite }, + { name: "buy_yes", test: (row) => row.side === "YES" }, + { name: "buy_no", test: (row) => row.side === "NO" }, + ...["Politics", "Sports", "Crypto", "Economy", "Pop Culture", "Other"].flatMap((category) => [ + { name: `buy_favorite_${category.toLowerCase().replace(/\s+/g, "_")}`, test: (row) => row.favorite && row.category === category }, + { name: `buy_underdog_${category.toLowerCase().replace(/\s+/g, "_")}`, test: (row) => !row.favorite && row.category === category }, + ]), +]; + +function evaluateRules(rows) { + return Object.fromEntries(RULES.map((rule) => [rule.name, summarize(rows.filter(rule.test))])); +} + +function chronologicalEvaluation(rows) { + const ordered = [...rows].sort((a, b) => a.closedAt - b.closedAt); + const splitTime = ordered[Math.floor(ordered.length * 0.70)]?.closedAt || 0; + const cut1 = ordered[Math.floor(ordered.length / 3)]?.closedAt || 0; + const cut2 = ordered[Math.floor(ordered.length * 2 / 3)]?.closedAt || 0; + const train = ordered.filter((row) => row.closedAt < splitTime), test = ordered.filter((row) => row.closedAt >= splitTime); + const thirds = [ordered.filter((row) => row.closedAt < cut1), + ordered.filter((row) => row.closedAt >= cut1 && row.closedAt < cut2), + ordered.filter((row) => row.closedAt >= cut2)]; + const pooled = evaluateRules(ordered), trainRules = evaluateRules(train), testRules = evaluateRules(test), thirdRules = thirds.map(evaluateRules); + const robustRules = Object.fromEntries(RULES.map((rule) => { + const segments = thirdRules.map((result) => result[rule.name]), all = pooled[rule.name]; + const enoughData = all.events >= 15 && segments.every((segment) => segment.count >= 15 && segment.events >= 5) + && trainRules[rule.name].count >= 30 && trainRules[rule.name].events >= 10 + && testRules[rule.name].count >= 15 && testRules[rule.name].events >= 5; + const allPositive = enoughData && all.eventLower90 > 0 && trainRules[rule.name].eventLower90 > 0 + && testRules[rule.name].eventLower90 > 0 && segments.every((segment) => segment.mean > 0 && segment.eventMean > 0); + const allNegative = enoughData && all.eventUpper90 < 0 && trainRules[rule.name].eventUpper90 < 0 + && testRules[rule.name].eventUpper90 < 0 && segments.every((segment) => segment.mean < 0 && segment.eventMean < 0); + return [rule.name, { enoughData, allPositive, allNegative, pooled: all, train: trainRules[rule.name], test: testRules[rule.name], segments }]; + })); + return { splitTime: splitTime ? new Date(splitTime * 1000).toISOString() : null, trainCount: train.length, + testCount: test.length, train: trainRules, test: testRules, thirds: thirdRules, robustRules }; +} + +const markets = await fetchResolvedMarkets(MARKET_LIMIT); +const histories = await mapLimit(markets, CONCURRENCY, async (market) => { + const data = await fetchJson(`${CLOB}/prices-history?market=${encodeURIComponent(market.tokenId)}&interval=max&fidelity=1440`); + const points = (data.history || []).map((point) => ({ t: Number(point.t), p: Number(point.p) })) + .filter((point) => Number.isFinite(point.t) && Number.isFinite(point.p)).sort((a, b) => a.t - b.t); + return { market, points, rows: evaluateMarket(market, points) }; +}); +const successful = histories.filter((result) => result && !result.error && result.points.length); +const rows = successful.flatMap((result) => result.rows); +const report = { + generatedAt: new Date().toISOString(), requestedMarkets: MARKET_LIMIT, resolvedMarkets: markets.length, + marketsWithHistory: successful.length, failures: histories.filter((result) => result?.error).length, + methodology: { horizonDays: HORIZON_DAYS, estimatedRoundTripCostCents: COST_CENTS, + historyFidelityMinutes: 1440, + clusterUnit: "event", + note: "Each rule uses a daily price timestamp at or before the decision horizon and a subsequently published binary settlement. Confidence bounds cluster related markets by event. Markets are selected by resolved volume, so results still carry historical-selection and execution-model limitations." }, + horizons: Object.fromEntries(HORIZON_DAYS.map((horizon) => { + const horizonRows = rows.filter((row) => row.horizonDays === horizon); + return [horizon, { observations: horizonRows.length / 2, chronological: chronologicalEvaluation(horizonRows) }]; + })), +}; +console.log(JSON.stringify(report, null, 2));