mirror of
https://github.com/theodore-song/polymarket-analyst.git
synced 2026-08-19 18:48:10 +00:00
Expand chronological signal audit
This commit is contained in:
Theodore Song
@@ -47,9 +47,12 @@ clean live comparison.
|
||||
|
||||
Run `npm run evaluate:signals` to test the price-signal rules against one month
|
||||
of hourly Polymarket history. The evaluator forms signals only from prior
|
||||
one-hour, one-day, and one-week prices, marks them 12 hours later, applies a
|
||||
conservative half-cent cost estimate, and reports a chronological 70/30 split.
|
||||
Set `EVAL_MARKETS` or `EVAL_CONCURRENCY` to change the default 80-market run.
|
||||
one-hour, one-day, and one-week prices, marks them 6, 12, 24, and 72 hours later,
|
||||
applies a conservative half-cent cost estimate, and reports a chronological
|
||||
70/30 split plus three consecutive time segments. Results are also clustered by
|
||||
market so repeated observations from one contract cannot masquerade as broad
|
||||
evidence. Set `EVAL_MARKETS`, `EVAL_CONCURRENCY`, `EVAL_HORIZONS`, or
|
||||
`EVAL_COST_CENTS` to change the audit.
|
||||
The first 80-market audit found that reversal signals lost 4.34% on average in
|
||||
both chronological partitions, while crypto and longshot samples were also
|
||||
negative overall. Engine v37 therefore blocks reversal entries outside the fixed
|
||||
@@ -57,6 +60,13 @@ negative overall. Engine v37 therefore blocks reversal entries outside the fixed
|
||||
It does not boost any rule from this audit because no positive rule was robust
|
||||
across the chronological split.
|
||||
|
||||
A corrected 200-market audit paged through 197 markets with usable history and
|
||||
1,912 twelve-hour outcomes. Reversals remained negative in every chronological
|
||||
segment and averaged -4.13%. Sports trends were negative in train and test and
|
||||
averaged -3.53% at 72 hours. Politics trends were the sole cohort with positive
|
||||
row-level returns in all three 72-hour segments, but its market-cluster interval
|
||||
still crossed zero; that supports a longer hold test, not a larger entry bet.
|
||||
|
||||
Paper accounts created with a password are also saved through the backend, so a
|
||||
user can log in from another device and see the same paper portfolio, activity,
|
||||
and value history. Passwordless paper accounts remain local-only.
|
||||
|
||||
+101
-29
@@ -1,7 +1,10 @@
|
||||
const GAMMA = "https://gamma-api.polymarket.com";
|
||||
const CLOB = "https://clob.polymarket.com";
|
||||
const MARKET_LIMIT = Math.max(10, Math.min(200, Number(process.env.EVAL_MARKETS || 80)));
|
||||
const MARKET_LIMIT = Math.max(10, Math.min(500, Number(process.env.EVAL_MARKETS || 80)));
|
||||
const CONCURRENCY = Math.max(1, Math.min(12, Number(process.env.EVAL_CONCURRENCY || 6)));
|
||||
const HORIZONS = [...new Set(String(process.env.EVAL_HORIZONS || "6,12,24,72").split(",")
|
||||
.map(Number).filter((value) => Number.isFinite(value) && value >= 1 && value <= 168))].sort((a, b) => a - b);
|
||||
const COST_CENTS = Math.max(0, Math.min(5, Number(process.env.EVAL_COST_CENTS || 0.5)));
|
||||
const HOUR = 3600;
|
||||
|
||||
const CATEGORY_RULES = [
|
||||
@@ -102,21 +105,26 @@ function evaluateMarket(market, points) {
|
||||
if (bucket === previousBucket || current.p < 0.08 || current.p > 0.92) continue;
|
||||
const signal = signalAt(points, index);
|
||||
if (!signal) continue;
|
||||
const future = atOrAfter(points, current.t + 12 * HOUR);
|
||||
if (!future || future.t - (current.t + 12 * HOUR) > 3 * HOUR) continue;
|
||||
const entry = signal.side === "YES" ? current.p : 1 - current.p;
|
||||
const exit = signal.side === "YES" ? future.p : 1 - future.p;
|
||||
const fadeEntry = signal.side === "YES" ? 1 - current.p : current.p;
|
||||
const fadeExit = signal.side === "YES" ? 1 - future.p : future.p;
|
||||
if (entry <= 0.02 || entry >= 0.98) continue;
|
||||
const grossReturn = exit / entry - 1;
|
||||
const netReturn = grossReturn - 0.005 / entry;
|
||||
const fadeNetReturn = fadeEntry > 0.02 && fadeEntry < 0.98 ? fadeExit / fadeEntry - 1 - 0.005 / fadeEntry : null;
|
||||
outcomes.push({ marketId: market.id, question: market.question, category: market.category,
|
||||
type: signal.type, side: signal.side, band: priceBand(entry), entry, exit,
|
||||
grossReturn, netReturn, fadeNetReturn, hourMove: signal.hourMove, dayMove: signal.dayMove, weekMove: signal.weekMove,
|
||||
observedAt: current.t, evaluatedAt: future.t });
|
||||
previousBucket = bucket;
|
||||
let captured = false;
|
||||
for (const horizonHours of HORIZONS) {
|
||||
const future = atOrAfter(points, current.t + horizonHours * HOUR);
|
||||
if (!future || future.t - (current.t + horizonHours * HOUR) > 3 * HOUR) continue;
|
||||
const exit = signal.side === "YES" ? future.p : 1 - future.p;
|
||||
const fadeExit = signal.side === "YES" ? 1 - future.p : future.p;
|
||||
const grossReturn = exit / entry - 1;
|
||||
const netReturn = grossReturn - (COST_CENTS / 100) / entry;
|
||||
const fadeNetReturn = fadeEntry > 0.02 && fadeEntry < 0.98
|
||||
? fadeExit / fadeEntry - 1 - (COST_CENTS / 100) / fadeEntry : null;
|
||||
outcomes.push({ marketId: market.id, question: market.question, category: market.category,
|
||||
type: signal.type, side: signal.side, band: priceBand(entry), entry, exit, horizonHours,
|
||||
grossReturn, netReturn, fadeNetReturn, hourMove: signal.hourMove, dayMove: signal.dayMove, weekMove: signal.weekMove,
|
||||
observedAt: current.t, evaluatedAt: future.t });
|
||||
captured = true;
|
||||
}
|
||||
if (captured) previousBucket = bucket;
|
||||
}
|
||||
return outcomes;
|
||||
}
|
||||
@@ -127,13 +135,26 @@ function median(values) {
|
||||
}
|
||||
|
||||
function summarize(rows, field = "netReturn") {
|
||||
if (!rows.length) return { count: 0, mean: 0, median: 0, winRate: 0, worst: 0, best: 0 };
|
||||
if (!rows.length) return { count: 0, markets: 0, mean: 0, median: 0, winRate: 0, worst: 0, best: 0, marketMean: 0, lower90: 0, upper90: 0 };
|
||||
const returns = rows.map((row) => row[field]).filter(Number.isFinite);
|
||||
if (!returns.length) return { count: 0, mean: 0, median: 0, winRate: 0, worst: 0, best: 0 };
|
||||
return { count: rows.length,
|
||||
if (!returns.length) return { count: 0, markets: 0, mean: 0, median: 0, winRate: 0, worst: 0, best: 0, marketMean: 0, lower90: 0, upper90: 0 };
|
||||
const marketBuckets = new Map();
|
||||
rows.forEach((row) => {
|
||||
const value = row[field];
|
||||
if (!Number.isFinite(value)) return;
|
||||
const bucket = marketBuckets.get(row.marketId) || [];
|
||||
bucket.push(value); marketBuckets.set(row.marketId, bucket);
|
||||
});
|
||||
const marketReturns = [...marketBuckets.values()].map((values) => values.reduce((sum, value) => sum + value, 0) / values.length);
|
||||
const marketMean = marketReturns.reduce((sum, value) => sum + value, 0) / Math.max(1, marketReturns.length);
|
||||
const variance = marketReturns.length > 1
|
||||
? marketReturns.reduce((sum, value) => sum + (value - marketMean) ** 2, 0) / (marketReturns.length - 1) : 0;
|
||||
const margin90 = 1.645 * Math.sqrt(variance / Math.max(1, marketReturns.length));
|
||||
return { count: returns.length,
|
||||
mean: returns.reduce((sum, value) => sum + value, 0) / returns.length,
|
||||
median: median(returns), winRate: returns.filter((value) => value > 0).length / returns.length,
|
||||
worst: Math.min(...returns), best: Math.max(...returns) };
|
||||
worst: Math.min(...returns), best: Math.max(...returns), markets: marketReturns.length,
|
||||
marketMean, lower90: marketMean - margin90, upper90: marketMean + margin90 };
|
||||
}
|
||||
|
||||
function grouped(rows, key) {
|
||||
@@ -144,8 +165,17 @@ const RULES = [
|
||||
{ name: "follow_all", field: "netReturn", test: () => true },
|
||||
{ name: "follow_trend", field: "netReturn", test: (row) => row.type === "trend" },
|
||||
{ name: "follow_trend_no", field: "netReturn", test: (row) => row.type === "trend" && row.side === "NO" },
|
||||
{ name: "follow_trend_yes", field: "netReturn", test: (row) => row.type === "trend" && row.side === "YES" },
|
||||
{ name: "follow_trend_mid", field: "netReturn", test: (row) => row.type === "trend" && row.band === "mid" },
|
||||
{ name: "follow_trend_favorites", field: "netReturn", test: (row) => row.type === "trend" && ["favorite", "heavy-favorite"].includes(row.band) },
|
||||
{ name: "follow_trend_non_longshot", field: "netReturn", test: (row) => row.type === "trend" && row.band !== "longshot" },
|
||||
{ name: "follow_strong_trend", field: "netReturn", test: (row) => row.type === "trend" && Math.abs(row.dayMove) >= 0.015 && Math.abs(row.weekMove) >= 0.03 },
|
||||
{ name: "follow_moderate_trend", field: "netReturn", test: (row) => row.type === "trend" && Math.abs(row.dayMove) <= 0.03 && Math.abs(row.weekMove) <= 0.10 },
|
||||
{ name: "follow_hour_confirmed_trend", field: "netReturn", test: (row) => row.type === "trend" && Math.sign(row.hourMove) === Math.sign(row.dayMove) },
|
||||
...["Politics", "Sports", "Crypto", "Economy", "Pop Culture", "Other"].map((category) => ({
|
||||
name: `follow_trend_${category.toLowerCase().replace(/\s+/g, "_")}`, field: "netReturn",
|
||||
test: (row) => row.type === "trend" && row.category === category,
|
||||
})),
|
||||
{ name: "follow_reversal", field: "netReturn", test: (row) => row.type === "reversal" },
|
||||
{ name: "fade_trend", field: "fadeNetReturn", test: (row) => row.type === "trend" },
|
||||
{ name: "fade_trend_yes_move", field: "fadeNetReturn", test: (row) => row.type === "trend" && row.side === "YES" },
|
||||
@@ -156,9 +186,48 @@ function evaluateRules(rows) {
|
||||
return Object.fromEntries(RULES.map((rule) => [rule.name, summarize(rows.filter(rule.test), rule.field)]));
|
||||
}
|
||||
|
||||
const params = new URLSearchParams({ active: "true", closed: "false", archived: "false", include_tag: "true",
|
||||
limit: String(MARKET_LIMIT), order: "volume24hr", ascending: "false" });
|
||||
const rawMarkets = await fetchJson(`${GAMMA}/markets?${params}`);
|
||||
function chronologicalEvaluation(rows) {
|
||||
const ordered = [...rows].sort((a, b) => a.observedAt - b.observedAt);
|
||||
const splitTime = ordered[Math.floor(ordered.length * 0.70)]?.observedAt || 0;
|
||||
const train = ordered.filter((row) => row.observedAt < splitTime), test = ordered.filter((row) => row.observedAt >= splitTime);
|
||||
const cut1 = ordered[Math.floor(ordered.length / 3)]?.observedAt || 0;
|
||||
const cut2 = ordered[Math.floor(ordered.length * 2 / 3)]?.observedAt || 0;
|
||||
const thirds = [ordered.filter((row) => row.observedAt < cut1),
|
||||
ordered.filter((row) => row.observedAt >= cut1 && row.observedAt < cut2),
|
||||
ordered.filter((row) => row.observedAt >= cut2)];
|
||||
const thirdRules = thirds.map(evaluateRules), pooled = evaluateRules(ordered);
|
||||
const robustRules = Object.fromEntries(RULES.map((rule) => {
|
||||
const segments = thirdRules.map((result) => result[rule.name]);
|
||||
const enoughData = segments.every((segment) => segment.count >= 20 && segment.markets >= 5);
|
||||
const allPositive = enoughData && segments.every((segment) => segment.mean > 0 && segment.marketMean > 0);
|
||||
const allNegative = enoughData && segments.every((segment) => segment.mean < 0 && segment.marketMean < 0);
|
||||
return [rule.name, { enoughData, allPositive, allNegative,
|
||||
minimumSegmentMean: Math.min(...segments.map((segment) => segment.mean)),
|
||||
maximumSegmentMean: Math.max(...segments.map((segment) => segment.mean)), pooled: pooled[rule.name] }];
|
||||
}));
|
||||
return { splitTime: splitTime ? new Date(splitTime * 1000).toISOString() : null,
|
||||
trainCount: train.length, testCount: test.length, train: evaluateRules(train), test: evaluateRules(test),
|
||||
thirds: thirdRules, robustRules };
|
||||
}
|
||||
|
||||
async function fetchActiveMarkets(limit) {
|
||||
const markets = [], seen = new Set(), pageSize = 100;
|
||||
for (let offset = 0; offset < limit; offset += pageSize) {
|
||||
const params = new URLSearchParams({ active: "true", closed: "false", archived: "false", include_tag: "true",
|
||||
limit: String(Math.min(pageSize, limit - offset)), offset: String(offset), order: "volume24hr", ascending: "false" });
|
||||
const page = await fetchJson(`${GAMMA}/markets?${params}`);
|
||||
if (!Array.isArray(page) || !page.length) break;
|
||||
for (const market of page) {
|
||||
const id = String(market.id || "");
|
||||
if (!id || seen.has(id)) continue;
|
||||
seen.add(id); markets.push(market);
|
||||
}
|
||||
if (page.length < Math.min(pageSize, limit - offset)) break;
|
||||
}
|
||||
return markets.slice(0, limit);
|
||||
}
|
||||
|
||||
const rawMarkets = await fetchActiveMarkets(MARKET_LIMIT);
|
||||
const markets = rawMarkets.map((raw) => ({ id: String(raw.id), question: raw.question || "", category: categoryOf(raw),
|
||||
tokenId: String(parseJson(raw.clobTokenIds)[0] || "") })).filter((market) => market.id && market.tokenId);
|
||||
const histories = await mapLimit(markets, CONCURRENCY, async (market) => {
|
||||
@@ -169,17 +238,20 @@ const histories = await mapLimit(markets, CONCURRENCY, async (market) => {
|
||||
});
|
||||
const successful = histories.filter((result) => result && !result.error && result.points.length);
|
||||
const outcomes = successful.flatMap((result) => result.outcomes);
|
||||
const ordered = [...outcomes].sort((a, b) => a.observedAt - b.observedAt);
|
||||
const splitTime = ordered[Math.floor(ordered.length * 0.70)]?.observedAt || 0;
|
||||
const train = ordered.filter((row) => row.observedAt < splitTime), test = ordered.filter((row) => row.observedAt >= splitTime);
|
||||
const primaryHorizon = HORIZONS.includes(12) ? 12 : HORIZONS[0];
|
||||
const primaryOutcomes = outcomes.filter((row) => row.horizonHours === primaryHorizon);
|
||||
const report = {
|
||||
generatedAt: new Date().toISOString(), marketLimit: MARKET_LIMIT, marketsWithHistory: successful.length,
|
||||
methodology: { horizonHours: 12, observationBucketHours: 6, historyInterval: "1m", fidelityMinutes: 60,
|
||||
estimatedRoundTripCostCents: 0.5, note: "Current active-market selection and current category tags are a survivorship-biased proxy; signal inputs and future marks are time-ordered without lookahead." },
|
||||
overall: summarize(outcomes), byType: grouped(outcomes, "type"), byCategory: grouped(outcomes, "category"),
|
||||
byBand: grouped(outcomes, "band"), bySide: grouped(outcomes, "side"),
|
||||
chronologicalSplit: { splitTime: splitTime ? new Date(splitTime * 1000).toISOString() : null,
|
||||
trainCount: train.length, testCount: test.length, train: evaluateRules(train), test: evaluateRules(test) },
|
||||
methodology: { horizonHours: HORIZONS, primaryHorizon, observationBucketHours: 6, historyInterval: "1m", fidelityMinutes: 60,
|
||||
estimatedRoundTripCostCents: COST_CENTS, clusterUnit: "market",
|
||||
note: "Current active-market selection and current category tags are a survivorship-biased proxy; signal inputs and future marks are time-ordered without lookahead. Confidence intervals use per-market means to reduce repeated-observation distortion." },
|
||||
overall: summarize(primaryOutcomes), byType: grouped(primaryOutcomes, "type"), byCategory: grouped(primaryOutcomes, "category"),
|
||||
byBand: grouped(primaryOutcomes, "band"), bySide: grouped(primaryOutcomes, "side"),
|
||||
chronologicalSplit: chronologicalEvaluation(primaryOutcomes),
|
||||
horizons: Object.fromEntries(HORIZONS.map((horizon) => {
|
||||
const rows = outcomes.filter((row) => row.horizonHours === horizon);
|
||||
return [horizon, { overall: summarize(rows), chronological: chronologicalEvaluation(rows) }];
|
||||
})),
|
||||
failures: histories.filter((result) => result?.error).length,
|
||||
};
|
||||
console.log(JSON.stringify(report, null, 2));
|
||||
|
||||
Reference in New Issue
Block a user