Add audited adaptive resolution-window pilot

This commit is contained in:
Theodore Song
2026-08-21 18:56:58 -04:00
parent b956f50cbe
commit c0ce373833
9 changed files with 376 additions and 31 deletions
+21 -1
View File
@@ -175,6 +175,13 @@ const RULES = [
{ name: "buy_underdog", test: (row) => !row.favorite },
{ name: "buy_yes", test: (row) => row.side === "YES" },
{ name: "buy_no", test: (row) => row.side === "NO" },
{ name: "buy_no_favorite", test: (row) => row.side === "NO" && row.favorite },
{ name: "buy_no_55_90", test: (row) => row.side === "NO" && row.entry >= 0.55 && row.entry < 0.90 },
{ name: "buy_no_50_55", test: (row) => row.side === "NO" && row.entry >= 0.50 && row.entry < 0.55 },
{ name: "buy_no_60_90", test: (row) => row.side === "NO" && row.entry >= 0.60 && row.entry < 0.90 },
{ name: "buy_no_90_95", test: (row) => row.side === "NO" && row.entry >= 0.90 && row.entry < 0.95 },
{ name: "buy_no_95_99", test: (row) => row.side === "NO" && row.entry >= 0.95 && row.entry < 0.99 },
{ name: "buy_no_90_99", test: (row) => row.side === "NO" && row.entry >= 0.90 && row.entry < 0.99 },
{ name: "follow_trend", test: (row) => row.trend },
{ name: "follow_trend_yes", test: (row) => row.trend && row.side === "YES" },
{ name: "follow_trend_no", test: (row) => row.trend && row.side === "NO" },
@@ -248,6 +255,14 @@ const compactRules = (rules = {}) => Object.fromEntries(Object.entries(rules)
.filter(([, result]) => result.enoughData && (result.allPositive || result.allNegative))
.map(([name, result]) => [name, { direction: result.allPositive ? "positive" : "negative",
pooled: compactStats(result.pooled), train: compactStats(result.train), test: compactStats(result.test) }]));
const TARGET_RULE_NAMES = String(process.env.SETTLEMENT_TARGET_RULES || "buy_no_favorite,buy_no_50_55,buy_no_55_90,buy_no_60_90,buy_no_90_95,buy_no_95_99,buy_no_90_99")
.split(",").map((name) => name.trim()).filter(Boolean);
const targetRuleStats = (chronological, names = TARGET_RULE_NAMES) => Object.fromEntries(names.map((name) => {
const result = chronological.robustRules[name] || {};
return [name, { enoughData: Boolean(result.enoughData), allPositive: Boolean(result.allPositive), allNegative: Boolean(result.allNegative),
pooled: compactStats(result.pooled), train: compactStats(result.train), test: compactStats(result.test),
segments: (result.segments || []).map(compactStats) }];
}));
const summary = {
generatedAt: report.generatedAt, requestedMarkets: report.requestedMarkets, resolvedMarkets: report.resolvedMarkets,
marketsWithHistory: report.marketsWithHistory, failures: report.failures,
@@ -263,7 +278,12 @@ const summary = {
noTest: compactStats(value.chronological.test.buy_no),
trend: compactStats(value.chronological.train.follow_trend),
trendTest: compactStats(value.chronological.test.follow_trend),
targetRules: targetRuleStats(value.chronological),
robustRules: compactRules(value.chronological.robustRules),
}])),
};
console.log(JSON.stringify(compact ? summary : report, null, 2));
const targetOnly = process.env.SETTLEMENT_TARGET_ONLY === "1";
const targetSummary = { generatedAt: summary.generatedAt, requestedMarkets: summary.requestedMarkets,
resolvedMarkets: summary.resolvedMarkets, marketsWithHistory: summary.marketsWithHistory, failures: summary.failures,
horizons: Object.fromEntries(Object.entries(summary.horizons).map(([days, value]) => [days, { observations: value.observations, targetRules: value.targetRules }])) };
console.log(JSON.stringify(targetOnly ? targetSummary : (compact ? summary : report), null, 2));
+9 -5
View File
@@ -1,6 +1,7 @@
const GAMMA = "https://gamma-api.polymarket.com";
const CLOB = "https://clob.polymarket.com";
const MARKET_LIMIT = Math.max(10, Math.min(500, Number(process.env.EVAL_MARKETS || 80)));
const ACTIVE_SKIP = Math.max(0, Math.min(5000, Number(process.env.EVAL_SKIP || 0)));
const CONCURRENCY = Math.max(1, Math.min(12, Number(process.env.EVAL_CONCURRENCY || 6)));
const HORIZONS = [...new Set(String(process.env.EVAL_HORIZONS || "6,12,24,72").split(",")
.map(Number).filter((value) => Number.isFinite(value) && value >= 1 && value <= 168))].sort((a, b) => a - b);
@@ -258,9 +259,10 @@ function chronologicalEvaluation(rows) {
thirds: thirdRules, robustRules };
}
async function fetchActiveMarkets(limit) {
async function fetchActiveMarkets(limit, skip = 0) {
const markets = [], seen = new Set(), pageSize = 100;
for (let offset = 0; markets.length < limit && offset < limit * 4; offset += pageSize) {
let eligibleSkipped = 0;
for (let offset = 0; markets.length < limit && offset < (limit + skip) * 4; offset += pageSize) {
const params = new URLSearchParams({ active: "true", closed: "false", archived: "false", include_tag: "true",
limit: String(pageSize), offset: String(offset), order: "volume24hr", ascending: "false" });
const page = await fetchJson(`${GAMMA}/markets?${params}`);
@@ -268,6 +270,7 @@ async function fetchActiveMarkets(limit) {
for (const market of page) {
const id = String(market.id || ""), labels = parseJson(market.outcomes).map((outcome) => String(outcome).trim().toLowerCase());
if (!id || seen.has(id) || labels[0] !== "yes" || labels[1] !== "no") continue;
if (eligibleSkipped < skip) { seen.add(id); eligibleSkipped++; continue; }
seen.add(id); markets.push(market);
if (markets.length >= limit) break;
}
@@ -276,7 +279,7 @@ async function fetchActiveMarkets(limit) {
return markets.slice(0, limit);
}
const rawMarkets = await fetchActiveMarkets(MARKET_LIMIT);
const rawMarkets = await fetchActiveMarkets(MARKET_LIMIT, ACTIVE_SKIP);
const markets = rawMarkets.map((raw) => ({ id: String(raw.id), question: raw.question || "", category: categoryOf(raw),
binaryLabels: parseJson(raw.outcomes).map((outcome) => String(outcome).trim().toLowerCase()),
eventKey: String(raw.events?.[0]?.id || raw.events?.[0]?.slug || raw.eventId || raw.id),
@@ -293,7 +296,7 @@ const outcomes = successful.flatMap((result) => result.outcomes);
const primaryHorizon = HORIZONS.includes(12) ? 12 : HORIZONS[0];
const primaryOutcomes = outcomes.filter((row) => row.horizonHours === primaryHorizon);
const report = {
generatedAt: new Date().toISOString(), marketLimit: MARKET_LIMIT, marketsWithHistory: successful.length,
generatedAt: new Date().toISOString(), marketLimit: MARKET_LIMIT, activeMarketsSkipped: ACTIVE_SKIP, marketsWithHistory: successful.length,
methodology: { horizonHours: HORIZONS, primaryHorizon, observationBucketHours: 6, historyInterval: "1m", fidelityMinutes: 60,
estimatedRoundTripCostCents: COST_CENTS, clusterUnit: "event",
note: "Current active-market selection and current category tags are a survivorship-biased proxy; signal inputs and future marks are time-ordered without lookahead. Confidence intervals cluster correlated markets by Polymarket event." },
@@ -316,7 +319,8 @@ const compactRules = (rules = {}) => Object.fromEntries(Object.entries(rules)
minimumSegmentMean: result.minimumSegmentMean, maximumSegmentMean: result.maximumSegmentMean,
pooled: compactStats(result.pooled) }]));
const summary = {
generatedAt: report.generatedAt, marketLimit: report.marketLimit, marketsWithHistory: report.marketsWithHistory,
generatedAt: report.generatedAt, marketLimit: report.marketLimit, activeMarketsSkipped: report.activeMarketsSkipped,
marketsWithHistory: report.marketsWithHistory,
primaryHorizon: report.methodology.primaryHorizon, failures: report.failures,
overall: compactStats(report.overall),
byType: Object.fromEntries(Object.entries(report.byType).map(([key, value]) => [key, compactStats(value)])),
+1 -1
View File
@@ -128,7 +128,7 @@ async function main() {
const branch = process.env.RUNTIME_BRANCH || "runtime-state";
const pathname = process.env.RUNTIME_STATE_PATH || "runtime/state.json";
const arenaUrl = (process.env.ARENA_URL || "https://polymarket-site-eta.vercel.app").replace(/\/$/, "");
const expectedBuild = Number(required("EXPECTED_BUILD", "95"));
const expectedBuild = Number(required("EXPECTED_BUILD", "96"));
await ensureRuntimeBranch(repository, branch);
const prior = await readRuntimeFile(repository, branch, pathname);
if (prior.snapshot) validateRuntimeSnapshot(prior.snapshot, 0, { allowIncomplete: true });
+12 -2
View File
@@ -6,9 +6,10 @@ const index = fs.readFileSync(new URL("../index.html", import.meta.url), "utf8")
const api = fs.readFileSync(new URL("../api/state.js", import.meta.url), "utf8");
const workflow = fs.readFileSync(new URL("../.github/workflows/autonomous-cycle.yml", import.meta.url), "utf8");
const runner = fs.readFileSync(new URL("./run-autonomous-cycle.mjs", import.meta.url), "utf8");
const resolutionAudit = JSON.parse(fs.readFileSync(new URL("../research/resolution-week-no-audit.json", import.meta.url), "utf8"));
const build = Number(index.match(/const BUILD_VERSION = (\d+);/)?.[1]);
assert.equal(build, 95);
assert.equal(build, 96);
assert.deepEqual([...ALLOWED_RUNTIME_KEYS].sort(), ["pma_agents_v2", "pma_suggestions_v5"]);
assert.match(index, /function collectPublicRuntimeItems\(\)/);
assert.match(index, /const PUBLIC_RUNTIME_KEYS=Object\.freeze\(\[AGENTS_KEY,SUG_KEY\]\)/);
@@ -25,8 +26,17 @@ assert.match(api, /Buffer\.from\(file\.content/);
assert.match(api, /searchParams\.set\("runtime", `\$\{Date\.now\(\)\}/);
assert.match(workflow, /cron: "7,22,37,52 \* \* \* \*"/);
assert.match(workflow, /contents: write/);
assert.match(workflow, /EXPECTED_BUILD: "95"/);
assert.match(workflow, /EXPECTED_BUILD: "96"/);
assert.match(runner, /MAX_STATE_BYTES = 900_000/);
assert.equal(resolutionAudit.strategy, "resolution-window-no-50-55");
assert.deepEqual(resolutionAudit.selection.horizon_days_enabled, [3, 7]);
assert.ok(resolutionAudit.rules["3_day"].train_event_lower_90 > 0);
assert.ok(resolutionAudit.rules["3_day"].holdout_event_lower_90 > 0);
assert.ok(resolutionAudit.rules["3_day"].chronological_thirds_event_lower_90.every((value) => value > 0));
assert.ok(resolutionAudit.rules["7_day"].train_event_lower_90 > 0);
assert.ok(resolutionAudit.rules["7_day"].holdout_event_lower_90 > 0);
assert.ok(resolutionAudit.rules["7_day"].chronological_thirds_event_lower_90.every((value) => value > 0));
assert.ok(resolutionAudit.rejected_neighboring_rule.event_lower_90 < 0);
const agentIds = ["value", "momentum", "favorite", "longshot", "diversifier", "catalyst", "reversal", "breakout", "tailalpha", "conviction"];
const agents = Object.fromEntries(agentIds.map(id => [id, { cash: 10000, positions: [], lastDecision: { mode: "test" } }]));