From 4b3e03a353df76d71fff59603fc002b9126ab215 Mon Sep 17 00:00:00 2001
From: Theodore Song <johnsong10086@hotmail.com>
Date: Sun, 26 Jul 2026 16:15:48 -0400
Subject: [PATCH] Use keyset pagination for full market scans

---
 index.html | 48 ++++++++++++++++++++++++++++++++++--------------
 1 file changed, 34 insertions(+), 14 deletions(-)

diff --git a/index.html b/index.html
index b17b82f..4e4c262 100644
--- a/index.html
+++ b/index.html
@@ -739,7 +739,7 @@ footer{margin-top:34px;padding-top:22px;border-top:1px solid var(--border);color
   </section>
 
   <footer>
-    Build all-market-suggestions · Paper trading only · Live prices from Polymarket's public Gamma API · Not financial advice ·
+    Build keyset-market-scan · Paper trading only · Live prices from Polymarket's public Gamma API · Not financial advice ·
     <a class="market-link" href="https://github.com/theodore-song/polymarket-analyst" target="_blank" rel="noopener">Source on GitHub</a>
   </footer>
 </div>
@@ -766,7 +766,7 @@ const EXIT_STALE_DAYS = 10;
 const EXIT_RESOLUTION_DAYS = 2;
 const AGENTS_KEY = "pma_agents_v2";
 const SUG_KEY = "pma_suggestions_v5";
-const SUGGESTION_ENGINE_VERSION = 8;
+const SUGGESTION_ENGINE_VERSION = 9;
 const FOCUS_KEY = "pma_focus_v1";
 const VIEW_KEY = "pma_view_v1";
 const PF_SORT_KEY = "pma_portfolio_sort_v1";
@@ -950,12 +950,21 @@ async function fetchMarkets(pages=Infinity,perPage=100,onProgress=null){
   const all=[];
   const maxPages=Number.isFinite(pages)?pages:MAX_ACTIVE_MARKET_PAGES;
   const seen=new Set();
+  const useKeyset=!Number.isFinite(pages)||pages>20;
+  let cursor=null;
   for(let i=0;i<maxPages;i++){
     const params=new URLSearchParams({closed:"false",active:"true",archived:"false",include_tag:"true",
-      limit:String(perPage),offset:String(i*perPage),order:"volume24hr",ascending:"false"});
-    const r=await fetch(`${GAMMA}/markets?${params}`);
+      limit:String(perPage),order:"volume24hr",ascending:"false"});
+    if(useKeyset){
+      if(cursor)params.set("after_cursor",cursor);
+    }else{
+      params.set("offset",String(i*perPage));
+    }
+    const r=await fetch(`${GAMMA}/markets${useKeyset?"/keyset":""}?${params}`);
     if(!r.ok){if(i===0)throw new Error("Polymarket API "+r.status);break;}
-    const raw=await r.json(); if(!raw.length)break;
+    const page=await r.json();
+    const raw=useKeyset?(page.markets||[]):page;
+    if(!raw.length)break;
     let added=0;
     for(const m of raw){
       const id=String(m.id||"");
@@ -963,8 +972,9 @@ async function fetchMarkets(pages=Infinity,perPage=100,onProgress=null){
       if(id)seen.add(id);
       all.push(m);added++;
     }
+    if(useKeyset)cursor=page.next_cursor||null;
     if(onProgress)onProgress(all.length);
-    if(raw.length<perPage||added===0)break;
+    if(raw.length<perPage||added===0||(useKeyset&&!cursor))break;
   }
   return all.map(normalizeMarket).filter(m=>m&&m.question);
 }
@@ -3106,19 +3116,24 @@ async function loadPaperMarketPage(reset=false,loadAll=false){
     if(reset){PAPER_SEARCH_RESULTS=[];PAPER_MARKET_OFFSET=0;}
     let loaded=0,rawCount=0,keepGoing=true;
     const limit=100,maxPages=loadAll?MAX_ACTIVE_MARKET_PAGES:1;
+    let cursor=null;
     for(let page=0;page<maxPages&&keepGoing;page++){
-      const params=new URLSearchParams({closed:"false",active:"true",archived:"false",include_tag:"true",limit:String(limit),offset:String(PAPER_MARKET_OFFSET),order:"volume24hr",ascending:"false"});
+      const params=new URLSearchParams({closed:"false",active:"true",archived:"false",include_tag:"true",limit:String(limit),order:"volume24hr",ascending:"false"});
+      if(loadAll){if(cursor)params.set("after_cursor",cursor);}
+      else params.set("offset",String(PAPER_MARKET_OFFSET));
       if(q)params.set("search",q);
-      const r=await fetch(`${GAMMA}/markets?${params}`);
+      const r=await fetch(`${GAMMA}/markets${loadAll?"/keyset":""}?${params}`);
       if(!r.ok)throw new Error("market load failed");
-      const raw=await r.json();
+      const pageData=await r.json();
+      const raw=loadAll?(pageData.markets||[]):pageData;
       rawCount+=raw.length;
       const next=raw.map(normalizeMarket).filter(Boolean).map(paperTradeFromMarket);
       PAPER_SEARCH_RESULTS=(PAPER_SEARCH_RESULTS||[]).concat(next);
       PAPER_MARKET_OFFSET+=raw.length;
+      if(loadAll)cursor=pageData.next_cursor||null;
       loaded+=next.length;
       if(loadAll)setStatus(`loading active markets (${PAPER_SEARCH_RESULTS.length})…`,true);
-      if(raw.length<limit)keepGoing=false;
+      if(raw.length<limit||(loadAll&&!cursor))keepGoing=false;
     }
     renderPaperTab();setStatus("up to date",false);
     toast(loaded?`Loaded ${PAPER_SEARCH_RESULTS.length} active markets.`:rawCount?"No tradable binary markets found in this page.":"No more active markets found.");
@@ -3467,19 +3482,24 @@ async function loadLiveMarketPage(reset=false,loadAll=false){
     if(reset){LIVE_SEARCH_RESULTS=[];LIVE_MARKET_OFFSET=0;}
     let loaded=0,rawCount=0,keepGoing=true;
     const limit=100,maxPages=loadAll?MAX_ACTIVE_MARKET_PAGES:1;
+    let cursor=null;
     for(let page=0;page<maxPages&&keepGoing;page++){
-      const params=new URLSearchParams({closed:"false",active:"true",archived:"false",include_tag:"true",limit:String(limit),offset:String(LIVE_MARKET_OFFSET),order:"volume24hr",ascending:"false"});
+      const params=new URLSearchParams({closed:"false",active:"true",archived:"false",include_tag:"true",limit:String(limit),order:"volume24hr",ascending:"false"});
+      if(loadAll){if(cursor)params.set("after_cursor",cursor);}
+      else params.set("offset",String(LIVE_MARKET_OFFSET));
       if(q)params.set("search",q);
-      const r=await fetch(`${GAMMA}/markets?${params}`);
+      const r=await fetch(`${GAMMA}/markets${loadAll?"/keyset":""}?${params}`);
       if(!r.ok)throw new Error("market load failed");
-      const raw=await r.json();
+      const pageData=await r.json();
+      const raw=loadAll?(pageData.markets||[]):pageData;
       rawCount+=raw.length;
       const next=raw.map(normalizeMarket).filter(Boolean).map(paperTradeFromMarket);
       LIVE_SEARCH_RESULTS=(LIVE_SEARCH_RESULTS||[]).concat(next);
       LIVE_MARKET_OFFSET+=raw.length;
+      if(loadAll)cursor=pageData.next_cursor||null;
       loaded+=next.length;
       if(loadAll)setStatus(`loading active markets (${LIVE_SEARCH_RESULTS.length})…`,true);
-      if(raw.length<limit)keepGoing=false;
+      if(raw.length<limit||(loadAll&&!cursor))keepGoing=false;
     }
     renderLiveMarkets();setStatus("up to date",false);
     toast(loaded?`Loaded ${LIVE_SEARCH_RESULTS.length} active markets.`:rawCount?"No tradable binary markets found in this page.":"No more active markets found.");