SQLite 开启 WAL 模式 + busy_timeout 解决并行扫描写入锁冲突

scratch_query_gamma.py

@@ -0,0 +1,36 @@
+import httpx
+
+gamma_url = "https://gamma-api.polymarket.com"
+
+# 1. Search markets
+print("Searching markets for 'temperature'...")
+try:
+    resp = httpx.get(f"{gamma_url}/markets", params={"active": "true", "search": "temperature", "limit": 100}, timeout=10.0)
+    data = resp.json()
+    print(f"Found {len(data)} markets matching 'temperature':")
+    for m in data[:10]:
+        print(f"  - {m.get('question')} | Slug: {m.get('slug')}")
+except Exception as e:
+    print("Error:", e)
+
+# 2. Search events
+print("\nSearching events for 'temperature'...")
+try:
+    resp = httpx.get(f"{gamma_url}/events", params={"active": "true", "search": "temperature", "limit": 100}, timeout=10.0)
+    data = resp.json()
+    print(f"Found {len(data)} events matching 'temperature':")
+    for e in data[:10]:
+        print(f"  - {e.get('title')} | Slug: {e.get('slug')}")
+except Exception as e:
+    print("Error:", e)
+
+# 3. Search events for 'weather'
+print("\nSearching events for 'weather'...")
+try:
+    resp = httpx.get(f"{gamma_url}/events", params={"active": "true", "search": "weather", "limit": 100}, timeout=10.0)
+    data = resp.json()
+    print(f"Found {len(data)} events matching 'weather':")
+    for e in data[:10]:
+        print(f"  - {e.get('title')} | Slug: {e.get('slug')}")
+except Exception as e:
+    print("Error:", e)

scratch_query_slugs.py

@@ -0,0 +1,23 @@
+import httpx
+
+gamma_url = "https://gamma-api.polymarket.com"
+
+cities = ["hong-kong", "beijing", "tokyo", "seoul", "shanghai"]
+date_str = "may-25-2026"
+
+for city in cities:
+    slug = f"highest-temperature-in-{city}-on-{date_str}"
+    try:
+        resp = httpx.get(f"{gamma_url}/events", params={"slug": slug}, timeout=10.0)
+        data = resp.json()
+        print(f"Slug: {slug} -> Status: {resp.status_code}")
+        if data and isinstance(data, list):
+            print(f"  Found Event: {data[0].get('title')} | ID: {data[0].get('id')}")
+            markets = data[0].get("markets", [])
+            print(f"  Markets: {len(markets)}")
+            for m in markets:
+                print(f"    - Market: {m.get('question')} | Group: {m.get('group_id')}")
+        else:
+            print("  No event found.")
+    except Exception as e:
+        print(f"  Error for {slug}: {e}")

scratch_search_markets.py

@@ -0,0 +1,43 @@
+import httpx
+
+gamma_url = "https://gamma-api.polymarket.com"
+all_markets = []
+offset = 0
+limit = 100
+
+while True:
+    params = {
+        "archived": "false",
+        "limit": limit,
+        "offset": offset,
+        "active": "true",
+        "closed": "false"
+    }
+    try:
+        resp = httpx.get(f"{gamma_url}/markets", params=params, timeout=10.0)
+        batch = resp.json()
+    except Exception as e:
+        print("Fetch error:", e)
+        break
+    if not isinstance(batch, list) and isinstance(batch, dict):
+        batch = batch.get("markets", [])
+    if not batch:
+        break
+    all_markets.extend(batch)
+    if len(batch) < limit:
+        break
+    offset += len(batch)
+
+print(f"Total active markets: {len(all_markets)}")
+
+cities = ["beijing", "tokyo", "seoul", "shanghai", "singapore", "manila", "jakarta", "taipei", "hong kong", "busan", "shenzhen"]
+
+for city in cities:
+    matched = []
+    for m in all_markets:
+        q = m.get("question", "").lower()
+        if city in q:
+            matched.append(m)
+    print(f"City '{city}': matched {len(matched)} active markets")
+    for m in matched:
+        print(f"  - {m.get('question')} (slug: {m.get('slug')})")

scratch_test.py

@@ -1,62 +0,0 @@
-import sys
-import time
-import httpx
-from loguru import logger
-
-logger.remove()
-logger.add(sys.stdout, format="[{time:HH:mm:ss}] <level>{message}</level>")
-
-print("1. Testing direct httpx request to Gamma API...")
-t0 = time.time()
-try:
-    resp = httpx.get("https://gamma-api.polymarket.com/markets", params={"active": "true", "closed": "false", "limit": 200}, timeout=10.0)
-    print(f"Gamma response status: {resp.status_code}")
-    print(f"Gamma response took: {time.time() - t0:.2f}s")
-    markets = resp.json()
-    print(f"Markets count: {len(markets)}")
-except Exception as e:
-    print(f"Failed to fetch from Gamma API: {e}")
-
-print("\n2. Testing loading PolymarketReadOnlyLayer...")
-try:
-    from src.data_collection.polymarket_readonly import PolymarketReadOnlyLayer
-    layer = PolymarketReadOnlyLayer()
-    print(f"Polymarket layer enabled: {layer.enabled}")
-    
-    t0 = time.time()
-    print("Loading active markets via layer...")
-    m = layer._load_markets(active_only=True)
-    print(f"Layer active markets count: {len(m)} (took {time.time() - t0:.2f}s)")
-    
-    t0 = time.time()
-    print("Loading broad markets via layer...")
-    m_broad = layer._load_markets(active_only=False)
-    print(f"Layer broad markets count: {len(m_broad)} (took {time.time() - t0:.2f}s)")
-except Exception as e:
-    print(f"Layer test failed: {e}")
-
-print("\n3. Testing scan terminal payload for East Asia cities...")
-try:
-    from web.scan_terminal_service import _build_scan_terminal_payload_uncached
-    filters = {
-        "scan_mode": "tradable",
-        "min_price": 0.05,
-        "max_price": 0.95,
-        "min_edge_pct": 2.0,
-        "min_liquidity": 1000.0,
-        "high_liquidity_only": False,
-        "market_type": "maxtemp",
-        "time_range": "today",
-        "limit": 28,
-        "trading_region": "east_asia"
-    }
-    t0 = time.time()
-    res = _build_scan_terminal_payload_uncached(filters, force_refresh=True)
-    print(f"Build scan terminal payload for east_asia took {time.time() - t0:.2f}s")
-    print(f"Result Status: {res.get('status')}")
-    print(f"Result Rows count: {len(res.get('rows', []))}")
-    for row in res.get('rows', []):
-        print(f"- City: {row.get('city')}, Question: {row.get('market_question')}, Midpoint: {row.get('midpoint')}, IsPrimary: {row.get('is_primary_signal')}")
-except Exception as e:
-    import traceback
-    traceback.print_exc()

src/database/db_manager.py

@@ -27,7 +27,10 @@ class DBManager:
         return raw
 
     def _get_connection(self):
-        return sqlite3.connect(self.db_path)
+        conn = sqlite3.connect(self.db_path, timeout=10)
+        conn.execute("PRAGMA journal_mode=WAL")
+        conn.execute("PRAGMA busy_timeout=5000")
+        return conn
 
     def _init_cache_key(self) -> str:
         return os.path.abspath(self.db_path)