gavindiaz/DinQuant
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
c3cf230104cb98e0a5b85819a75e471995df53da
DinQuant/backend_api_python/app/data_sources/polymarket.py
T
dienakdz c3cf230104 feat(trading-assistant): refactor strategy creation and enhance script mode functionality 
- Removed conditional rendering for the assistant guide bar.
- Simplified strategy overview and strategy list item components.
- Introduced a new modal for selecting strategy mode.
- Enhanced strategy creation modal to support script strategies with a dedicated editor.
- Updated form handling for script strategies, including validation and submission logic.
- Improved user experience with better messaging and streamlined UI components.
- Updated translations for better clarity in Chinese.
2026-04-08 07:27:26 +07:00

1227 lines
65 KiB
Python
Raw Blame History

This file contains invisible Unicode characters
This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""
Polymarket prediction market data source
Get prediction market data from Polymarket
"""
import time
import requests
import json
from typing import Dict, List, Any, Optional
from datetime import datetime, timedelta
from app.utils.logger import get_logger
from app.utils.db import get_db_connection
logger = get_logger(__name__)
class PolymarketDataSource:
"""Polymarket prediction market data source"""
def __init__(self):
# Polymarket official API endpoint (according to official documentation)
# Gamma API: markets, events, tags, searches, etc. (fully public, no authentication required)
self.gamma_api = "https://gamma-api.polymarket.com"
# Data API: User positions, transactions, activities, etc. (fully public, no authentication required)
self.data_api = "https://data-api.polymarket.com"
# CLOB API: Order book, prices, trading operations (public endpoints require no authentication)
self.clob_api = "https://clob.polymarket.com"
self.cache_ttl = 300 # 5 minutes cache
self.session = requests.Session()
self.session.headers.update({
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
'Accept': 'application/json'
})
def get_trending_markets(self, category: str = None, limit: int = 50) -> List[Dict]:
"""
Get popular prediction markets
Args:
category: category filter (crypto, politics, economics, sports, all)
limit: return quantity limit
Returns:
Prediction market list
"""
try:
# Read from database cache first
cached = self._get_cached_markets(category, limit)
if cached:
return cached
# Fetch from real API - Fetch data from multiple categories to ensure diversity
all_markets = []
if category and category != "all":
# If a category is specified, only data for that category will be obtained
markets = self._fetch_markets_from_api(category, limit * 2)
all_markets.extend(markets)
else:
# If no category is specified or "all" is specified, get data from multiple categories
categories_to_fetch = ["crypto", "politics", "economics", "sports"]
for cat in categories_to_fetch:
markets = self._fetch_markets_from_api(cat, limit // len(categories_to_fetch) + 10)
all_markets.extend(markets)
# Deduplication (by market_id)
seen = set()
unique_markets = []
for market in all_markets:
market_id = market.get("market_id")
if market_id and market_id not in seen:
seen.add(market_id)
unique_markets.append(market)
# Sort by transaction volume
unique_markets.sort(key=lambda x: x.get('volume_24h', 0), reverse=True)
# Save to database cache
if unique_markets:
self._save_markets_to_db(unique_markets)
return unique_markets[:limit]
# If the API fails, an empty list is returned (sample data is no longer used)
logger.warning("Polymarket API unavailable, returning empty list")
return []
except Exception as e:
logger.error(f"Failed to get trending markets: {e}", exc_info=True)
return []
def get_market_details(self, market_id: str) -> Optional[Dict]:
"""Get individual market details"""
try:
# Make sure market_id is a string
market_id = str(market_id).strip()
if not market_id:
logger.warning("Empty market_id provided")
return None
# Read from database first
try:
with get_db_connection() as db:
cur = db.cursor()
cur.execute("""
SELECT market_id, question, category, current_probability,
volume_24h, liquidity, end_date_iso, status, outcome_tokens
FROM qd_polymarket_markets
WHERE market_id = %s
""", (market_id,))
row = cur.fetchone()
cur.close()
if row:
#RealDictCursor returns the dictionary, accessed using keys
db_market_id = str(row.get('market_id') or market_id)
# Parse outcome_tokens (may be a JSON string)
outcome_tokens = {}
outcome_tokens_raw = row.get('outcome_tokens')
if outcome_tokens_raw:
try:
if isinstance(outcome_tokens_raw, str):
outcome_tokens = json.loads(outcome_tokens_raw)
else:
outcome_tokens = outcome_tokens_raw if isinstance(outcome_tokens_raw, dict) else {}
except:
outcome_tokens = {}
return {
"market_id": db_market_id,
"question": row.get('question') or '',
"category": row.get('category') or 'other',
"current_probability": float(row.get('current_probability') or 0),
"volume_24h": float(row.get('volume_24h') or 0),
"liquidity": float(row.get('liquidity') or 0),
"end_date_iso": row.get('end_date_iso'),
"status": row.get('status') or 'active',
"outcome_tokens": outcome_tokens,
"polymarket_url": self._build_polymarket_url(row.get('slug'), db_market_id),
"slug": row.get('slug') if row.get('slug') and not str(row.get('slug', '')).isdigit() else None
}
except Exception as db_error:
logger.warning(f"Database query failed for market {market_id}: {db_error}")
# Continue trying to get it from the API
# If the database does not exist, get it from the API
logger.info(f"Market {market_id} not in database, fetching from API")
market = self._fetch_market_from_api(market_id)
if market:
try:
self._save_markets_to_db([market])
except Exception as save_error:
logger.warning(f"Failed to save market to DB: {save_error}")
return market
logger.warning(f"Market {market_id} not found in API")
return None
except Exception as e:
logger.error(f"Failed to get market details for {market_id}: {e}", exc_info=True)
return None
def get_market_history(self, market_id: str, days: int = 30) -> List[Dict]:
"""Get historical market price data."""
# Here you need to implement historical data acquisition logic
# Temporarily returns an empty list
return []
def search_markets(self, keyword: str, limit: int = 20, use_cache: bool = True) -> List[Dict]:
"""
Search related prediction markets
Priority is given to obtaining real-time data from the API, and the database is only used as an optional cache.
Args:
keyword: search keyword
limit: limit on the number of returned results
use_cache: whether to use database cache (should be set to False during AI analysis to obtain the latest data)
"""
try:
logger.info(f"Searching Polymarket markets for keyword: '{keyword}' (limit={limit}, use_cache={use_cache})")
# If caching is allowed, try searching from the database first
if use_cache:
with get_db_connection() as db:
cur = db.cursor()
# Improved search: search question and slug fields at the same time, also support market_id exact matching
keyword_lower = keyword.lower()
is_numeric = keyword_lower.isdigit()
has_hyphens = '-' in keyword_lower
if is_numeric:
# If it is a pure number, it may be market_id, an exact match
cur.execute("""
SELECT market_id, question, category, current_probability,
volume_24h, liquidity, end_date_iso, status, slug
FROM qd_polymarket_markets
WHERE market_id = %s AND status = 'active'
ORDER BY volume_24h DESC
LIMIT %s
""", (keyword, limit))
elif has_hyphens:
# If it contains a hyphen, it may be a slug, and the slug will be matched first.
cur.execute("""
SELECT market_id, question, category, current_probability,
volume_24h, liquidity, end_date_iso, status, slug
FROM qd_polymarket_markets
WHERE (slug ILIKE %s OR question ILIKE %s) AND status = 'active'
ORDER BY
CASE WHEN slug ILIKE %s THEN 1 ELSE 2 END,
volume_24h DESC
LIMIT %s
""", (f"%{keyword}%", f"%{keyword}%", f"%{keyword}%", limit))
else:
# Normal text search
cur.execute("""
SELECT market_id, question, category, current_probability,
volume_24h, liquidity, end_date_iso, status, slug
FROM qd_polymarket_markets
WHERE (question ILIKE %s OR slug ILIKE %s) AND status = 'active'
ORDER BY volume_24h DESC
LIMIT %s
""", (f"%{keyword}%", f"%{keyword}%", limit))
rows = cur.fetchall()
cur.close()
if rows:
logger.info(f"Found {len(rows)} markets in database for keyword '{keyword}'")
return [{
"market_id": str(row.get('market_id') or ''),
"question": row.get('question') or '',
"category": row.get('category') or 'other',
"current_probability": float(row.get('current_probability') or 0),
"volume_24h": float(row.get('volume_24h') or 0),
"liquidity": float(row.get('liquidity') or 0),
"end_date_iso": row.get('end_date_iso'),
"status": row.get('status') or 'active',
"polymarket_url": self._build_polymarket_url(row.get('slug'), row.get('market_id') or ''),
"slug": row.get('slug') if row.get('slug') and not str(row.get('slug', '')).isdigit() else None
} for row in rows]
# Obtain and filter directly from Gamma API (used during AI analysis)
logger.info(f"Fetching from API for keyword '{keyword}' (use_cache={use_cache})...")
# Optimization: If the keyword looks like a slug, try direct query first (avoid fetching the full amount)
import re
keyword_lower = keyword.lower().strip()
is_slug_like = '-' in keyword_lower and not keyword_lower.isdigit()
if is_slug_like:
# Try to query directly through slug (most efficient, according to Polymarket API documentation)
direct_market = self._fetch_market_by_slug(keyword_lower)
if direct_market:
logger.info(f"Found market directly by slug (no need to fetch all markets): {keyword_lower}")
return [direct_market]
# If direct query fails, get more data so that there is enough room for selection
# Make multiple requests to get more markets (up to 100 events each time, but each event may contain multiple markets)
all_markets = []
max_requests = 3 # Request up to 3 times to get 300 events (about 4500 markets)
for page in range(max_requests):
page_markets = self._fetch_from_gamma_api(category=None, limit=100)
if not page_markets:
break
all_markets.extend(page_markets)
# If enough markets have been acquired, you can stop early
if len(all_markets) >= 3000: # Get up to 3000 markets
break
logger.info(f"Fetched page {page + 1}/{max_requests}, total markets: {len(all_markets)}")
# Short delay to avoid API current limit
if page < max_requests - 1:
time.sleep(0.5)
logger.info(f"Fetched {len(all_markets)} markets from API, filtering for keyword '{keyword}'...")
# Filter by keyword (supports multiple keyword matching)
# If the keyword looks like a slug (contains a hyphen), also try to match the slug
keyword_is_slug = '-' in keyword_lower
# Extract keywords (remove common stop words and punctuation)
# Extract keywords: remove punctuation, retain alphanumeric characters and hyphens
keyword_words = re.findall(r'\b\w+\b', keyword_lower)
# Filter out words that are too short (less than 3 characters) and common stop words
stop_words = {'the', 'a', 'an', 'and', 'or', 'but', 'in', 'on', 'at', 'to', 'for', 'of', 'with', 'by', 'is', 'are', 'was', 'were', 'be', 'been', 'will', 'would', 'should', 'could', 'may', 'might', 'can', 'must'}
keyword_words = [w for w in keyword_words if len(w) >= 3 and w not in stop_words]
# If no keywords are extracted, use the original keywords
if not keyword_words:
keyword_words = [keyword_lower]
logger.info(f"Extracted keywords: {keyword_words} from '{keyword}'")
filtered = []
scored_markets = [] # Used to store scored results
top_candidates = [] # Used to store close matching candidates (for debugging)
for market in all_markets:
question = market.get("question", "").lower()
slug = (market.get("slug") or "").lower()
market_id = str(market.get("market_id") or "")
score = 0
match_reason = ""
# 1. Exact match (highest priority, score 100)
if keyword_lower in question:
score = 100
match_reason = "exact_match_question"
elif keyword_lower == slug:
score = 100
match_reason = "exact_match_slug"
# 2. If the keyword looks like a slug, check the slug field
if score < 100 and keyword_is_slug:
if keyword_lower == slug:
score = 100
match_reason = "exact_slug_match"
elif keyword_lower in slug or slug in keyword_lower:
score = 90
match_reason = "partial_slug_match"
# 3. If the keyword is a pure number, check market_id
if score < 90 and keyword_lower.isdigit():
if keyword_lower == market_id:
score = 100
match_reason = "market_id_match"
# 4. Keyword matching: Check whether all keywords are in the question
if score < 90 and keyword_words:
# Calculate the number of matching keywords
matched_words = sum(1 for word in keyword_words if word in question or word in slug)
if matched_words > 0:
# Match rate
match_ratio = matched_words / len(keyword_words)
# Lower the threshold: from 60% to 40% to improve the matching rate
if match_ratio >= 0.4:
score = int(60 + match_ratio * 30) # 60-90 minutes
match_reason = f"keyword_match_{matched_words}/{len(keyword_words)}"
else:
# Log close matching candidates (for debugging)
if matched_words >= 1 and len(top_candidates) < 5:
top_candidates.append((match_ratio, market.get('question', '')[:80], matched_words, len(keyword_words)))
# 5. Partial matching: Check whether the main part of the keyword is in the question
if score < 60 and keyword_words:
# If the keyword contains multiple words, try to match the main part
if len(keyword_words) > 1:
# Take the first 3 most important words (usually nouns)
important_words = keyword_words[:3]
matched_important = sum(1 for word in important_words if word in question or word in slug)
# Lower the requirement: match at least 1 important word
if matched_important >= 1:
score = 50
match_reason = f"important_words_match_{matched_important}/{len(important_words)}"
if score >= 50: # Lower the minimum score requirement from 60 to 50
scored_markets.append((score, market, match_reason))
logger.debug(f"Matched (score={score}, reason={match_reason}): {market.get('question', '')[:60]}")
# Sort by score, take the first limit
scored_markets.sort(key=lambda x: x[0], reverse=True)
filtered = [market for score, market, reason in scored_markets[:limit]]
# Output debugging information
if len(scored_markets) == 0 and top_candidates:
logger.warning(f"No exact matches found. Top candidates (partial matches):")
for ratio, question, matched, total in top_candidates:
logger.warning(f" - {question} (matched {matched}/{total} keywords, ratio={ratio:.2f})")
logger.info(f"Filtered {len(filtered)} markets matching keyword '{keyword}' from API (from {len(all_markets)} total markets, {len(scored_markets)} scored matches)")
if len(scored_markets) > 0:
logger.info(f"Top match: {filtered[0].get('question', '')[:80]} (score={scored_markets[0][0]})")
return filtered
except Exception as e:
logger.error(f"Failed to search markets: {e}", exc_info=True)
return []
def _get_cached_markets(self, category: str = None, limit: int = 50) -> Optional[List[Dict]]:
"""Read market data from the database cache."""
try:
with get_db_connection() as db:
cur = db.cursor()
# Check cache is fresh (within 5 minutes)
cutoff_time = datetime.now() - timedelta(seconds=self.cache_ttl)
query = """
SELECT market_id, question, category, current_probability,
volume_24h, liquidity, end_date_iso, status, outcome_tokens
FROM qd_polymarket_markets
WHERE status = 'active' AND updated_at > %s
"""
params = [cutoff_time]
if category:
query += " AND category = %s"
params.append(category)
query += " ORDER BY volume_24h DESC LIMIT %s"
params.append(limit)
cur.execute(query, params)
rows = cur.fetchall()
cur.close()
if rows:
result = []
for row in rows:
market_id = str(row.get('market_id') or '')
slug = row.get('slug')
# Make sure to use the correct URL building method
polymarket_url = self._build_polymarket_url(slug, market_id)
result.append({
"market_id": market_id,
"question": row.get('question') or '',
"category": row.get('category') or 'other',
"current_probability": float(row.get('current_probability') or 0),
"volume_24h": float(row.get('volume_24h') or 0),
"liquidity": float(row.get('liquidity') or 0),
"end_date_iso": row.get('end_date_iso'),
"status": row.get('status') or 'active',
"outcome_tokens": row.get('outcome_tokens') if row.get('outcome_tokens') else {},
"polymarket_url": polymarket_url,
"slug": slug if slug and not str(slug).isdigit() else None
})
return result
return None
except Exception as e:
logger.debug(f"Failed to get cached markets: {e}")
return None
def _fetch_markets_from_api(self, category: str = None, limit: int = 50) -> List[Dict]:
"""
Retrieve market data from the Polymarket Gamma API
Use the officially recommended /events endpoint
"""
try:
# Use the /events endpoint of Gamma API (official recommended method)
markets = self._fetch_from_gamma_api(category, limit)
if markets:
# Sort by volume_24h in descending order (because the API does not support the order parameter, local sorting is required)
markets.sort(key=lambda x: x.get('volume_24h', 0), reverse=True)
return markets[:limit] # Return the first limit after sorting
# If the API returns an empty list, record a warning (it may be that the API is temporarily unavailable, network problems or throttling)
logger.warning(f"Gamma API failed to fetch markets for category '{category}' (possible reasons: API is temporarily unavailable, network problems, current limiting or returning empty data)")
return []
except Exception as e:
logger.error(f"Failed to fetch markets from API: {e}", exc_info=True)
return []
def _fetch_from_gamma_api(self, category: str = None, limit: int = 50) -> List[Dict]:
"""
Retrieve market data from the Polymarket Gamma API
Use the officially recommended /events endpoint
"""
try:
# Use the /events endpoint to obtain active markets (recommended method)
# According to official documentation: https://docs.polymarket.com/market-data/fetching-markets
# Supported values ​​for the order parameter: volume_24hr, volume, liquidity, competitive, start_date, end_date
# But some endpoints may not support it, try without the order parameter first.
url = f"{self.gamma_api}/events"
params = {
"active": "true",
"closed": "false",
"limit": min(limit * 2, 100) # Get more data for sorting and filtering
}
# Try adding sorting parameters (if supported by API)
# According to the documentation, possible sorting fields: volume_24hr, volume, liquidity, etc.
# If the API is not supported, it will be removed after a 422 error.
# If a category is specified, it needs to be filtered by tag_id
# Note: You need to get the tag_id first, and use keywords to infer it here.
if category:
# You can try filtering by search or tags
# Get them all temporarily, then filter when parsing
pass
logger.info(f"Fetching from Gamma API: {url} with params: {params}")
response = self.session.get(url, params=params, timeout=15)
logger.info(f"Gamma API response status: {response.status_code}")
if response.status_code == 200:
try:
data = response.json()
logger.debug(f"Gamma API returned data type: {type(data)}, keys: {list(data.keys()) if isinstance(data, dict) else 'list'}")
#The Gamma API may return a list or an object containing a data field
if isinstance(data, list):
logger.info(f"Gamma API returned list with {len(data)} items")
markets = self._parse_gamma_events(data, category)
logger.info(f"Parsed {len(markets)} markets from Gamma API")
return markets
elif isinstance(data, dict):
# Possibly {"data": [...]} format
if "data" in data:
events_list = data["data"]
logger.info(f"Gamma API returned dict with 'data' field containing {len(events_list) if isinstance(events_list, list) else 'non-list'} items")
markets = self._parse_gamma_events(events_list, category)
logger.info(f"Parsed {len(markets)} markets from Gamma API")
return markets
# Or directly the event object
elif "id" in data or "slug" in data:
logger.info("Gamma API returned single event object")
markets = self._parse_gamma_events([data], category)
logger.info(f"Parsed {len(markets)} markets from Gamma API")
return markets
else:
logger.warning(f"Gamma API returned dict with unexpected keys: {list(data.keys())}")
logger.debug(f"Full response: {str(data)[:500]}")
logger.warning(f"Gamma API returned unexpected format: {type(data)}")
return []
except json.JSONDecodeError as je:
logger.error(f"Gamma API returned invalid JSON: {je}")
logger.error(f"Response text (first 500 chars): {response.text[:500]}")
return []
# Non-200 status code
status_code = response.status_code
if status_code == 429:
logger.warning(f"Gamma API rate limited (429). Suggestion: Try again later or reduce the request frequency")
elif status_code == 503:
logger.warning(f"Gamma API service unavailable (503). Polymarket API may be under maintenance")
elif status_code >= 500:
logger.warning(f"Gamma API server error ({status_code}). The Polymarket server may be temporarily unavailable")
else:
logger.warning(f"Gamma API returned status {status_code}")
logger.debug(f"Response headers: {dict(response.headers)}")
logger.debug(f"Response text (first 500 chars): {response.text[:500]}")
return []
except requests.exceptions.Timeout:
logger.warning("Gamma API request timeout after 15 seconds (possible reason: network delay or slow API response)")
return []
except requests.exceptions.ConnectionError as ce:
logger.warning(f"Gamma API connection error: {ce} (Possible reason: network connection problem or Polymarket API is unreachable)")
return []
except Exception as e:
logger.warning(f"Gamma API failed: {e} (Possible reasons: API format change, network problem or service abnormality)")
return []
def _parse_gamma_events(self, events_data: List[Dict], category_filter: str = None) -> List[Dict]:
"""
Parse event data returned by the Gamma API
The /events endpoint of the Gamma API returns event objects, each containing associated market data
According to the official documentation, the event object structure is:
- The event object contains a markets array
- Each market contains fields such as clobTokenIds and outcomePrices
"""
parsed = []
if not events_data:
logger.warning("_parse_gamma_events received empty events_data")
return parsed
logger.info(f"Parsing {len(events_data)} events from Gamma API")
# Record the key of the first event for debugging
if events_data:
first_event_keys = list(events_data[0].keys())[:10]
logger.info(f"First event keys: {first_event_keys}")
logger.debug(f"First event sample: {str(events_data[0])[:500]}")
for idx, event in enumerate(events_data):
try:
#Gamma API event object structure
# The event may have multiple markets (markets field), or directly contain market information
markets = event.get("markets", [])
# If the event does not have a markets field, the event itself may be market data.
if not markets:
# Check whether it is a market object directly (with question or title field)
if "question" in event or "title" in event or "slug" in event:
markets = [event]
else:
if idx < 3: # Only record the details of the first 3
logger.debug(f"Event {idx} has no markets and doesn't look like a market. Keys: {list(event.keys())[:10]}")
continue
if idx < 3: # Only record the details of the first 3
logger.debug(f"Processing event {idx} with {len(markets)} markets")
for market_idx, market in enumerate(markets):
# Extract basic market information
market_id = market.get("id") or market.get("slug") or event.get("id") or event.get("slug", "")
question = market.get("question") or event.get("question") or market.get("title") or event.get("title", "")
if idx < 3 and market_idx < 2: # Record detailed information of the first few markets
logger.info(f"Event {idx}, Market {market_idx}: id={market_id}, question={question[:50] if question else 'None'}, event_slug={event.get('slug')}, market_slug={market.get('slug')}, keys={list(market.keys())[:10]}")
if not question:
if idx < 3:
logger.warning(f"Event {idx}, Market {market_idx}: No question found, skipping. Market keys: {list(market.keys())[:10]}")
continue
# Infer category
inferred_category = self._infer_category(question)
# If category filtering is specified, filter
if category_filter and inferred_category != category_filter:
continue
# Get probability and outcome data
current_probability = 50.0
outcome_tokens = {}
# Method 1: Get real-time prices from CLOB API (most accurate)
try:
condition_id = market.get("conditionId") or event.get("conditionId")
if condition_id:
prices = self._get_market_prices_from_clob(condition_id)
if prices:
yes_price = prices.get("YES", 0)
no_price = prices.get("NO", 0)
if yes_price > 0:
current_probability = yes_price * 100 if yes_price <= 1 else yes_price
outcome_tokens["YES"] = {"price": yes_price if yes_price <= 1 else yes_price / 100, "volume": 0}
if no_price > 0:
outcome_tokens["NO"] = {"price": no_price if no_price <= 1 else no_price / 100, "volume": 0}
except Exception as e:
logger.debug(f"Failed to get prices from CLOB API: {e}")
# Method 2: Process the outcomePrices field (may be a JSON string)
if current_probability == 50.0:
outcome_prices_str = market.get("outcomePrices") or event.get("outcomePrices")
if outcome_prices_str:
try:
if isinstance(outcome_prices_str, str):
outcome_prices = json.loads(outcome_prices_str)
else:
outcome_prices = outcome_prices_str
# outcomePrices is usually in the format ["0.65", "0.35"], corresponding to YES and NO
if isinstance(outcome_prices, list) and len(outcome_prices) >= 2:
yes_price = float(outcome_prices[0]) if outcome_prices[0] else 0
no_price = float(outcome_prices[1]) if outcome_prices[1] else 0
current_probability = yes_price * 100 if yes_price <= 1 else yes_price
outcome_tokens["YES"] = {"price": yes_price if yes_price <= 1 else yes_price / 100, "volume": 0}
outcome_tokens["NO"] = {"price": no_price if no_price <= 1 else no_price / 100, "volume": 0}
except Exception as e:
logger.debug(f"Failed to parse outcomePrices: {e}")
# Get outcomes from market or event
#outcomes may be an object array, a string array, or need to be parsed from other fields
outcomes = market.get("outcomes") or market.get("tokens") or event.get("outcomes") or []
# Process the outcomes array (may be an object or a string)
for outcome in outcomes:
try:
# If outcome is a string, skip or try to parse
if isinstance(outcome, str):
# May be a simple string identifier such as "YES" or "NO"
outcome_upper = outcome.upper()
if "YES" in outcome_upper:
if "YES" not in outcome_tokens:
outcome_tokens["YES"] = {"price": 0.5, "volume": 0}
elif "NO" in outcome_upper:
if "NO" not in outcome_tokens:
outcome_tokens["NO"] = {"price": 0.5, "volume": 0}
continue
# outcome is an object
if not isinstance(outcome, dict):
continue
title = str(outcome.get("title") or outcome.get("name", "")).upper()
# Get the price (may be price, probability or currentPrice)
price = float(outcome.get("price") or outcome.get("probability") or outcome.get("currentPrice") or 0)
if "YES" in title or title == "YES" or outcome.get("outcome") == "Yes":
current_probability = price * 100 if price <= 1 else price
outcome_tokens["YES"] = {
"price": price if price <= 1 else price / 100,
"volume": float(outcome.get("volume", outcome.get("volume24hr", 0)) or 0)
}
elif "NO" in title or title == "NO" or outcome.get("outcome") == "No":
outcome_tokens["NO"] = {
"price": price if price <= 1 else price / 100,
"volume": float(outcome.get("volume", outcome.get("volume24hr", 0)) or 0)
}
except Exception as e:
logger.debug(f"Failed to parse outcome: {e}")
continue
# If outcomes are not found, try to get probabilities from other fields
if current_probability == 50.0:
# Try to get it from the probability field of the market
prob = market.get("probability") or market.get("yesProbability") or event.get("probability")
if prob:
current_probability = float(prob) * 100 if float(prob) <= 1 else float(prob)
# Get trading volume and liquidity
volume_24h = float(
market.get("volume_24hr") or
market.get("volume24hr") or
market.get("volume_24h") or
event.get("volume_24hr") or
event.get("volume24hr") or
0
)
liquidity = float(
market.get("liquidity") or
market.get("totalLiquidity") or
event.get("liquidity") or
0
)
# Parse end date
end_date_iso = None
end_date = market.get("endDate") or market.get("end_date") or event.get("endDate") or event.get("end_date")
if end_date:
try:
if isinstance(end_date, (int, float)):
end_date_iso = datetime.fromtimestamp(end_date).isoformat() + "Z"
elif isinstance(end_date, str):
# Try to parse the ISO format string
end_date_iso = end_date
except:
pass
# Get the slug used to build the URL
# According to Polymarket API documentation: slug should be obtained directly from the data returned by the API
# URL format: https://polymarket.com/event/{slug}
# slug is a string identifier, not a numeric ID
slug = None
# Get the slug from the event first (because the event contains markets)
if event.get("slug"):
slug_str = str(event.get("slug", "")).strip()
# If the slug is not a pure number and contains letters or hyphens, it is a valid slug
if slug_str and not slug_str.isdigit() and ('-' in slug_str or any(c.isalpha() for c in slug_str)):
slug = slug_str
# If the event does not have a valid slug, try to get it from the market
if not slug and market.get("slug"):
slug_str = str(market.get("slug", "")).strip()
if slug_str and not slug_str.isdigit() and ('-' in slug_str or any(c.isalpha() for c in slug_str)):
slug = slug_str
# If there is still no valid slug, try to obtain it through API query
if not slug and market_id:
try:
# Use the markets endpoint to query by ID to obtain complete slug information
detail_market = self._fetch_market_detail_by_id(market_id)
if detail_market and detail_market.get("slug"):
slug_str = str(detail_market.get("slug", "")).strip()
if slug_str and not slug_str.isdigit() and ('-' in slug_str or any(c.isalpha() for c in slug_str)):
slug = slug_str
except Exception as e:
logger.debug(f"Failed to fetch slug for market {market_id}: {e}")
# Build URL (using unified helper methods)
polymarket_url = self._build_polymarket_url(slug, market_id)
if not slug:
logger.warning(f"Market {market_id} has no valid slug, using markets endpoint as fallback")
market_data = {
"market_id": market_id,
"question": question,
"category": inferred_category,
"current_probability": round(current_probability, 2),
"volume_24h": volume_24h,
"liquidity": liquidity,
"end_date_iso": end_date_iso,
"status": "active" if market.get("active", event.get("active", True)) else "closed",
"outcome_tokens": outcome_tokens,
"polymarket_url": polymarket_url,
"slug": slug if slug else None # Save slug (if not a number)
}
parsed.append(market_data)
if idx < 3 and market_idx < 2: # Record successfully parsed markets
logger.info(f"Successfully parsed market: {question[:50]}, prob={current_probability:.1f}%, volume={volume_24h}")
except Exception as e:
logger.warning(f"Failed to parse event {idx} (id={event.get('id', event.get('slug', 'unknown'))}): {e}", exc_info=True)
continue
logger.info(f"Successfully parsed {len(parsed)} markets from {len(events_data)} events")
return parsed
def _parse_rest_markets(self, markets_data: List[Dict]) -> List[Dict]:
"""Parse REST API data"""
parsed = []
for market in markets_data:
try:
# Extract basic information
market_id = market.get("id") or market.get("slug") or market.get("market_id", "")
question = market.get("question") or market.get("title", "")
# Calculate probability
current_probability = 50.0
outcome_tokens = {}
if "outcomes" in market:
for outcome in market["outcomes"]:
title = str(outcome.get("title", "")).upper()
price = float(outcome.get("price", outcome.get("probability", 0)) or 0)
if "YES" in title or title == "YES":
current_probability = price * 100
outcome_tokens["YES"] = {
"price": price,
"volume": float(outcome.get("volume", 0) or 0)
}
elif "NO" in title or title == "NO":
outcome_tokens["NO"] = {
"price": price,
"volume": float(outcome.get("volume", 0) or 0)
}
volume_24h = float(market.get("volume_24h", market.get("volume", 0)) or 0)
liquidity = float(market.get("liquidity", 0) or 0)
# inferred category
category = self._infer_category(question)
# parse end date
end_date_iso = market.get("end_date") or market.get("endDate")
if isinstance(end_date_iso, (int, float)):
try:
end_date_iso = datetime.fromtimestamp(end_date_iso).isoformat() + "Z"
except:
end_date_iso = None
# Get the slug used to build the URL
slug = None
slug_str = str(market.get('slug', '')).strip() if market.get('slug') else ''
# Check if the slug is valid (not a number and contains letters or hyphens)
if slug_str and not slug_str.isdigit() and ('-' in slug_str or any(c.isalpha() for c in slug_str)):
slug = slug_str
else:
# If the slug is invalid, try to obtain it through API query
try:
detail_market = self._fetch_market_detail_by_id(market_id)
if detail_market and detail_market.get("slug"):
slug_str = str(detail_market.get("slug", "")).strip()
if slug_str and not slug_str.isdigit() and ('-' in slug_str or any(c.isalpha() for c in slug_str)):
slug = slug_str
except Exception as e:
logger.debug(f"Failed to fetch slug for market {market_id}: {e}")
# Build URL (using unity helper methods)
polymarket_url = self._build_polymarket_url(slug, market_id)
if not slug:
logger.warning(f"Market {market_id} has no valid slug, using markets endpoint as fallback")
parsed.append({
"market_id": market_id,
"question": question,
"category": category,
"current_probability": round(current_probability, 2),
"volume_24h": volume_24h,
"liquidity": liquidity,
"end_date_iso": end_date_iso,
"status": "active" if market.get("active", True) else "closed",
"outcome_tokens": outcome_tokens,
"polymarket_url": polymarket_url,
"slug": slug if slug else None
})
except Exception as e:
logger.debug(f"Failed to parse market {market.get('id')}: {e}")
continue
return parsed
def _infer_category(self, question: str) -> str:
"""Infer categories from questions"""
question_lower = question.lower()
# Cryptocurrency Keywords
crypto_keywords = ['btc', 'bitcoin', 'eth', 'ethereum', 'sol', 'solana', 'crypto', 'token', 'coin', 'defi', 'nft']
if any(kw in question_lower for kw in crypto_keywords):
return "crypto"
# political keywords
politics_keywords = ['election', 'president', 'trump', 'biden', 'senate', 'congress', 'vote', 'political', 'democrat', 'republican']
if any(kw in question_lower for kw in politics_keywords):
return "politics"
# economic keywords
economics_keywords = ['gdp', 'inflation', 'unemployment', 'fed', 'federal reserve', 'interest rate', 'economic', 'economy', 'recession', 'gdp growth', 'cpi', 'ppi']
if any(kw in question_lower for kw in economics_keywords):
return "economics"
# Sports keywords
sports_keywords = ['nfl', 'nba', 'mlb', 'soccer', 'football', 'basketball', 'baseball', 'championship', 'world cup', 'olympics', 'super bowl', 'stanley cup', 'world series']
if any(kw in question_lower for kw in sports_keywords):
return "sports"
# Technology keywords
tech_keywords = ['ai', 'artificial intelligence', 'chatgpt', 'openai', 'tech', 'technology', 'apple', 'google', 'microsoft', 'meta', 'tesla', 'ipo', 'startup']
if any(kw in question_lower for kw in tech_keywords):
return "tech"
# financial keywords
finance_keywords = ['stock', 's&p', 'dow', 'nasdaq', 'market cap', 'earnings', 'revenue', 'profit', 'bank', 'banking', 'financial', 'trading']
if any(kw in question_lower for kw in finance_keywords):
return "finance"
# geopolitical keywords
geopolitics_keywords = ['war', 'conflict', 'russia', 'ukraine', 'china', 'taiwan', 'north korea', 'iran', 'israel', 'palestine', 'middle east', 'nato', 'sanctions']
if any(kw in question_lower for kw in geopolitics_keywords):
return "geopolitics"
# Cultural keywords
culture_keywords = ['movie', 'film', 'oscar', 'grammy', 'award', 'celebrity', 'music', 'album', 'tv show', 'series', 'netflix', 'disney']
if any(kw in question_lower for kw in culture_keywords):
return "culture"
# climate keywords
climate_keywords = ['climate', 'global warming', 'temperature', 'carbon', 'emission', 'renewable', 'solar', 'wind energy', 'paris agreement', 'cop']
if any(kw in question_lower for kw in climate_keywords):
return "climate"
# Entertainment keywords
entertainment_keywords = ['game', 'gaming', 'esports', 'tournament', 'streaming', 'youtube', 'twitch', 'podcast', 'comic', 'anime', 'manga']
if any(kw in question_lower for kw in entertainment_keywords):
return "entertainment"
return "other"
def _build_polymarket_url(self, slug: Optional[str], market_id: str) -> str:
"""
Build Polymarket URL based on slug
Reference: https://docs.polymarket.com/market-data/fetching-markets
Args:
slug: slug obtained from API or database (may be None or numeric string)
market_id: Market ID (as an alternative)
Returns:
Polymarket URL string
"""
if slug:
slug_str = str(slug).strip()
# Check if the slug is valid (not a number and contains letters or hyphens)
if slug_str and not slug_str.isdigit() and ('-' in slug_str or any(c.isalpha() for c in slug_str)):
import re
slug_clean = re.sub(r'[^a-zA-Z0-9\-]', '-', slug_str)
slug_clean = slug_clean.strip('-')
if slug_clean:
return f"https://polymarket.com/event/{slug_clean}"
# If there is no valid slug, try to obtain the slug through the API
if market_id:
try:
detail_market = self._fetch_market_detail_by_id(market_id)
if detail_market:
# Try to get the slug from detail
event_slug = detail_market.get('slug')
if event_slug:
slug_str = str(event_slug).strip()
if slug_str and not slug_str.isdigit() and ('-' in slug_str or any(c.isalpha() for c in slug_str)):
import re
slug_clean = re.sub(r'[^a-zA-Z0-9\-]', '-', slug_str)
slug_clean = slug_clean.strip('-')
if slug_clean:
return f"https://polymarket.com/event/{slug_clean}"
# If the event does not have a slug, try to get it from markets
markets = detail_market.get('markets', [])
if markets:
for m in markets:
market_slug = m.get('slug')
if market_slug:
slug_str = str(market_slug).strip()
if slug_str and not slug_str.isdigit() and ('-' in slug_str or any(c.isalpha() for c in slug_str)):
import re
slug_clean = re.sub(r'[^a-zA-Z0-9\-]', '-', slug_str)
slug_clean = slug_clean.strip('-')
if slug_clean:
return f"https://polymarket.com/event/{slug_clean}"
except Exception as e:
logger.debug(f"Failed to fetch slug for market {market_id}: {e}")
# If all else fails, return to the search page (more reliable)
# Note: Polymarket's URL format may have changed, use search as fallback
return f"https://polymarket.com/search?q={market_id}"
def _fetch_market_detail_by_id(self, market_id: str) -> Optional[Dict]:
"""
Get market details from API by market ID (used to get slug)
Reference: https://docs.polymarket.com/market-data/fetching-markets
"""
try:
# Method 1: Try querying through the events endpoint (recommended because events include markets)
url = f"{self.gamma_api}/events"
params = {"active": "true", "closed": "false", "limit": 100}
response = self.session.get(url, params=params, timeout=10)
if response.status_code == 200:
events = response.json()
if isinstance(events, list):
for event in events:
markets = event.get("markets", [])
if not markets and ("question" in event or "slug" in event):
markets = [event]
for market in markets:
m_id = market.get("id") or market.get("slug") or ""
e_id = event.get("id") or event.get("slug") or ""
# Match market_id or event_id
if str(m_id) == str(market_id) or str(e_id) == str(market_id):
# Return event (because event contains slug)
return event
elif isinstance(events, dict):
if "data" in events:
events_list = events["data"]
for event in events_list:
markets = event.get("markets", [])
if not markets and ("question" in event or "slug" in event):
markets = [event]
for market in markets:
m_id = market.get("id") or market.get("slug") or ""
e_id = event.get("id") or event.get("slug") or ""
if str(m_id) == str(market_id) or str(e_id) == str(market_id):
return event
# Method 2: Try querying through the markets endpoint
url = f"{self.gamma_api}/markets"
params = {"id": market_id, "limit": 1}
response = self.session.get(url, params=params, timeout=10)
if response.status_code == 200:
data = response.json()
if isinstance(data, list) and len(data) > 0:
return data[0]
elif isinstance(data, dict) and "id" in data:
return data
return None
except Exception as e:
logger.debug(f"Failed to fetch market detail by ID {market_id}: {e}")
return None
def _fetch_market_by_slug(self, slug: str) -> Optional[Dict]:
"""
Query the market directly through slug (the most efficient way)
According to Polymarket API documentation: https://docs.polymarket.com/market-data/fetching-markets
You can use /markets?slug=xxx to query directly
"""
try:
# Method 1: Try querying the slug directly through the markets endpoint
url = f"{self.gamma_api}/markets"
params = {"slug": slug, "limit": 10}
logger.info(f"Fetching market by slug from Gamma API: {url} with params: {params}")
response = self.session.get(url, params=params, timeout=10)
if response.status_code == 200:
data = response.json()
if isinstance(data, list) and len(data) > 0:
# Parse the returned market data
markets = self._parse_gamma_events(data)
# Exact match slug
for market in markets:
market_slug = market.get("slug", "").lower()
if market_slug == slug.lower() or slug.lower() in market_slug:
logger.info(f"Found market by slug: {slug}")
return market
# If there is no exact match, return the first
if markets:
logger.info(f"Found market by slug (fuzzy match): {slug}")
return markets[0]
elif isinstance(data, dict):
# single market object
markets = self._parse_gamma_events([data])
if markets:
logger.info(f"Found market by slug: {slug}")
return markets[0]
# Method 2: Try to query through the events endpoint (events may contain slug information)
url = f"{self.gamma_api}/events"
params = {"active": "true", "closed": "false", "limit": 100}
response = self.session.get(url, params=params, timeout=10)
if response.status_code == 200:
data = response.json()
events = data if isinstance(data, list) else (data.get("data", []) if isinstance(data, dict) else [])
# Find the matching slug in the returned event
for event in events:
event_slug = (event.get("slug") or "").lower()
if event_slug == slug.lower() or slug.lower() in event_slug:
parsed = self._parse_gamma_events([event])
if parsed:
logger.info(f"Found market by slug via events: {slug}")
return parsed[0]
logger.warning(f"Market with slug '{slug}' not found via direct query")
return None
except Exception as e:
logger.error(f"Failed to fetch market by slug {slug}: {e}", exc_info=True)
return None
def _fetch_market_from_api(self, market_id: str) -> Optional[Dict]:
"""
Get individual market data from Gamma API
Support query by slug or id
"""
try:
# Determine whether it is slug or market_id
is_slug = not market_id.isdigit() and ('-' in market_id or any(c.isalpha() for c in market_id))
# If it is a slug, the direct query method is preferred.
if is_slug:
market = self._fetch_market_by_slug(market_id)
if market:
return market
# Method 1: Query through markets endpoint (supports id and slug)
url = f"{self.gamma_api}/markets"
params = {"id": market_id, "limit": 10} if not is_slug else {"slug": market_id, "limit": 10}
response = self.session.get(url, params=params, timeout=10)
if response.status_code == 200:
data = response.json()
if isinstance(data, list) and len(data) > 0:
markets = self._parse_gamma_events(data)
if markets:
return markets[0]
elif isinstance(data, dict):
markets = self._parse_gamma_events([data])
if markets:
return markets[0]
# Method 2: Search via events endpoint (as an alternative)
url = f"{self.gamma_api}/events"
params = {
"active": "true",
"closed": "false",
"limit": 100
}
response = self.session.get(url, params=params, timeout=10)
if response.status_code == 200:
data = response.json()
events = data if isinstance(data, list) else (data.get("data", []) if isinstance(data, dict) else [])
# Find matching markets in returned events
for event in events:
markets = event.get("markets", [])
if not markets:
markets = [event]
for market in markets:
m_id = market.get("id") or market.get("slug") or event.get("id") or event.get("slug", "")
if str(m_id) == str(market_id) or market.get("slug") == market_id:
parsed = self._parse_gamma_events([event])
if parsed:
return parsed[0]
return None
except Exception as e:
logger.error(f"Failed to fetch market {market_id}: {e}", exc_info=True)
return None
def _save_markets_to_db(self, markets: List[Dict]):
"""Save market data to database"""
try:
with get_db_connection() as db:
cur = db.cursor()
for market in markets:
# Get the slug, but don't use it if it's a number (numbers are not valid slugs)
slug = market.get('slug') or None
# If the slug is a number, it means it is not a valid slug and is set to None.
if slug and str(slug).isdigit():
slug = None
# Clean slug, keep only alphanumerics and hyphens
import re
if slug:
slug = re.sub(r'[^a-zA-Z0-9\-]', '-', str(slug))
slug = slug.strip('-')
# If it is empty or still a number after cleaning, set to None
if not slug or slug.isdigit():
slug = None
cur.execute("""
INSERT INTO qd_polymarket_markets
(market_id, question, category, current_probability, volume_24h,
liquidity, end_date_iso, status, outcome_tokens, slug, updated_at)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, NOW())
ON CONFLICT (market_id) DO UPDATE SET
question = EXCLUDED.question,
category = EXCLUDED.category,
current_probability = EXCLUDED.current_probability,
volume_24h = EXCLUDED.volume_24h,
liquidity = EXCLUDED.liquidity,
end_date_iso = EXCLUDED.end_date_iso,
status = EXCLUDED.status,
outcome_tokens = EXCLUDED.outcome_tokens,
slug = EXCLUDED.slug,
updated_at = NOW()
""", (
market.get('market_id'),
market.get('question'),
market.get('category', 'other'),
market.get('current_probability', 50.0),
market.get('volume_24h', 0),
market.get('liquidity', 0),
market.get('end_date_iso'),
market.get('status', 'active'),
json.dumps(market.get('outcome_tokens', {})),
slug
))
db.commit()
cur.close()
except Exception as e:
logger.error(f"Failed to save markets to DB: {type(e).__name__}: {e}", exc_info=True)
def _get_sample_markets(self, category: str = None, limit: int = 50) -> List[Dict]:
"""
Get sample market data (deprecated)
You should now use real API data.
"""
# No longer returns sample data, returns an empty list
logger.warning("Sample data method called, but real API should be used instead")
return []
Reference in New Issue View Git Blame Copy Permalink