""" Polymarket prediction market data source Get prediction market data from Polymarket """ import time import requests import json from typing import Dict, List, Any, Optional from datetime import datetime, timedelta from app.utils.logger import get_logger from app.utils.db import get_db_connection logger = get_logger(__name__) class PolymarketDataSource: """Polymarket prediction market data source""" def __init__(self): # Polymarket official API endpoint (according to official documentation) # Gamma API: markets, events, tags, searches, etc. (fully public, no authentication required) self.gamma_api = "https://gamma-api.polymarket.com" # Data API: User positions, transactions, activities, etc. (fully public, no authentication required) self.data_api = "https://data-api.polymarket.com" # CLOB API: Order book, prices, trading operations (public endpoints require no authentication) self.clob_api = "https://clob.polymarket.com" self.cache_ttl = 300 # 5 minutes cache self.session = requests.Session() self.session.headers.update({ 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36', 'Accept': 'application/json' }) def get_trending_markets(self, category: str = None, limit: int = 50) -> List[Dict]: """ Get popular prediction markets Args: category: category filter (crypto, politics, economics, sports, all) limit: return quantity limit Returns: Prediction market list """ try: # Read from database cache first cached = self._get_cached_markets(category, limit) if cached: return cached # Fetch from real API - Fetch data from multiple categories to ensure diversity all_markets = [] if category and category != "all": # If a category is specified, only data for that category will be obtained markets = self._fetch_markets_from_api(category, limit * 2) all_markets.extend(markets) else: # If no category is specified or "all" is specified, get data from multiple categories categories_to_fetch = ["crypto", "politics", "economics", "sports"] for cat in categories_to_fetch: markets = self._fetch_markets_from_api(cat, limit // len(categories_to_fetch) + 10) all_markets.extend(markets) # Deduplication (by market_id) seen = set() unique_markets = [] for market in all_markets: market_id = market.get("market_id") if market_id and market_id not in seen: seen.add(market_id) unique_markets.append(market) # Sort by transaction volume unique_markets.sort(key=lambda x: x.get('volume_24h', 0), reverse=True) # Save to database cache if unique_markets: self._save_markets_to_db(unique_markets) return unique_markets[:limit] # If the API fails, an empty list is returned (sample data is no longer used) logger.warning("Polymarket API unavailable, returning empty list") return [] except Exception as e: logger.error(f"Failed to get trending markets: {e}", exc_info=True) return [] def get_market_details(self, market_id: str) -> Optional[Dict]: """Get individual market details""" try: # Make sure market_id is a string market_id = str(market_id).strip() if not market_id: logger.warning("Empty market_id provided") return None # Read from database first try: with get_db_connection() as db: cur = db.cursor() cur.execute(""" SELECT market_id, question, category, current_probability, volume_24h, liquidity, end_date_iso, status, outcome_tokens FROM qd_polymarket_markets WHERE market_id = %s """, (market_id,)) row = cur.fetchone() cur.close() if row: #RealDictCursor returns the dictionary, accessed using keys db_market_id = str(row.get('market_id') or market_id) # Parse outcome_tokens (may be a JSON string) outcome_tokens = {} outcome_tokens_raw = row.get('outcome_tokens') if outcome_tokens_raw: try: if isinstance(outcome_tokens_raw, str): outcome_tokens = json.loads(outcome_tokens_raw) else: outcome_tokens = outcome_tokens_raw if isinstance(outcome_tokens_raw, dict) else {} except: outcome_tokens = {} return { "market_id": db_market_id, "question": row.get('question') or '', "category": row.get('category') or 'other', "current_probability": float(row.get('current_probability') or 0), "volume_24h": float(row.get('volume_24h') or 0), "liquidity": float(row.get('liquidity') or 0), "end_date_iso": row.get('end_date_iso'), "status": row.get('status') or 'active', "outcome_tokens": outcome_tokens, "polymarket_url": self._build_polymarket_url(row.get('slug'), db_market_id), "slug": row.get('slug') if row.get('slug') and not str(row.get('slug', '')).isdigit() else None } except Exception as db_error: logger.warning(f"Database query failed for market {market_id}: {db_error}") # Continue trying to get it from the API # If the database does not exist, get it from the API logger.info(f"Market {market_id} not in database, fetching from API") market = self._fetch_market_from_api(market_id) if market: try: self._save_markets_to_db([market]) except Exception as save_error: logger.warning(f"Failed to save market to DB: {save_error}") return market logger.warning(f"Market {market_id} not found in API") return None except Exception as e: logger.error(f"Failed to get market details for {market_id}: {e}", exc_info=True) return None def get_market_history(self, market_id: str, days: int = 30) -> List[Dict]: """Get historical market price data.""" # Here you need to implement historical data acquisition logic # Temporarily returns an empty list return [] def search_markets(self, keyword: str, limit: int = 20, use_cache: bool = True) -> List[Dict]: """ Search related prediction markets Priority is given to obtaining real-time data from the API, and the database is only used as an optional cache. Args: keyword: search keyword limit: limit on the number of returned results use_cache: whether to use database cache (should be set to False during AI analysis to obtain the latest data) """ try: logger.info(f"Searching Polymarket markets for keyword: '{keyword}' (limit={limit}, use_cache={use_cache})") # If caching is allowed, try searching from the database first if use_cache: with get_db_connection() as db: cur = db.cursor() # Improved search: search question and slug fields at the same time, also support market_id exact matching keyword_lower = keyword.lower() is_numeric = keyword_lower.isdigit() has_hyphens = '-' in keyword_lower if is_numeric: # If it is a pure number, it may be market_id, an exact match cur.execute(""" SELECT market_id, question, category, current_probability, volume_24h, liquidity, end_date_iso, status, slug FROM qd_polymarket_markets WHERE market_id = %s AND status = 'active' ORDER BY volume_24h DESC LIMIT %s """, (keyword, limit)) elif has_hyphens: # If it contains a hyphen, it may be a slug, and the slug will be matched first. cur.execute(""" SELECT market_id, question, category, current_probability, volume_24h, liquidity, end_date_iso, status, slug FROM qd_polymarket_markets WHERE (slug ILIKE %s OR question ILIKE %s) AND status = 'active' ORDER BY CASE WHEN slug ILIKE %s THEN 1 ELSE 2 END, volume_24h DESC LIMIT %s """, (f"%{keyword}%", f"%{keyword}%", f"%{keyword}%", limit)) else: # Normal text search cur.execute(""" SELECT market_id, question, category, current_probability, volume_24h, liquidity, end_date_iso, status, slug FROM qd_polymarket_markets WHERE (question ILIKE %s OR slug ILIKE %s) AND status = 'active' ORDER BY volume_24h DESC LIMIT %s """, (f"%{keyword}%", f"%{keyword}%", limit)) rows = cur.fetchall() cur.close() if rows: logger.info(f"Found {len(rows)} markets in database for keyword '{keyword}'") return [{ "market_id": str(row.get('market_id') or ''), "question": row.get('question') or '', "category": row.get('category') or 'other', "current_probability": float(row.get('current_probability') or 0), "volume_24h": float(row.get('volume_24h') or 0), "liquidity": float(row.get('liquidity') or 0), "end_date_iso": row.get('end_date_iso'), "status": row.get('status') or 'active', "polymarket_url": self._build_polymarket_url(row.get('slug'), row.get('market_id') or ''), "slug": row.get('slug') if row.get('slug') and not str(row.get('slug', '')).isdigit() else None } for row in rows] # Obtain and filter directly from Gamma API (used during AI analysis) logger.info(f"Fetching from API for keyword '{keyword}' (use_cache={use_cache})...") # Optimization: If the keyword looks like a slug, try direct query first (avoid fetching the full amount) import re keyword_lower = keyword.lower().strip() is_slug_like = '-' in keyword_lower and not keyword_lower.isdigit() if is_slug_like: # Try to query directly through slug (most efficient, according to Polymarket API documentation) direct_market = self._fetch_market_by_slug(keyword_lower) if direct_market: logger.info(f"Found market directly by slug (no need to fetch all markets): {keyword_lower}") return [direct_market] # If direct query fails, get more data so that there is enough room for selection # Make multiple requests to get more markets (up to 100 events each time, but each event may contain multiple markets) all_markets = [] max_requests = 3 # Request up to 3 times to get 300 events (about 4500 markets) for page in range(max_requests): page_markets = self._fetch_from_gamma_api(category=None, limit=100) if not page_markets: break all_markets.extend(page_markets) # If enough markets have been acquired, you can stop early if len(all_markets) >= 3000: # Get up to 3000 markets break logger.info(f"Fetched page {page + 1}/{max_requests}, total markets: {len(all_markets)}") # Short delay to avoid API current limit if page < max_requests - 1: time.sleep(0.5) logger.info(f"Fetched {len(all_markets)} markets from API, filtering for keyword '{keyword}'...") # Filter by keyword (supports multiple keyword matching) # If the keyword looks like a slug (contains a hyphen), also try to match the slug keyword_is_slug = '-' in keyword_lower # Extract keywords (remove common stop words and punctuation) # Extract keywords: remove punctuation, retain alphanumeric characters and hyphens keyword_words = re.findall(r'\b\w+\b', keyword_lower) # Filter out words that are too short (less than 3 characters) and common stop words stop_words = {'the', 'a', 'an', 'and', 'or', 'but', 'in', 'on', 'at', 'to', 'for', 'of', 'with', 'by', 'is', 'are', 'was', 'were', 'be', 'been', 'will', 'would', 'should', 'could', 'may', 'might', 'can', 'must'} keyword_words = [w for w in keyword_words if len(w) >= 3 and w not in stop_words] # If no keywords are extracted, use the original keywords if not keyword_words: keyword_words = [keyword_lower] logger.info(f"Extracted keywords: {keyword_words} from '{keyword}'") filtered = [] scored_markets = [] # Used to store scored results top_candidates = [] # Used to store close matching candidates (for debugging) for market in all_markets: question = market.get("question", "").lower() slug = (market.get("slug") or "").lower() market_id = str(market.get("market_id") or "") score = 0 match_reason = "" # 1. Exact match (highest priority, score 100) if keyword_lower in question: score = 100 match_reason = "exact_match_question" elif keyword_lower == slug: score = 100 match_reason = "exact_match_slug" # 2. If the keyword looks like a slug, check the slug field if score < 100 and keyword_is_slug: if keyword_lower == slug: score = 100 match_reason = "exact_slug_match" elif keyword_lower in slug or slug in keyword_lower: score = 90 match_reason = "partial_slug_match" # 3. If the keyword is a pure number, check market_id if score < 90 and keyword_lower.isdigit(): if keyword_lower == market_id: score = 100 match_reason = "market_id_match" # 4. Keyword matching: Check whether all keywords are in the question if score < 90 and keyword_words: # Calculate the number of matching keywords matched_words = sum(1 for word in keyword_words if word in question or word in slug) if matched_words > 0: # Match rate match_ratio = matched_words / len(keyword_words) # Lower the threshold: from 60% to 40% to improve the matching rate if match_ratio >= 0.4: score = int(60 + match_ratio * 30) # 60-90 minutes match_reason = f"keyword_match_{matched_words}/{len(keyword_words)}" else: # Log close matching candidates (for debugging) if matched_words >= 1 and len(top_candidates) < 5: top_candidates.append((match_ratio, market.get('question', '')[:80], matched_words, len(keyword_words))) # 5. Partial matching: Check whether the main part of the keyword is in the question if score < 60 and keyword_words: # If the keyword contains multiple words, try to match the main part if len(keyword_words) > 1: # Take the first 3 most important words (usually nouns) important_words = keyword_words[:3] matched_important = sum(1 for word in important_words if word in question or word in slug) # Lower the requirement: match at least 1 important word if matched_important >= 1: score = 50 match_reason = f"important_words_match_{matched_important}/{len(important_words)}" if score >= 50: # Lower the minimum score requirement from 60 to 50 scored_markets.append((score, market, match_reason)) logger.debug(f"Matched (score={score}, reason={match_reason}): {market.get('question', '')[:60]}") # Sort by score, take the first limit scored_markets.sort(key=lambda x: x[0], reverse=True) filtered = [market for score, market, reason in scored_markets[:limit]] # Output debugging information if len(scored_markets) == 0 and top_candidates: logger.warning(f"No exact matches found. Top candidates (partial matches):") for ratio, question, matched, total in top_candidates: logger.warning(f" - {question} (matched {matched}/{total} keywords, ratio={ratio:.2f})") logger.info(f"Filtered {len(filtered)} markets matching keyword '{keyword}' from API (from {len(all_markets)} total markets, {len(scored_markets)} scored matches)") if len(scored_markets) > 0: logger.info(f"Top match: {filtered[0].get('question', '')[:80]} (score={scored_markets[0][0]})") return filtered except Exception as e: logger.error(f"Failed to search markets: {e}", exc_info=True) return [] def _get_cached_markets(self, category: str = None, limit: int = 50) -> Optional[List[Dict]]: """Read market data from the database cache.""" try: with get_db_connection() as db: cur = db.cursor() # Check cache is fresh (within 5 minutes) cutoff_time = datetime.now() - timedelta(seconds=self.cache_ttl) query = """ SELECT market_id, question, category, current_probability, volume_24h, liquidity, end_date_iso, status, outcome_tokens FROM qd_polymarket_markets WHERE status = 'active' AND updated_at > %s """ params = [cutoff_time] if category: query += " AND category = %s" params.append(category) query += " ORDER BY volume_24h DESC LIMIT %s" params.append(limit) cur.execute(query, params) rows = cur.fetchall() cur.close() if rows: result = [] for row in rows: market_id = str(row.get('market_id') or '') slug = row.get('slug') # Make sure to use the correct URL building method polymarket_url = self._build_polymarket_url(slug, market_id) result.append({ "market_id": market_id, "question": row.get('question') or '', "category": row.get('category') or 'other', "current_probability": float(row.get('current_probability') or 0), "volume_24h": float(row.get('volume_24h') or 0), "liquidity": float(row.get('liquidity') or 0), "end_date_iso": row.get('end_date_iso'), "status": row.get('status') or 'active', "outcome_tokens": row.get('outcome_tokens') if row.get('outcome_tokens') else {}, "polymarket_url": polymarket_url, "slug": slug if slug and not str(slug).isdigit() else None }) return result return None except Exception as e: logger.debug(f"Failed to get cached markets: {e}") return None def _fetch_markets_from_api(self, category: str = None, limit: int = 50) -> List[Dict]: """ Retrieve market data from the Polymarket Gamma API Use the officially recommended /events endpoint """ try: # Use the /events endpoint of Gamma API (official recommended method) markets = self._fetch_from_gamma_api(category, limit) if markets: # Sort by volume_24h in descending order (because the API does not support the order parameter, local sorting is required) markets.sort(key=lambda x: x.get('volume_24h', 0), reverse=True) return markets[:limit] # Return the first limit after sorting # If the API returns an empty list, record a warning (it may be that the API is temporarily unavailable, network problems or throttling) logger.warning(f"Gamma API failed to fetch markets for category '{category}' (possible reasons: API is temporarily unavailable, network problems, current limiting or returning empty data)") return [] except Exception as e: logger.error(f"Failed to fetch markets from API: {e}", exc_info=True) return [] def _fetch_from_gamma_api(self, category: str = None, limit: int = 50) -> List[Dict]: """ Retrieve market data from the Polymarket Gamma API Use the officially recommended /events endpoint """ try: # Use the /events endpoint to obtain active markets (recommended method) # According to official documentation: https://docs.polymarket.com/market-data/fetching-markets # Supported values ​​for the order parameter: volume_24hr, volume, liquidity, competitive, start_date, end_date # But some endpoints may not support it, try without the order parameter first. url = f"{self.gamma_api}/events" params = { "active": "true", "closed": "false", "limit": min(limit * 2, 100) # Get more data for sorting and filtering } # Try adding sorting parameters (if supported by API) # According to the documentation, possible sorting fields: volume_24hr, volume, liquidity, etc. # If the API is not supported, it will be removed after a 422 error. # If a category is specified, it needs to be filtered by tag_id # Note: You need to get the tag_id first, and use keywords to infer it here. if category: # You can try filtering by search or tags # Get them all temporarily, then filter when parsing pass logger.info(f"Fetching from Gamma API: {url} with params: {params}") response = self.session.get(url, params=params, timeout=15) logger.info(f"Gamma API response status: {response.status_code}") if response.status_code == 200: try: data = response.json() logger.debug(f"Gamma API returned data type: {type(data)}, keys: {list(data.keys()) if isinstance(data, dict) else 'list'}") #The Gamma API may return a list or an object containing a data field if isinstance(data, list): logger.info(f"Gamma API returned list with {len(data)} items") markets = self._parse_gamma_events(data, category) logger.info(f"Parsed {len(markets)} markets from Gamma API") return markets elif isinstance(data, dict): # Possibly {"data": [...]} format if "data" in data: events_list = data["data"] logger.info(f"Gamma API returned dict with 'data' field containing {len(events_list) if isinstance(events_list, list) else 'non-list'} items") markets = self._parse_gamma_events(events_list, category) logger.info(f"Parsed {len(markets)} markets from Gamma API") return markets # Or directly the event object elif "id" in data or "slug" in data: logger.info("Gamma API returned single event object") markets = self._parse_gamma_events([data], category) logger.info(f"Parsed {len(markets)} markets from Gamma API") return markets else: logger.warning(f"Gamma API returned dict with unexpected keys: {list(data.keys())}") logger.debug(f"Full response: {str(data)[:500]}") logger.warning(f"Gamma API returned unexpected format: {type(data)}") return [] except json.JSONDecodeError as je: logger.error(f"Gamma API returned invalid JSON: {je}") logger.error(f"Response text (first 500 chars): {response.text[:500]}") return [] # Non-200 status code status_code = response.status_code if status_code == 429: logger.warning(f"Gamma API rate limited (429). Suggestion: Try again later or reduce the request frequency") elif status_code == 503: logger.warning(f"Gamma API service unavailable (503). Polymarket API may be under maintenance") elif status_code >= 500: logger.warning(f"Gamma API server error ({status_code}). The Polymarket server may be temporarily unavailable") else: logger.warning(f"Gamma API returned status {status_code}") logger.debug(f"Response headers: {dict(response.headers)}") logger.debug(f"Response text (first 500 chars): {response.text[:500]}") return [] except requests.exceptions.Timeout: logger.warning("Gamma API request timeout after 15 seconds (possible reason: network delay or slow API response)") return [] except requests.exceptions.ConnectionError as ce: logger.warning(f"Gamma API connection error: {ce} (Possible reason: network connection problem or Polymarket API is unreachable)") return [] except Exception as e: logger.warning(f"Gamma API failed: {e} (Possible reasons: API format change, network problem or service abnormality)") return [] def _parse_gamma_events(self, events_data: List[Dict], category_filter: str = None) -> List[Dict]: """ Parse event data returned by the Gamma API The /events endpoint of the Gamma API returns event objects, each containing associated market data According to the official documentation, the event object structure is: - The event object contains a markets array - Each market contains fields such as clobTokenIds and outcomePrices """ parsed = [] if not events_data: logger.warning("_parse_gamma_events received empty events_data") return parsed logger.info(f"Parsing {len(events_data)} events from Gamma API") # Record the key of the first event for debugging if events_data: first_event_keys = list(events_data[0].keys())[:10] logger.info(f"First event keys: {first_event_keys}") logger.debug(f"First event sample: {str(events_data[0])[:500]}") for idx, event in enumerate(events_data): try: #Gamma API event object structure # The event may have multiple markets (markets field), or directly contain market information markets = event.get("markets", []) # If the event does not have a markets field, the event itself may be market data. if not markets: # Check whether it is a market object directly (with question or title field) if "question" in event or "title" in event or "slug" in event: markets = [event] else: if idx < 3: # Only record the details of the first 3 logger.debug(f"Event {idx} has no markets and doesn't look like a market. Keys: {list(event.keys())[:10]}") continue if idx < 3: # Only record the details of the first 3 logger.debug(f"Processing event {idx} with {len(markets)} markets") for market_idx, market in enumerate(markets): # Extract basic market information market_id = market.get("id") or market.get("slug") or event.get("id") or event.get("slug", "") question = market.get("question") or event.get("question") or market.get("title") or event.get("title", "") if idx < 3 and market_idx < 2: # Record detailed information of the first few markets logger.info(f"Event {idx}, Market {market_idx}: id={market_id}, question={question[:50] if question else 'None'}, event_slug={event.get('slug')}, market_slug={market.get('slug')}, keys={list(market.keys())[:10]}") if not question: if idx < 3: logger.warning(f"Event {idx}, Market {market_idx}: No question found, skipping. Market keys: {list(market.keys())[:10]}") continue # Infer category inferred_category = self._infer_category(question) # If category filtering is specified, filter if category_filter and inferred_category != category_filter: continue # Get probability and outcome data current_probability = 50.0 outcome_tokens = {} # Method 1: Get real-time prices from CLOB API (most accurate) try: condition_id = market.get("conditionId") or event.get("conditionId") if condition_id: prices = self._get_market_prices_from_clob(condition_id) if prices: yes_price = prices.get("YES", 0) no_price = prices.get("NO", 0) if yes_price > 0: current_probability = yes_price * 100 if yes_price <= 1 else yes_price outcome_tokens["YES"] = {"price": yes_price if yes_price <= 1 else yes_price / 100, "volume": 0} if no_price > 0: outcome_tokens["NO"] = {"price": no_price if no_price <= 1 else no_price / 100, "volume": 0} except Exception as e: logger.debug(f"Failed to get prices from CLOB API: {e}") # Method 2: Process the outcomePrices field (may be a JSON string) if current_probability == 50.0: outcome_prices_str = market.get("outcomePrices") or event.get("outcomePrices") if outcome_prices_str: try: if isinstance(outcome_prices_str, str): outcome_prices = json.loads(outcome_prices_str) else: outcome_prices = outcome_prices_str # outcomePrices is usually in the format ["0.65", "0.35"], corresponding to YES and NO if isinstance(outcome_prices, list) and len(outcome_prices) >= 2: yes_price = float(outcome_prices[0]) if outcome_prices[0] else 0 no_price = float(outcome_prices[1]) if outcome_prices[1] else 0 current_probability = yes_price * 100 if yes_price <= 1 else yes_price outcome_tokens["YES"] = {"price": yes_price if yes_price <= 1 else yes_price / 100, "volume": 0} outcome_tokens["NO"] = {"price": no_price if no_price <= 1 else no_price / 100, "volume": 0} except Exception as e: logger.debug(f"Failed to parse outcomePrices: {e}") # Get outcomes from market or event #outcomes may be an object array, a string array, or need to be parsed from other fields outcomes = market.get("outcomes") or market.get("tokens") or event.get("outcomes") or [] # Process the outcomes array (may be an object or a string) for outcome in outcomes: try: # If outcome is a string, skip or try to parse if isinstance(outcome, str): # May be a simple string identifier such as "YES" or "NO" outcome_upper = outcome.upper() if "YES" in outcome_upper: if "YES" not in outcome_tokens: outcome_tokens["YES"] = {"price": 0.5, "volume": 0} elif "NO" in outcome_upper: if "NO" not in outcome_tokens: outcome_tokens["NO"] = {"price": 0.5, "volume": 0} continue # outcome is an object if not isinstance(outcome, dict): continue title = str(outcome.get("title") or outcome.get("name", "")).upper() # Get the price (may be price, probability or currentPrice) price = float(outcome.get("price") or outcome.get("probability") or outcome.get("currentPrice") or 0) if "YES" in title or title == "YES" or outcome.get("outcome") == "Yes": current_probability = price * 100 if price <= 1 else price outcome_tokens["YES"] = { "price": price if price <= 1 else price / 100, "volume": float(outcome.get("volume", outcome.get("volume24hr", 0)) or 0) } elif "NO" in title or title == "NO" or outcome.get("outcome") == "No": outcome_tokens["NO"] = { "price": price if price <= 1 else price / 100, "volume": float(outcome.get("volume", outcome.get("volume24hr", 0)) or 0) } except Exception as e: logger.debug(f"Failed to parse outcome: {e}") continue # If outcomes are not found, try to get probabilities from other fields if current_probability == 50.0: # Try to get it from the probability field of the market prob = market.get("probability") or market.get("yesProbability") or event.get("probability") if prob: current_probability = float(prob) * 100 if float(prob) <= 1 else float(prob) # Get trading volume and liquidity volume_24h = float( market.get("volume_24hr") or market.get("volume24hr") or market.get("volume_24h") or event.get("volume_24hr") or event.get("volume24hr") or 0 ) liquidity = float( market.get("liquidity") or market.get("totalLiquidity") or event.get("liquidity") or 0 ) # Parse end date end_date_iso = None end_date = market.get("endDate") or market.get("end_date") or event.get("endDate") or event.get("end_date") if end_date: try: if isinstance(end_date, (int, float)): end_date_iso = datetime.fromtimestamp(end_date).isoformat() + "Z" elif isinstance(end_date, str): # Try to parse the ISO format string end_date_iso = end_date except: pass # Get the slug used to build the URL # According to Polymarket API documentation: slug should be obtained directly from the data returned by the API # URL format: https://polymarket.com/event/{slug} # slug is a string identifier, not a numeric ID slug = None # Get the slug from the event first (because the event contains markets) if event.get("slug"): slug_str = str(event.get("slug", "")).strip() # If the slug is not a pure number and contains letters or hyphens, it is a valid slug if slug_str and not slug_str.isdigit() and ('-' in slug_str or any(c.isalpha() for c in slug_str)): slug = slug_str # If the event does not have a valid slug, try to get it from the market if not slug and market.get("slug"): slug_str = str(market.get("slug", "")).strip() if slug_str and not slug_str.isdigit() and ('-' in slug_str or any(c.isalpha() for c in slug_str)): slug = slug_str # If there is still no valid slug, try to obtain it through API query if not slug and market_id: try: # Use the markets endpoint to query by ID to obtain complete slug information detail_market = self._fetch_market_detail_by_id(market_id) if detail_market and detail_market.get("slug"): slug_str = str(detail_market.get("slug", "")).strip() if slug_str and not slug_str.isdigit() and ('-' in slug_str or any(c.isalpha() for c in slug_str)): slug = slug_str except Exception as e: logger.debug(f"Failed to fetch slug for market {market_id}: {e}") # Build URL (using unified helper methods) polymarket_url = self._build_polymarket_url(slug, market_id) if not slug: logger.warning(f"Market {market_id} has no valid slug, using markets endpoint as fallback") market_data = { "market_id": market_id, "question": question, "category": inferred_category, "current_probability": round(current_probability, 2), "volume_24h": volume_24h, "liquidity": liquidity, "end_date_iso": end_date_iso, "status": "active" if market.get("active", event.get("active", True)) else "closed", "outcome_tokens": outcome_tokens, "polymarket_url": polymarket_url, "slug": slug if slug else None # Save slug (if not a number) } parsed.append(market_data) if idx < 3 and market_idx < 2: # Record successfully parsed markets logger.info(f"Successfully parsed market: {question[:50]}, prob={current_probability:.1f}%, volume={volume_24h}") except Exception as e: logger.warning(f"Failed to parse event {idx} (id={event.get('id', event.get('slug', 'unknown'))}): {e}", exc_info=True) continue logger.info(f"Successfully parsed {len(parsed)} markets from {len(events_data)} events") return parsed def _parse_rest_markets(self, markets_data: List[Dict]) -> List[Dict]: """Parse REST API data""" parsed = [] for market in markets_data: try: # Extract basic information market_id = market.get("id") or market.get("slug") or market.get("market_id", "") question = market.get("question") or market.get("title", "") # Calculate probability current_probability = 50.0 outcome_tokens = {} if "outcomes" in market: for outcome in market["outcomes"]: title = str(outcome.get("title", "")).upper() price = float(outcome.get("price", outcome.get("probability", 0)) or 0) if "YES" in title or title == "YES": current_probability = price * 100 outcome_tokens["YES"] = { "price": price, "volume": float(outcome.get("volume", 0) or 0) } elif "NO" in title or title == "NO": outcome_tokens["NO"] = { "price": price, "volume": float(outcome.get("volume", 0) or 0) } volume_24h = float(market.get("volume_24h", market.get("volume", 0)) or 0) liquidity = float(market.get("liquidity", 0) or 0) # inferred category category = self._infer_category(question) # parse end date end_date_iso = market.get("end_date") or market.get("endDate") if isinstance(end_date_iso, (int, float)): try: end_date_iso = datetime.fromtimestamp(end_date_iso).isoformat() + "Z" except: end_date_iso = None # Get the slug used to build the URL slug = None slug_str = str(market.get('slug', '')).strip() if market.get('slug') else '' # Check if the slug is valid (not a number and contains letters or hyphens) if slug_str and not slug_str.isdigit() and ('-' in slug_str or any(c.isalpha() for c in slug_str)): slug = slug_str else: # If the slug is invalid, try to obtain it through API query try: detail_market = self._fetch_market_detail_by_id(market_id) if detail_market and detail_market.get("slug"): slug_str = str(detail_market.get("slug", "")).strip() if slug_str and not slug_str.isdigit() and ('-' in slug_str or any(c.isalpha() for c in slug_str)): slug = slug_str except Exception as e: logger.debug(f"Failed to fetch slug for market {market_id}: {e}") # Build URL (using unity helper methods) polymarket_url = self._build_polymarket_url(slug, market_id) if not slug: logger.warning(f"Market {market_id} has no valid slug, using markets endpoint as fallback") parsed.append({ "market_id": market_id, "question": question, "category": category, "current_probability": round(current_probability, 2), "volume_24h": volume_24h, "liquidity": liquidity, "end_date_iso": end_date_iso, "status": "active" if market.get("active", True) else "closed", "outcome_tokens": outcome_tokens, "polymarket_url": polymarket_url, "slug": slug if slug else None }) except Exception as e: logger.debug(f"Failed to parse market {market.get('id')}: {e}") continue return parsed def _infer_category(self, question: str) -> str: """Infer categories from questions""" question_lower = question.lower() # Cryptocurrency Keywords crypto_keywords = ['btc', 'bitcoin', 'eth', 'ethereum', 'sol', 'solana', 'crypto', 'token', 'coin', 'defi', 'nft'] if any(kw in question_lower for kw in crypto_keywords): return "crypto" # political keywords politics_keywords = ['election', 'president', 'trump', 'biden', 'senate', 'congress', 'vote', 'political', 'democrat', 'republican'] if any(kw in question_lower for kw in politics_keywords): return "politics" # economic keywords economics_keywords = ['gdp', 'inflation', 'unemployment', 'fed', 'federal reserve', 'interest rate', 'economic', 'economy', 'recession', 'gdp growth', 'cpi', 'ppi'] if any(kw in question_lower for kw in economics_keywords): return "economics" # Sports keywords sports_keywords = ['nfl', 'nba', 'mlb', 'soccer', 'football', 'basketball', 'baseball', 'championship', 'world cup', 'olympics', 'super bowl', 'stanley cup', 'world series'] if any(kw in question_lower for kw in sports_keywords): return "sports" # Technology keywords tech_keywords = ['ai', 'artificial intelligence', 'chatgpt', 'openai', 'tech', 'technology', 'apple', 'google', 'microsoft', 'meta', 'tesla', 'ipo', 'startup'] if any(kw in question_lower for kw in tech_keywords): return "tech" # financial keywords finance_keywords = ['stock', 's&p', 'dow', 'nasdaq', 'market cap', 'earnings', 'revenue', 'profit', 'bank', 'banking', 'financial', 'trading'] if any(kw in question_lower for kw in finance_keywords): return "finance" # geopolitical keywords geopolitics_keywords = ['war', 'conflict', 'russia', 'ukraine', 'china', 'taiwan', 'north korea', 'iran', 'israel', 'palestine', 'middle east', 'nato', 'sanctions'] if any(kw in question_lower for kw in geopolitics_keywords): return "geopolitics" # Cultural keywords culture_keywords = ['movie', 'film', 'oscar', 'grammy', 'award', 'celebrity', 'music', 'album', 'tv show', 'series', 'netflix', 'disney'] if any(kw in question_lower for kw in culture_keywords): return "culture" # climate keywords climate_keywords = ['climate', 'global warming', 'temperature', 'carbon', 'emission', 'renewable', 'solar', 'wind energy', 'paris agreement', 'cop'] if any(kw in question_lower for kw in climate_keywords): return "climate" # Entertainment keywords entertainment_keywords = ['game', 'gaming', 'esports', 'tournament', 'streaming', 'youtube', 'twitch', 'podcast', 'comic', 'anime', 'manga'] if any(kw in question_lower for kw in entertainment_keywords): return "entertainment" return "other" def _build_polymarket_url(self, slug: Optional[str], market_id: str) -> str: """ Build Polymarket URL based on slug Reference: https://docs.polymarket.com/market-data/fetching-markets Args: slug: slug obtained from API or database (may be None or numeric string) market_id: Market ID (as an alternative) Returns: Polymarket URL string """ if slug: slug_str = str(slug).strip() # Check if the slug is valid (not a number and contains letters or hyphens) if slug_str and not slug_str.isdigit() and ('-' in slug_str or any(c.isalpha() for c in slug_str)): import re slug_clean = re.sub(r'[^a-zA-Z0-9\-]', '-', slug_str) slug_clean = slug_clean.strip('-') if slug_clean: return f"https://polymarket.com/event/{slug_clean}" # If there is no valid slug, try to obtain the slug through the API if market_id: try: detail_market = self._fetch_market_detail_by_id(market_id) if detail_market: # Try to get the slug from detail event_slug = detail_market.get('slug') if event_slug: slug_str = str(event_slug).strip() if slug_str and not slug_str.isdigit() and ('-' in slug_str or any(c.isalpha() for c in slug_str)): import re slug_clean = re.sub(r'[^a-zA-Z0-9\-]', '-', slug_str) slug_clean = slug_clean.strip('-') if slug_clean: return f"https://polymarket.com/event/{slug_clean}" # If the event does not have a slug, try to get it from markets markets = detail_market.get('markets', []) if markets: for m in markets: market_slug = m.get('slug') if market_slug: slug_str = str(market_slug).strip() if slug_str and not slug_str.isdigit() and ('-' in slug_str or any(c.isalpha() for c in slug_str)): import re slug_clean = re.sub(r'[^a-zA-Z0-9\-]', '-', slug_str) slug_clean = slug_clean.strip('-') if slug_clean: return f"https://polymarket.com/event/{slug_clean}" except Exception as e: logger.debug(f"Failed to fetch slug for market {market_id}: {e}") # If all else fails, return to the search page (more reliable) # Note: Polymarket's URL format may have changed, use search as fallback return f"https://polymarket.com/search?q={market_id}" def _fetch_market_detail_by_id(self, market_id: str) -> Optional[Dict]: """ Get market details from API by market ID (used to get slug) Reference: https://docs.polymarket.com/market-data/fetching-markets """ try: # Method 1: Try querying through the events endpoint (recommended because events include markets) url = f"{self.gamma_api}/events" params = {"active": "true", "closed": "false", "limit": 100} response = self.session.get(url, params=params, timeout=10) if response.status_code == 200: events = response.json() if isinstance(events, list): for event in events: markets = event.get("markets", []) if not markets and ("question" in event or "slug" in event): markets = [event] for market in markets: m_id = market.get("id") or market.get("slug") or "" e_id = event.get("id") or event.get("slug") or "" # Match market_id or event_id if str(m_id) == str(market_id) or str(e_id) == str(market_id): # Return event (because event contains slug) return event elif isinstance(events, dict): if "data" in events: events_list = events["data"] for event in events_list: markets = event.get("markets", []) if not markets and ("question" in event or "slug" in event): markets = [event] for market in markets: m_id = market.get("id") or market.get("slug") or "" e_id = event.get("id") or event.get("slug") or "" if str(m_id) == str(market_id) or str(e_id) == str(market_id): return event # Method 2: Try querying through the markets endpoint url = f"{self.gamma_api}/markets" params = {"id": market_id, "limit": 1} response = self.session.get(url, params=params, timeout=10) if response.status_code == 200: data = response.json() if isinstance(data, list) and len(data) > 0: return data[0] elif isinstance(data, dict) and "id" in data: return data return None except Exception as e: logger.debug(f"Failed to fetch market detail by ID {market_id}: {e}") return None def _fetch_market_by_slug(self, slug: str) -> Optional[Dict]: """ Query the market directly through slug (the most efficient way) According to Polymarket API documentation: https://docs.polymarket.com/market-data/fetching-markets You can use /markets?slug=xxx to query directly """ try: # Method 1: Try querying the slug directly through the markets endpoint url = f"{self.gamma_api}/markets" params = {"slug": slug, "limit": 10} logger.info(f"Fetching market by slug from Gamma API: {url} with params: {params}") response = self.session.get(url, params=params, timeout=10) if response.status_code == 200: data = response.json() if isinstance(data, list) and len(data) > 0: # Parse the returned market data markets = self._parse_gamma_events(data) # Exact match slug for market in markets: market_slug = market.get("slug", "").lower() if market_slug == slug.lower() or slug.lower() in market_slug: logger.info(f"Found market by slug: {slug}") return market # If there is no exact match, return the first if markets: logger.info(f"Found market by slug (fuzzy match): {slug}") return markets[0] elif isinstance(data, dict): # single market object markets = self._parse_gamma_events([data]) if markets: logger.info(f"Found market by slug: {slug}") return markets[0] # Method 2: Try to query through the events endpoint (events may contain slug information) url = f"{self.gamma_api}/events" params = {"active": "true", "closed": "false", "limit": 100} response = self.session.get(url, params=params, timeout=10) if response.status_code == 200: data = response.json() events = data if isinstance(data, list) else (data.get("data", []) if isinstance(data, dict) else []) # Find the matching slug in the returned event for event in events: event_slug = (event.get("slug") or "").lower() if event_slug == slug.lower() or slug.lower() in event_slug: parsed = self._parse_gamma_events([event]) if parsed: logger.info(f"Found market by slug via events: {slug}") return parsed[0] logger.warning(f"Market with slug '{slug}' not found via direct query") return None except Exception as e: logger.error(f"Failed to fetch market by slug {slug}: {e}", exc_info=True) return None def _fetch_market_from_api(self, market_id: str) -> Optional[Dict]: """ Get individual market data from Gamma API Support query by slug or id """ try: # Determine whether it is slug or market_id is_slug = not market_id.isdigit() and ('-' in market_id or any(c.isalpha() for c in market_id)) # If it is a slug, the direct query method is preferred. if is_slug: market = self._fetch_market_by_slug(market_id) if market: return market # Method 1: Query through markets endpoint (supports id and slug) url = f"{self.gamma_api}/markets" params = {"id": market_id, "limit": 10} if not is_slug else {"slug": market_id, "limit": 10} response = self.session.get(url, params=params, timeout=10) if response.status_code == 200: data = response.json() if isinstance(data, list) and len(data) > 0: markets = self._parse_gamma_events(data) if markets: return markets[0] elif isinstance(data, dict): markets = self._parse_gamma_events([data]) if markets: return markets[0] # Method 2: Search via events endpoint (as an alternative) url = f"{self.gamma_api}/events" params = { "active": "true", "closed": "false", "limit": 100 } response = self.session.get(url, params=params, timeout=10) if response.status_code == 200: data = response.json() events = data if isinstance(data, list) else (data.get("data", []) if isinstance(data, dict) else []) # Find matching markets in returned events for event in events: markets = event.get("markets", []) if not markets: markets = [event] for market in markets: m_id = market.get("id") or market.get("slug") or event.get("id") or event.get("slug", "") if str(m_id) == str(market_id) or market.get("slug") == market_id: parsed = self._parse_gamma_events([event]) if parsed: return parsed[0] return None except Exception as e: logger.error(f"Failed to fetch market {market_id}: {e}", exc_info=True) return None def _save_markets_to_db(self, markets: List[Dict]): """Save market data to database""" try: with get_db_connection() as db: cur = db.cursor() for market in markets: # Get the slug, but don't use it if it's a number (numbers are not valid slugs) slug = market.get('slug') or None # If the slug is a number, it means it is not a valid slug and is set to None. if slug and str(slug).isdigit(): slug = None # Clean slug, keep only alphanumerics and hyphens import re if slug: slug = re.sub(r'[^a-zA-Z0-9\-]', '-', str(slug)) slug = slug.strip('-') # If it is empty or still a number after cleaning, set to None if not slug or slug.isdigit(): slug = None cur.execute(""" INSERT INTO qd_polymarket_markets (market_id, question, category, current_probability, volume_24h, liquidity, end_date_iso, status, outcome_tokens, slug, updated_at) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, NOW()) ON CONFLICT (market_id) DO UPDATE SET question = EXCLUDED.question, category = EXCLUDED.category, current_probability = EXCLUDED.current_probability, volume_24h = EXCLUDED.volume_24h, liquidity = EXCLUDED.liquidity, end_date_iso = EXCLUDED.end_date_iso, status = EXCLUDED.status, outcome_tokens = EXCLUDED.outcome_tokens, slug = EXCLUDED.slug, updated_at = NOW() """, ( market.get('market_id'), market.get('question'), market.get('category', 'other'), market.get('current_probability', 50.0), market.get('volume_24h', 0), market.get('liquidity', 0), market.get('end_date_iso'), market.get('status', 'active'), json.dumps(market.get('outcome_tokens', {})), slug )) db.commit() cur.close() except Exception as e: logger.error(f"Failed to save markets to DB: {type(e).__name__}: {e}", exc_info=True) def _get_sample_markets(self, category: str = None, limit: int = 50) -> List[Dict]: """ Get sample market data (deprecated) You should now use real API data. """ # No longer returns sample data, returns an empty list logger.warning("Sample data method called, but real API should be used instead") return []