From b8de7e7bb1ff5695368855e70afbbf635a0c13bd Mon Sep 17 00:00:00 2001 From: warproxxx Date: Sun, 22 Jun 2025 08:25:29 -0700 Subject: [PATCH 01/11] Made data retreiver faster --- data_updater/find_markets.py | 48 ++++++++++++++++++++++++------------ update_markets.py | 5 +++- 2 files changed, 36 insertions(+), 17 deletions(-) diff --git a/data_updater/find_markets.py b/data_updater/find_markets.py index cc177b2..448f000 100644 --- a/data_updater/find_markets.py +++ b/data_updater/find_markets.py @@ -216,20 +216,27 @@ def process_single_row(row, client): return ret -def get_all_results(all_df, client): +def get_all_results(all_df, client, max_workers=5): all_results = [] - for idx, row in all_df.iterrows(): - + def process_with_progress(args): + idx, row = args try: - if idx % 10 == 0: - print(f'{idx} of {len(all_df)}') - - time.sleep(1) - result = process_single_row(row, client) - all_results.append(result) + return process_single_row(row, client) except: print("error fetching market") + return None + + with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor: + futures = [executor.submit(process_with_progress, (idx, row)) for idx, row in all_df.iterrows()] + + for future in concurrent.futures.as_completed(futures): + result = future.result() + if result is not None: + all_results.append(result) + + if len(all_results) % (max_workers * 2) == 0: + print(f'{len(all_results)} of {len(all_df)}') return all_results @@ -283,21 +290,30 @@ def add_volatility(row): new_dict = {**row_dict, **stats} return new_dict -def add_volatility_to_df(df): +def add_volatility_to_df(df, max_workers=3): results = [] df = df.reset_index(drop=True) - for idx, row in df.iterrows(): + def process_volatility_with_progress(args): + idx, row = args try: - if idx % 10 == 0: - print(f'{idx} of {len(df)}') - ret = add_volatility(row.to_dict()) - time.sleep(1) - results.append(ret) + return ret except: print("Error fetching volatility") + return None + + with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor: + futures = [executor.submit(process_volatility_with_progress, (idx, row)) for idx, row in df.iterrows()] + + for future in concurrent.futures.as_completed(futures): + result = future.result() + if result is not None: + results.append(result) + + if len(results) % (max_workers * 2) == 0: + print(f'{len(results)} of {len(df)}') return pd.DataFrame(results) diff --git a/update_markets.py b/update_markets.py index ce1d19e..d3a215f 100644 --- a/update_markets.py +++ b/update_markets.py @@ -88,8 +88,11 @@ def fetch_and_process_data(): all_df = get_all_markets(client) + print("Got all Markets") all_results = get_all_results(all_df, client) + print("Got all Results") m_data, all_markets = get_markets(all_results, sel_df, maker_reward=0.75) + print("Got all orderbook") print(f'{pd.to_datetime("now")}: Fetched all markets data of length {len(all_markets)}.') new_df = add_volatility_to_df(all_markets) @@ -124,7 +127,7 @@ if __name__ == "__main__": while True: try: fetch_and_process_data() - time.sleep(60 * 5) # Sleep for 5 minutes (300 seconds) + time.sleep(60 * 60) # Sleep for an hour except Exception as e: traceback.print_exc() print(str(e)) From 845669757722b925c8f5b3182e9184b4b76b36c6 Mon Sep 17 00:00:00 2001 From: warproxxx Date: Sun, 22 Jun 2025 09:38:27 -0700 Subject: [PATCH 02/11] Implement progressive position sizing with max_size parameter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add max_size support to trading logic, defaulting to trade_size - Continue quoting trade_size amounts until max_size is reached - Implement progressive exit strategy: sell trade_size increments when at max_size - Track both token positions for better exposure management - Update buy conditions to use max_size instead of 0.9 * trade_size 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude --- poly_data/trading_utils.py | 37 ++++++++++++++++++++++++++++++++----- trading.py | 24 ++++++++++++++++++------ 2 files changed, 50 insertions(+), 11 deletions(-) diff --git a/poly_data/trading_utils.py b/poly_data/trading_utils.py index 25b84ee..eef5704 100644 --- a/poly_data/trading_utils.py +++ b/poly_data/trading_utils.py @@ -128,18 +128,45 @@ def round_up(number, decimals): factor = 10 ** decimals return math.ceil(number * factor) / factor -def get_buy_sell_amount(position, bid_price, row): +def get_buy_sell_amount(position, bid_price, row, other_token_position=0): buy_amount = 0 sell_amount = 0 - sell_amount = position - buy_amount = row['trade_size'] - position + # Get max_size, defaulting to trade_size if not specified + max_size = row.get('max_size', row['trade_size']) + trade_size = row['trade_size'] + + # Calculate total exposure across both sides + total_exposure = position + other_token_position + + # If we haven't reached max_size on either side, continue building + if position < max_size: + # Continue quoting trade_size amounts until we reach max_size + remaining_to_max = max_size - position + buy_amount = min(trade_size, remaining_to_max) + + # Only sell if we have substantial position (to allow for exit when needed) + if position >= trade_size: + sell_amount = min(position, trade_size) + else: + sell_amount = 0 + else: + # We've reached max_size, implement progressive exit strategy + # Always offer to sell trade_size amount when at max_size + sell_amount = min(position, trade_size) + + # Continue quoting to buy if total exposure warrants it + if total_exposure < max_size * 2: # Allow some flexibility for market making + buy_amount = trade_size + else: + buy_amount = 0 + # Ensure minimum order size compliance if buy_amount > 0.7 * row['min_size'] and buy_amount < row['min_size']: buy_amount = row['min_size'] - if bid_price < 0.1: - + # Apply multiplier for low-priced assets + if bid_price < 0.1 and buy_amount > 0: if row['multiplier'] != '': print(f"Multiplying buy amount by {int(row['multiplier'])}") buy_amount = buy_amount * int(row['multiplier']) diff --git a/trading.py b/trading.py index 638a18a..0be12e6 100644 --- a/trading.py +++ b/trading.py @@ -161,6 +161,10 @@ async def perform_trade(market): # Get market depth and price information deets = get_best_bid_ask_deets(market, detail['name'], 100, 0.1) + + #if deet has None for one these values below, call it with min size of 20 + if deets['best_bid'] is None or deets['best_ask'] is None or deets['best_bid_size'] is None or deets['best_ask_size'] is None: + deets = get_best_bid_ask_deets(market, detail['name'], 20, 0.1) # Extract all order book details best_bid = deets['best_bid'] @@ -217,8 +221,12 @@ async def perform_trade(market): f"avgPrice: {avgPrice}, Best Bid: {best_bid}, Best Ask: {best_ask}, " f"Bid Price: {bid_price}, Ask Price: {ask_price}, Mid Price: {mid_price}") + # Get position for the opposite token to calculate total exposure + other_token = global_state.REVERSE_TOKENS[str(token)] + other_position = get_position(other_token)['size'] + # Calculate how much to buy or sell based on our position - buy_amount, sell_amount = get_buy_sell_amount(position, bid_price, row) + buy_amount, sell_amount = get_buy_sell_amount(position, bid_price, row, other_position) # Prepare order object with all necessary information order = { @@ -231,7 +239,8 @@ async def perform_trade(market): 'row': row } - print(f"Position: {position}, Trade Size: {row['trade_size']}, " + print(f"Position: {position}, Other Position: {other_position}, " + f"Trade Size: {row['trade_size']}, Max Size: {max_size}, " f"buy_amount: {buy_amount}, sell_amount: {sell_amount}") # File to store risk management information for this market @@ -298,11 +307,14 @@ async def perform_trade(market): continue # ------- BUY ORDER LOGIC ------- + # Get max_size, defaulting to trade_size if not specified + max_size = row.get('max_size', row['trade_size']) + # Only buy if: - # 1. Position is less than 90% of target size + # 1. Position is less than max_size (new logic) # 2. Position is less than absolute cap (250) # 3. Buy amount is above minimum size - if position < 0.9 * row['trade_size'] and position < 250 and buy_amount > 0 and buy_amount >= row['min_size']: + if position < max_size and position < 250 and buy_amount > 0 and buy_amount >= row['min_size']: # Get reference price from market data sheet_value = row['best_bid'] @@ -366,8 +378,8 @@ async def perform_trade(market): print(f"Sending Buy Order for {token} because better price. " f"Orders look like this: {orders['buy']}. Best Bid: {best_bid}") send_buy_order(order) - # 2. Current position + orders is not enough to reach target - elif position + orders['buy']['size'] < 0.95 * row['trade_size']: + # 2. Current position + orders is not enough to reach max_size + elif position + orders['buy']['size'] < 0.95 * max_size: print(f"Sending Buy Order for {token} because not enough position + size") send_buy_order(order) # 3. Our current order is too large and needs to be resized From 7ed78050f3e13be38bf034e0eed8178907ed2081 Mon Sep 17 00:00:00 2001 From: warproxxx Date: Sun, 22 Jun 2025 10:05:20 -0700 Subject: [PATCH 03/11] Created a read only mode if no credentials.json --- .env.example | 3 +- README.md | 2 +- data_updater/google_utils.py | 93 ++++++++++++++++++++++++++++++++-- poly_data/utils.py | 24 +++++++-- poly_utils/google_utils.py | 98 +++++++++++++++++++++++++++++++++--- requirements.txt | 3 +- 6 files changed, 204 insertions(+), 19 deletions(-) diff --git a/.env.example b/.env.example index 8f0f8b2..c0fbcc7 100644 --- a/.env.example +++ b/.env.example @@ -3,4 +3,5 @@ PK=your_private_key_here BROWSER_ADDRESS=your_wallet_address_here # Google Sheets (for data_updater) -SPREADSHEET_URL=your_google_spreadsheet_url_here \ No newline at end of file +SPREADSHEET_URL=https://docs.google.com/spreadsheets/d/1Kt6yGY7CZpB75cLJJAdWo7LSp9Oz7pjqfuVWwgtn7Ns/edit?gid=97507557#gid=97507557 +#replace with YOUR url diff --git a/README.md b/README.md index cbfc3af..a57e9bb 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ The repository consists of several interconnected modules: ## Requirements -- Python 3.8+ +- Python 3.9 with latest setuptools - Node.js (for poly_merger) - Google Sheets API credentials - Polymarket account and API credentials diff --git a/data_updater/google_utils.py b/data_updater/google_utils.py index 09d3de7..99562ac 100644 --- a/data_updater/google_utils.py +++ b/data_updater/google_utils.py @@ -1,13 +1,96 @@ from google.oauth2.service_account import Credentials import gspread import os +import pandas as pd +import requests +import re -def get_spreadsheet(): - """Get the main Google Spreadsheet using credentials and URL from environment variables""" +def get_spreadsheet(read_only=False): + """ + Get the main Google Spreadsheet using credentials and URL from environment variables + + Args: + read_only (bool): If True, uses public CSV export when credentials are missing + """ + spreadsheet_url = os.getenv("SPREADSHEET_URL") + if not spreadsheet_url: + raise ValueError("SPREADSHEET_URL environment variable is not set") + + # Check for credentials + if not os.path.exists('credentials.json'): + if read_only: + return ReadOnlySpreadsheet(spreadsheet_url) + else: + raise FileNotFoundError("credentials.json not found. Use read_only=True for read-only access.") + + # Normal authenticated access scope = ["https://spreadsheets.google.com/feeds", "https://www.googleapis.com/auth/drive"] - credentials = Credentials.from_service_account_file('credentials.json', scopes=scope) client = gspread.authorize(credentials) - spreadsheet = client.open_by_url(os.getenv("SPREADSHEET_URL")) - return spreadsheet \ No newline at end of file + spreadsheet = client.open_by_url(spreadsheet_url) + return spreadsheet + +class ReadOnlySpreadsheet: + """Read-only wrapper for Google Sheets using public CSV export""" + + def __init__(self, spreadsheet_url): + self.spreadsheet_url = spreadsheet_url + self.sheet_id = self._extract_sheet_id(spreadsheet_url) + + def _extract_sheet_id(self, url): + """Extract sheet ID from Google Sheets URL""" + match = re.search(r'/spreadsheets/d/([a-zA-Z0-9-_]+)', url) + if not match: + raise ValueError("Invalid Google Sheets URL") + return match.group(1) + + def worksheet(self, title): + """Return a read-only worksheet""" + return ReadOnlyWorksheet(self.sheet_id, title) + +class ReadOnlyWorksheet: + """Read-only worksheet that fetches data via CSV export""" + + def __init__(self, sheet_id, title): + self.sheet_id = sheet_id + self.title = title + + def get_all_records(self): + """Get all records from the worksheet as a list of dictionaries""" + try: + # Use the public CSV export URL + csv_url = f"https://docs.google.com/spreadsheets/d/{self.sheet_id}/gviz/tq?tqx=out:csv&sheet={self.title}" + response = requests.get(csv_url, timeout=30) + response.raise_for_status() + + # Read CSV data into DataFrame + from io import StringIO + df = pd.read_csv(StringIO(response.text)) + + # Convert to list of dictionaries (same format as gspread) + return df.to_dict('records') + + except Exception as e: + print(f"Warning: Could not fetch data from sheet '{self.title}': {e}") + return [] + + def get_all_values(self): + """Get all values from the worksheet as a list of lists""" + try: + csv_url = f"https://docs.google.com/spreadsheets/d/{self.sheet_id}/gviz/tq?tqx=out:csv&sheet={self.title}" + response = requests.get(csv_url, timeout=30) + response.raise_for_status() + + # Read CSV and return as list of lists + from io import StringIO + df = pd.read_csv(StringIO(response.text)) + + # Include headers and convert to list of lists + headers = [df.columns.tolist()] + data = df.values.tolist() + return headers + data + + except Exception as e: + print(f"Warning: Could not fetch data from sheet '{self.title}': {e}") + return [] \ No newline at end of file diff --git a/poly_data/utils.py b/poly_data/utils.py index a0a059a..3337c09 100644 --- a/poly_data/utils.py +++ b/poly_data/utils.py @@ -1,16 +1,33 @@ import json from poly_utils.google_utils import get_spreadsheet import pandas as pd +import os def pretty_print(txt, dic): print("\n", txt, json.dumps(dic, indent=4)) -def get_sheet_df(): - +def get_sheet_df(read_only=None): + """ + Get sheet data with optional read-only mode + + Args: + read_only (bool): If None, auto-detects based on credentials availability + """ all = 'All Markets' sel = 'Selected Markets' - spreadsheet = get_spreadsheet() + # Auto-detect read-only mode if not specified + if read_only is None: + creds_file = 'credentials.json' if os.path.exists('credentials.json') else '../credentials.json' + read_only = not os.path.exists(creds_file) + if read_only: + print("No credentials found, using read-only mode") + + try: + spreadsheet = get_spreadsheet(read_only=read_only) + except FileNotFoundError: + print("No credentials found, falling back to read-only mode") + spreadsheet = get_spreadsheet(read_only=True) wk = spreadsheet.worksheet(sel) df = pd.DataFrame(wk.get_all_records()) @@ -20,7 +37,6 @@ def get_sheet_df(): df2 = pd.DataFrame(wk2.get_all_records()) df2 = df2[df2['question'] != ""].reset_index(drop=True) - result = df.merge(df2, on='question', how='inner') wk_p = spreadsheet.worksheet('Hyperparameters') diff --git a/poly_utils/google_utils.py b/poly_utils/google_utils.py index 4ea1955..5020792 100644 --- a/poly_utils/google_utils.py +++ b/poly_utils/google_utils.py @@ -1,21 +1,105 @@ from google.oauth2.service_account import Credentials import gspread import os +import pandas as pd +import requests +import re from dotenv import load_dotenv load_dotenv() -def get_spreadsheet(): - scope = ["https://spreadsheets.google.com/feeds", "https://www.googleapis.com/auth/drive"] - - creds_file = 'credentials.json' if os.path.exists('credentials.json') else '../credentials.json' - credentials = Credentials.from_service_account_file(creds_file, scopes=scope) - client = gspread.authorize(credentials) +def get_spreadsheet(read_only=False): + """ + Get Google Spreadsheet with optional read-only mode. + + Args: + read_only (bool): If True, uses public CSV export when credentials are missing + + Returns: + Spreadsheet object or ReadOnlySpreadsheet wrapper for read-only mode + """ spreadsheet_url = os.getenv("SPREADSHEET_URL") if not spreadsheet_url: raise ValueError("SPREADSHEET_URL environment variable is not set") - + + # Check for credentials + creds_file = 'credentials.json' if os.path.exists('credentials.json') else '../credentials.json' + + if not os.path.exists(creds_file): + if read_only: + return ReadOnlySpreadsheet(spreadsheet_url) + else: + raise FileNotFoundError(f"Credentials file not found at {creds_file}. Use read_only=True for read-only access.") + + # Normal authenticated access + scope = ["https://spreadsheets.google.com/feeds", "https://www.googleapis.com/auth/drive"] + credentials = Credentials.from_service_account_file(creds_file, scopes=scope) + client = gspread.authorize(credentials) spreadsheet = client.open_by_url(spreadsheet_url) return spreadsheet +class ReadOnlySpreadsheet: + """Read-only wrapper for Google Sheets using public CSV export""" + + def __init__(self, spreadsheet_url): + self.spreadsheet_url = spreadsheet_url + self.sheet_id = self._extract_sheet_id(spreadsheet_url) + + def _extract_sheet_id(self, url): + """Extract sheet ID from Google Sheets URL""" + match = re.search(r'/spreadsheets/d/([a-zA-Z0-9-_]+)', url) + if not match: + raise ValueError("Invalid Google Sheets URL") + return match.group(1) + + def worksheet(self, title): + """Return a read-only worksheet""" + return ReadOnlyWorksheet(self.sheet_id, title) + +class ReadOnlyWorksheet: + """Read-only worksheet that fetches data via CSV export""" + + def __init__(self, sheet_id, title): + self.sheet_id = sheet_id + self.title = title + + def get_all_records(self): + """Get all records from the worksheet as a list of dictionaries""" + try: + # Use the public CSV export URL + csv_url = f"https://docs.google.com/spreadsheets/d/{self.sheet_id}/gviz/tq?tqx=out:csv&sheet={self.title}" + response = requests.get(csv_url, timeout=30) + response.raise_for_status() + + # Read CSV data into DataFrame + from io import StringIO + df = pd.read_csv(StringIO(response.text)) + + # Convert to list of dictionaries (same format as gspread) + return df.to_dict('records') + + except Exception as e: + print(f"Warning: Could not fetch data from sheet '{self.title}': {e}") + return [] + + def get_all_values(self): + """Get all values from the worksheet as a list of lists""" + try: + csv_url = f"https://docs.google.com/spreadsheets/d/{self.sheet_id}/gviz/tq?tqx=out:csv&sheet={self.title}" + response = requests.get(csv_url, timeout=30) + response.raise_for_status() + + # Read CSV and return as list of lists + from io import StringIO + df = pd.read_csv(StringIO(response.text)) + + # Include headers and convert to list of lists + headers = [df.columns.tolist()] + data = df.values.tolist() + return headers + data + + except Exception as e: + print(f"Warning: Could not fetch data from sheet '{self.title}': {e}") + return [] + diff --git a/requirements.txt b/requirements.txt index 0f227fa..854dc6d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,4 +16,5 @@ websockets==12.0 cryptography==42.0.8 asyncio requests -google-auth \ No newline at end of file +google-auth +web3==5.3.1 \ No newline at end of file From 9b327f451e8605e238edcdc6b87ae9ef3067eeec Mon Sep 17 00:00:00 2001 From: warproxxx Date: Sun, 22 Jun 2025 10:07:14 -0700 Subject: [PATCH 04/11] Fix a bug with max size calculation --- trading.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/trading.py b/trading.py index 0be12e6..0dd0d77 100644 --- a/trading.py +++ b/trading.py @@ -227,6 +227,9 @@ async def perform_trade(market): # Calculate how much to buy or sell based on our position buy_amount, sell_amount = get_buy_sell_amount(position, bid_price, row, other_position) + + # Get max_size for logging (same logic as in get_buy_sell_amount) + max_size = row.get('max_size', row['trade_size']) # Prepare order object with all necessary information order = { From dacee404822ae06dbafa2c6a8b2f4f0901b037ef Mon Sep 17 00:00:00 2001 From: warproxxx Date: Sun, 22 Jun 2025 10:15:07 -0700 Subject: [PATCH 05/11] Optimize read only mode --- poly_utils/google_utils.py | 68 +++++++++++++++++++++++++++++++++----- trading.py | 2 +- 2 files changed, 60 insertions(+), 10 deletions(-) diff --git a/poly_utils/google_utils.py b/poly_utils/google_utils.py index 5020792..59544f4 100644 --- a/poly_utils/google_utils.py +++ b/poly_utils/google_utils.py @@ -66,17 +66,67 @@ class ReadOnlyWorksheet: def get_all_records(self): """Get all records from the worksheet as a list of dictionaries""" try: - # Use the public CSV export URL - csv_url = f"https://docs.google.com/spreadsheets/d/{self.sheet_id}/gviz/tq?tqx=out:csv&sheet={self.title}" - response = requests.get(csv_url, timeout=30) - response.raise_for_status() + # URL encode the sheet title to handle spaces and special characters + import urllib.parse + encoded_title = urllib.parse.quote(self.title) - # Read CSV data into DataFrame - from io import StringIO - df = pd.read_csv(StringIO(response.text)) + # Map known sheet names to likely GID positions + # Based on the sheet order: Full Markets, All Markets, Volatility Markets, Selected Markets, Hyperparameters + sheet_gid_mapping = { + 'Full Markets': 0, + 'All Markets': 1, + 'Volatility Markets': 2, + 'Selected Markets': 3, + 'Hyperparameters': 4 + } - # Convert to list of dictionaries (same format as gspread) - return df.to_dict('records') + # Try multiple URL formats for accessing the sheet + urls_to_try = [ + f"https://docs.google.com/spreadsheets/d/{self.sheet_id}/gviz/tq?tqx=out:csv&sheet={encoded_title}", + f"https://docs.google.com/spreadsheets/d/{self.sheet_id}/gviz/tq?tqx=out:csv&sheet={self.title}", + ] + + # Add GID-based URL if we know the likely position for this sheet + if self.title in sheet_gid_mapping: + gid = sheet_gid_mapping[self.title] + urls_to_try.append(f"https://docs.google.com/spreadsheets/d/{self.sheet_id}/export?format=csv&gid={gid}") + + # Also try a few common GID positions as fallback + for gid in [0, 1, 2, 3, 4]: + urls_to_try.append(f"https://docs.google.com/spreadsheets/d/{self.sheet_id}/export?format=csv&gid={gid}") + + for csv_url in urls_to_try: + try: + print(f"Trying to fetch sheet '{self.title}' from: {csv_url}") + response = requests.get(csv_url, timeout=30) + response.raise_for_status() + + # Read CSV data into DataFrame + from io import StringIO + df = pd.read_csv(StringIO(response.text)) + + # Check if we got meaningful data (not empty or error response) + if not df.empty and len(df.columns) > 1: + # For Hyperparameters sheet, verify it has the expected columns + if self.title == 'Hyperparameters': + expected_cols = ['type', 'param', 'value'] + if all(col in df.columns for col in expected_cols): + print(f"Successfully fetched {len(df)} hyperparameter records") + return df.to_dict('records') + else: + print(f"Sheet doesn't match Hyperparameters format. Columns: {list(df.columns)}") + continue + else: + print(f"Successfully fetched {len(df)} records from sheet '{self.title}'") + # Convert to list of dictionaries (same format as gspread) + return df.to_dict('records') + + except Exception as url_error: + print(f"Failed with URL {csv_url}: {url_error}") + continue + + print(f"All URL attempts failed for sheet '{self.title}'") + return [] except Exception as e: print(f"Warning: Could not fetch data from sheet '{self.title}': {e}") diff --git a/trading.py b/trading.py index 0dd0d77..46a57ee 100644 --- a/trading.py +++ b/trading.py @@ -429,7 +429,7 @@ async def perform_trade(market): # send_sell_order(order) except Exception as ex: - print(f"Error performing trade for {market}") + print(f"Error performing trade for {market}: {ex}") traceback.print_exc() # Clean up memory and introduce a small delay From ee24553856090b24774fb8a15c9c8d57362f7fcf Mon Sep 17 00:00:00 2001 From: warproxxx Date: Sun, 22 Jun 2025 10:18:29 -0700 Subject: [PATCH 06/11] Fix hyperparameter parsing for read-only mode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Improved CSV parsing logic for hyperparameters with empty type cells - Enhanced read-only mode sheet access with multiple URL formats and GID mapping - Added debug logging to identify parameter type mismatches - Fixed UnboundLocalError for max_size variable - Fixed unused variable warnings 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude --- poly_data/data_utils.py | 13 ++++++++++++- poly_data/utils.py | 21 +++++++++++++++++++-- trading.py | 2 ++ 3 files changed, 33 insertions(+), 3 deletions(-) diff --git a/poly_data/data_utils.py b/poly_data/data_utils.py index 0ff03a4..1d9af35 100644 --- a/poly_data/data_utils.py +++ b/poly_data/data_utils.py @@ -150,8 +150,19 @@ def update_markets(): if len(received_df) > 0: global_state.df, global_state.params = received_df.copy(), received_params + + # Debug: Show what param_types are in the markets data + if 'param_type' in received_df.columns: + param_types_used = received_df['param_type'].unique() + print(f"DEBUG: param_types used in markets: {param_types_used}") + print(f"DEBUG: Available hyperparameter types: {list(received_params.keys())}") + + # Check for mismatches + missing_params = [pt for pt in param_types_used if pt not in received_params.keys()] + if missing_params: + print(f"WARNING: Missing hyperparameters for param_types: {missing_params}") - for idx, row in global_state.df.iterrows(): + for _, row in global_state.df.iterrows(): for col in ['token1', 'token2']: row[col] = str(row[col]) diff --git a/poly_data/utils.py b/poly_data/utils.py index 3337c09..5467ed0 100644 --- a/poly_data/utils.py +++ b/poly_data/utils.py @@ -44,7 +44,24 @@ def get_sheet_df(read_only=None): hyperparams, current_type = {}, None for r in records: - current_type = r['type'] or current_type - hyperparams.setdefault(current_type, {})[r['param']] = r['value'] + # Update current_type only when we have a non-empty type value + if r['type'] and r['type'].strip(): + current_type = r['type'].strip() + + # Skip rows where we don't have a current_type set + if current_type: + # Convert numeric values to appropriate types + value = r['value'] + try: + # Try to convert to float if it's numeric + if isinstance(value, str) and value.replace('.', '').replace('-', '').isdigit(): + value = float(value) + elif isinstance(value, (int, float)): + value = float(value) + except (ValueError, TypeError): + pass # Keep as string if conversion fails + + hyperparams.setdefault(current_type, {})[r['param']] = value + print(f"DEBUG: Loaded hyperparameters: {hyperparams}") return result, hyperparams diff --git a/trading.py b/trading.py index 46a57ee..df6bd77 100644 --- a/trading.py +++ b/trading.py @@ -119,6 +119,8 @@ async def perform_trade(market): # Get trading parameters for this market type params = global_state.params[row['param_type']] + print(f"DEBUG: Using param_type: {row['param_type']}, Available params: {params}") + print(f"DEBUG: All available param types: {list(global_state.params.keys())}") # Create a list with both outcomes for the market deets = [ From ed7a1f9be3fcf17b409f0abbb81f3ac20d38f4fc Mon Sep 17 00:00:00 2001 From: warproxxx Date: Sun, 22 Jun 2025 10:20:07 -0700 Subject: [PATCH 07/11] Fix AttributeError in hyperparameter parsing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Handle NaN values from pandas when CSV cells are empty - convert to string before calling strip() method. 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude --- poly_data/utils.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/poly_data/utils.py b/poly_data/utils.py index 5467ed0..349df39 100644 --- a/poly_data/utils.py +++ b/poly_data/utils.py @@ -45,8 +45,10 @@ def get_sheet_df(read_only=None): for r in records: # Update current_type only when we have a non-empty type value - if r['type'] and r['type'].strip(): - current_type = r['type'].strip() + # Handle both string and NaN values from pandas + type_value = r['type'] + if type_value and str(type_value).strip() and str(type_value) != 'nan': + current_type = str(type_value).strip() # Skip rows where we don't have a current_type set if current_type: From fa2c3eff972d8bba6688441a33e163e5dd028d0c Mon Sep 17 00:00:00 2001 From: warproxxx Date: Sun, 22 Jun 2025 10:22:31 -0700 Subject: [PATCH 08/11] Fix TypeError in market data processing when prices are None MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Handle None values returned from find_best_price_with_size() to prevent arithmetic operations on NoneType. Added null checks for: - Mid price calculation - Token2 price transformations (1-x operations) - Bid/ask sum calculations 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude --- poly_data/trading_utils.py | 35 ++++++++++++++++++++++++++++------- 1 file changed, 28 insertions(+), 7 deletions(-) diff --git a/poly_data/trading_utils.py b/poly_data/trading_utils.py index eef5704..752c034 100644 --- a/poly_data/trading_utils.py +++ b/poly_data/trading_utils.py @@ -30,15 +30,36 @@ def get_best_bid_ask_deets(market, name, size, deviation_threshold=0.05): best_bid, best_bid_size, second_best_bid, second_best_bid_size, top_bid = find_best_price_with_size(global_state.all_data[market]['bids'], size, reverse=True) best_ask, best_ask_size, second_best_ask, second_best_ask_size, top_ask = find_best_price_with_size(global_state.all_data[market]['asks'], size, reverse=False) - mid_price = (best_bid + best_ask) / 2 - - bid_sum_within_n_percent = sum(size for price, size in global_state.all_data[market]['bids'].items() if best_bid <= price <= mid_price * (1 + deviation_threshold)) - ask_sum_within_n_percent = sum(size for price, size in global_state.all_data[market]['asks'].items() if mid_price * (1 - deviation_threshold) <= price <= best_ask) + # Handle None values in mid_price calculation + if best_bid is not None and best_ask is not None: + mid_price = (best_bid + best_ask) / 2 + bid_sum_within_n_percent = sum(size for price, size in global_state.all_data[market]['bids'].items() if best_bid <= price <= mid_price * (1 + deviation_threshold)) + ask_sum_within_n_percent = sum(size for price, size in global_state.all_data[market]['asks'].items() if mid_price * (1 - deviation_threshold) <= price <= best_ask) + else: + mid_price = None + bid_sum_within_n_percent = 0 + ask_sum_within_n_percent = 0 if name == 'token2': - best_bid, second_best_bid, top_bid, best_ask, second_best_ask, top_ask = 1 - best_ask, 1 - second_best_ask, 1 - top_ask, 1 - best_bid, 1 - second_best_bid, 1 - top_bid - best_bid_size, second_best_bid_size, best_ask_size, second_best_ask_size = best_ask_size, second_best_ask_size, best_bid_size, second_best_bid_size - bid_sum_within_n_percent, ask_sum_within_n_percent = ask_sum_within_n_percent, bid_sum_within_n_percent + # Handle None values before arithmetic operations + if all(x is not None for x in [best_bid, best_ask, second_best_bid, second_best_ask, top_bid, top_ask]): + best_bid, second_best_bid, top_bid, best_ask, second_best_ask, top_ask = 1 - best_ask, 1 - second_best_ask, 1 - top_ask, 1 - best_bid, 1 - second_best_bid, 1 - top_bid + best_bid_size, second_best_bid_size, best_ask_size, second_best_ask_size = best_ask_size, second_best_ask_size, best_bid_size, second_best_bid_size + bid_sum_within_n_percent, ask_sum_within_n_percent = ask_sum_within_n_percent, bid_sum_within_n_percent + else: + # Handle case where some prices are None - use available values or defaults + if best_bid is not None and best_ask is not None: + best_bid, best_ask = 1 - best_ask, 1 - best_bid + best_bid_size, best_ask_size = best_ask_size, best_bid_size + if second_best_bid is not None: + second_best_bid = 1 - second_best_bid + if second_best_ask is not None: + second_best_ask = 1 - second_best_ask + if top_bid is not None: + top_bid = 1 - top_bid + if top_ask is not None: + top_ask = 1 - top_ask + bid_sum_within_n_percent, ask_sum_within_n_percent = ask_sum_within_n_percent, bid_sum_within_n_percent From 4d206e05d87fa1bd3992a0e63513e4f988811d0f Mon Sep 17 00:00:00 2001 From: warproxxx Date: Sun, 22 Jun 2025 10:28:13 -0700 Subject: [PATCH 09/11] Fix excessive order cancellation and recreation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implement smarter order management logic to avoid constantly cancelling and recreating orders: - Only cancel orders when price differs by >0.5 cents or size differs by >10% - Keep existing orders if changes are minor to reduce unnecessary market activity - Add detailed logging to show when orders are kept vs cancelled This should resolve the issue where orders were being cancelled and recreated repeatedly. 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude --- trading.py | 42 ++++++++++++++++++++++++++++++++++++++---- 1 file changed, 38 insertions(+), 4 deletions(-) diff --git a/trading.py b/trading.py index df6bd77..7434970 100644 --- a/trading.py +++ b/trading.py @@ -31,9 +31,26 @@ def send_buy_order(order): """ client = global_state.client - # Cancel existing orders for this token to avoid conflicts - if order['orders']['buy']['size'] > 0 or order['orders']['sell']['size'] > 0: + # Only cancel existing orders if we need to make significant changes + existing_buy_size = order['orders']['buy']['size'] + existing_buy_price = order['orders']['buy']['price'] + + # Cancel orders if price changed significantly or size needs major adjustment + price_diff = abs(existing_buy_price - order['price']) if existing_buy_price > 0 else float('inf') + size_diff = abs(existing_buy_size - order['size']) if existing_buy_size > 0 else float('inf') + + should_cancel = ( + price_diff > 0.005 or # Cancel if price diff > 0.5 cents + size_diff > order['size'] * 0.1 or # Cancel if size diff > 10% + existing_buy_size == 0 # Cancel if no existing buy order + ) + + if should_cancel and (existing_buy_size > 0 or order['orders']['sell']['size'] > 0): + print(f"Cancelling buy orders - price diff: {price_diff:.4f}, size diff: {size_diff:.1f}") client.cancel_all_asset(order['token']) + elif not should_cancel: + print(f"Keeping existing buy orders - minor changes: price diff: {price_diff:.4f}, size diff: {size_diff:.1f}") + return # Don't place new order if existing one is fine # Calculate minimum acceptable price based on market spread incentive_start = order['mid_price'] - order['max_spread']/100 @@ -75,9 +92,26 @@ def send_sell_order(order): """ client = global_state.client - # Cancel existing orders for this token to avoid conflicts - if order['orders']['buy']['size'] > 0 or order['orders']['sell']['size'] > 0: + # Only cancel existing orders if we need to make significant changes + existing_sell_size = order['orders']['sell']['size'] + existing_sell_price = order['orders']['sell']['price'] + + # Cancel orders if price changed significantly or size needs major adjustment + price_diff = abs(existing_sell_price - order['price']) if existing_sell_price > 0 else float('inf') + size_diff = abs(existing_sell_size - order['size']) if existing_sell_size > 0 else float('inf') + + should_cancel = ( + price_diff > 0.005 or # Cancel if price diff > 0.5 cents + size_diff > order['size'] * 0.1 or # Cancel if size diff > 10% + existing_sell_size == 0 # Cancel if no existing sell order + ) + + if should_cancel and (existing_sell_size > 0 or order['orders']['buy']['size'] > 0): + print(f"Cancelling sell orders - price diff: {price_diff:.4f}, size diff: {size_diff:.1f}") client.cancel_all_asset(order['token']) + elif not should_cancel: + print(f"Keeping existing sell orders - minor changes: price diff: {price_diff:.4f}, size diff: {size_diff:.1f}") + return # Don't place new order if existing one is fine print(f'Creating new order for {order["size"]} at {order["price"]}') client.create_order( From 652e3304248ad446190d4840d5f62a45238ef5df Mon Sep 17 00:00:00 2001 From: warproxxx Date: Sun, 22 Jun 2025 10:30:39 -0700 Subject: [PATCH 10/11] Remove debug logging statements MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Clean up debug prints that were added during troubleshooting of hyperparameter parsing issues. 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude --- poly_data/data_utils.py | 10 ---------- poly_data/utils.py | 1 - trading.py | 2 -- 3 files changed, 13 deletions(-) diff --git a/poly_data/data_utils.py b/poly_data/data_utils.py index 1d9af35..25976a8 100644 --- a/poly_data/data_utils.py +++ b/poly_data/data_utils.py @@ -151,16 +151,6 @@ def update_markets(): if len(received_df) > 0: global_state.df, global_state.params = received_df.copy(), received_params - # Debug: Show what param_types are in the markets data - if 'param_type' in received_df.columns: - param_types_used = received_df['param_type'].unique() - print(f"DEBUG: param_types used in markets: {param_types_used}") - print(f"DEBUG: Available hyperparameter types: {list(received_params.keys())}") - - # Check for mismatches - missing_params = [pt for pt in param_types_used if pt not in received_params.keys()] - if missing_params: - print(f"WARNING: Missing hyperparameters for param_types: {missing_params}") for _, row in global_state.df.iterrows(): for col in ['token1', 'token2']: diff --git a/poly_data/utils.py b/poly_data/utils.py index 349df39..4d9e658 100644 --- a/poly_data/utils.py +++ b/poly_data/utils.py @@ -65,5 +65,4 @@ def get_sheet_df(read_only=None): hyperparams.setdefault(current_type, {})[r['param']] = value - print(f"DEBUG: Loaded hyperparameters: {hyperparams}") return result, hyperparams diff --git a/trading.py b/trading.py index 7434970..6ed67ca 100644 --- a/trading.py +++ b/trading.py @@ -153,8 +153,6 @@ async def perform_trade(market): # Get trading parameters for this market type params = global_state.params[row['param_type']] - print(f"DEBUG: Using param_type: {row['param_type']}, Available params: {params}") - print(f"DEBUG: All available param types: {list(global_state.params.keys())}") # Create a list with both outcomes for the market deets = [ From a3ba8b7fcab26dbab8fa23cea4b6de998d05e6bc Mon Sep 17 00:00:00 2001 From: warproxxx Date: Sun, 22 Jun 2025 10:37:08 -0700 Subject: [PATCH 11/11] Update poly_merger configuration and documentation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Update RPC endpoint from llamarpc to polygon-rpc.com for better reliability - Improve .env file loading to check both local and parent directory - Update README example to use proper condition_id format (0x...) instead of numeric - Clarify that condition_id should be hex format in documentation 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude --- poly_merger/README.md | 4 ++-- poly_merger/merge.js | 8 ++++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/poly_merger/README.md b/poly_merger/README.md index 045e159..a940a02 100644 --- a/poly_merger/README.md +++ b/poly_merger/README.md @@ -20,10 +20,10 @@ node merge.js [amount_to_merge] [condition_id] [is_neg_risk_market] Example: ``` -node merge.js 1000000 1234567 true +node merge.js 1000000 0xasdasda true ``` -This would merge 1 USDC worth of opposing positions in market 1234567, which is a negative risk market. +This would merge 1 USDC worth of opposing positions in market 0xasdasda, which is a negative risk market. 0xasdasda should be condition_id ## Prerequisites diff --git a/poly_merger/merge.js b/poly_merger/merge.js index 83596ea..dc7e839 100644 --- a/poly_merger/merge.js +++ b/poly_merger/merge.js @@ -15,14 +15,18 @@ const { ethers } = require('ethers'); const { resolve } = require('path'); +const { existsSync } = require('fs'); const { signAndExecuteSafeTransaction } = require('./safe-helpers'); const { safeAbi } = require('./safeAbi'); // Load environment variables -require('dotenv').config() +const localEnvPath = resolve(__dirname, '.env'); +const parentEnvPath = resolve(__dirname, '../.env'); +const envPath = existsSync(localEnvPath) ? localEnvPath : parentEnvPath; +require('dotenv').config({ path: envPath }) // Connect to Polygon network -const provider = new ethers.providers.JsonRpcProvider("https://polygon.llamarpc.com"); +const provider = new ethers.providers.JsonRpcProvider("https://polygon-rpc.com"); const privateKey = process.env.PK; const wallet = new ethers.Wallet(privateKey, provider);