Merge pull request #6 from warproxxx/optimizations

Optimizations

.env.example

@@ -3,4 +3,5 @@ PK=your_private_key_here
 BROWSER_ADDRESS=your_wallet_address_here
 
 # Google Sheets (for data_updater)
-SPREADSHEET_URL=your_google_spreadsheet_url_here
+SPREADSHEET_URL=https://docs.google.com/spreadsheets/d/1Kt6yGY7CZpB75cLJJAdWo7LSp9Oz7pjqfuVWwgtn7Ns/edit?gid=97507557#gid=97507557
+#replace with YOUR url

README.md

@@ -24,7 +24,7 @@ The repository consists of several interconnected modules:
 
 ## Requirements
 
-- Python 3.8+
+- Python 3.9 with latest setuptools
 - Node.js (for poly_merger)
 - Google Sheets API credentials
 - Polymarket account and API credentials

data_updater/find_markets.py

@@ -216,20 +216,27 @@ def process_single_row(row, client):
     return ret
 
 
-def get_all_results(all_df, client):
+def get_all_results(all_df, client, max_workers=5):
     all_results = []
 
-    for idx, row in all_df.iterrows():
-
+    def process_with_progress(args):
+        idx, row = args
         try:
-            if idx % 10 == 0:
-                print(f'{idx} of {len(all_df)}')
-
-            time.sleep(1)
-            result = process_single_row(row, client)
-            all_results.append(result)
+            return process_single_row(row, client)
         except:
             print("error fetching market")
+            return None
+
+    with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
+        futures = [executor.submit(process_with_progress, (idx, row)) for idx, row in all_df.iterrows()]
+        
+        for future in concurrent.futures.as_completed(futures):
+            result = future.result()
+            if result is not None:
+                all_results.append(result)
+
+            if len(all_results) % (max_workers * 2) == 0:
+                print(f'{len(all_results)} of {len(all_df)}')
 
     return all_results
 
@@ -283,21 +290,30 @@ def add_volatility(row):
     new_dict = {**row_dict, **stats}
     return new_dict
 
-def add_volatility_to_df(df):
+def add_volatility_to_df(df, max_workers=3):
     
     results = []
     df = df.reset_index(drop=True)
 
-    for idx, row in df.iterrows():
+    def process_volatility_with_progress(args):
+        idx, row = args
         try:
-            if idx % 10 == 0:
-                print(f'{idx} of {len(df)}')
-                
             ret = add_volatility(row.to_dict())
-            time.sleep(1)
-            results.append(ret)
+            return ret
         except:
             print("Error fetching volatility")
+            return None
+
+    with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
+        futures = [executor.submit(process_volatility_with_progress, (idx, row)) for idx, row in df.iterrows()]
+        
+        for future in concurrent.futures.as_completed(futures):
+            result = future.result()
+            if result is not None:
+                results.append(result)
+                
+            if len(results) % (max_workers * 2) == 0:
+                print(f'{len(results)} of {len(df)}')
             
     return pd.DataFrame(results)
 

data_updater/google_utils.py

@@ -1,13 +1,96 @@
 from google.oauth2.service_account import Credentials
 import gspread
 import os
+import pandas as pd
+import requests
+import re
 
 
-def get_spreadsheet():
-    """Get the main Google Spreadsheet using credentials and URL from environment variables"""
+def get_spreadsheet(read_only=False):
+    """
+    Get the main Google Spreadsheet using credentials and URL from environment variables
+    
+    Args:
+        read_only (bool): If True, uses public CSV export when credentials are missing
+    """
+    spreadsheet_url = os.getenv("SPREADSHEET_URL")
+    if not spreadsheet_url:
+        raise ValueError("SPREADSHEET_URL environment variable is not set")
+    
+    # Check for credentials
+    if not os.path.exists('credentials.json'):
+        if read_only:
+            return ReadOnlySpreadsheet(spreadsheet_url)
+        else:
+            raise FileNotFoundError("credentials.json not found. Use read_only=True for read-only access.")
+    
+    # Normal authenticated access
     scope = ["https://spreadsheets.google.com/feeds", "https://www.googleapis.com/auth/drive"]
-
     credentials = Credentials.from_service_account_file('credentials.json', scopes=scope)
     client = gspread.authorize(credentials)
-    spreadsheet = client.open_by_url(os.getenv("SPREADSHEET_URL"))
-    return spreadsheet
+    spreadsheet = client.open_by_url(spreadsheet_url)
+    return spreadsheet
+
+class ReadOnlySpreadsheet:
+    """Read-only wrapper for Google Sheets using public CSV export"""
+    
+    def __init__(self, spreadsheet_url):
+        self.spreadsheet_url = spreadsheet_url
+        self.sheet_id = self._extract_sheet_id(spreadsheet_url)
+        
+    def _extract_sheet_id(self, url):
+        """Extract sheet ID from Google Sheets URL"""
+        match = re.search(r'/spreadsheets/d/([a-zA-Z0-9-_]+)', url)
+        if not match:
+            raise ValueError("Invalid Google Sheets URL")
+        return match.group(1)
+    
+    def worksheet(self, title):
+        """Return a read-only worksheet"""
+        return ReadOnlyWorksheet(self.sheet_id, title)
+
+class ReadOnlyWorksheet:
+    """Read-only worksheet that fetches data via CSV export"""
+    
+    def __init__(self, sheet_id, title):
+        self.sheet_id = sheet_id
+        self.title = title
+        
+    def get_all_records(self):
+        """Get all records from the worksheet as a list of dictionaries"""
+        try:
+            # Use the public CSV export URL
+            csv_url = f"https://docs.google.com/spreadsheets/d/{self.sheet_id}/gviz/tq?tqx=out:csv&sheet={self.title}"
+            response = requests.get(csv_url, timeout=30)
+            response.raise_for_status()
+            
+            # Read CSV data into DataFrame
+            from io import StringIO
+            df = pd.read_csv(StringIO(response.text))
+            
+            # Convert to list of dictionaries (same format as gspread)
+            return df.to_dict('records')
+            
+        except Exception as e:
+            print(f"Warning: Could not fetch data from sheet '{self.title}': {e}")
+            return []
+    
+    def get_all_values(self):
+        """Get all values from the worksheet as a list of lists"""
+        try:
+            csv_url = f"https://docs.google.com/spreadsheets/d/{self.sheet_id}/gviz/tq?tqx=out:csv&sheet={self.title}"
+            response = requests.get(csv_url, timeout=30)
+            response.raise_for_status()
+            
+            # Read CSV and return as list of lists
+            from io import StringIO
+            df = pd.read_csv(StringIO(response.text))
+            
+            # Include headers and convert to list of lists
+            headers = [df.columns.tolist()]
+            data = df.values.tolist()
+            return headers + data
+            
+        except Exception as e:
+            print(f"Warning: Could not fetch data from sheet '{self.title}': {e}")
+            return []

poly_data/data_utils.py

@@ -150,8 +150,9 @@ def update_markets():
 
     if len(received_df) > 0:
         global_state.df, global_state.params = received_df.copy(), received_params
+    
 
-    for idx, row in global_state.df.iterrows():
+    for _, row in global_state.df.iterrows():
         for col in ['token1', 'token2']:
             row[col] = str(row[col])
 

poly_data/trading_utils.py

@@ -30,15 +30,36 @@ def get_best_bid_ask_deets(market, name, size, deviation_threshold=0.05):
     best_bid, best_bid_size, second_best_bid, second_best_bid_size, top_bid = find_best_price_with_size(global_state.all_data[market]['bids'], size, reverse=True)
     best_ask, best_ask_size, second_best_ask, second_best_ask_size, top_ask = find_best_price_with_size(global_state.all_data[market]['asks'], size, reverse=False)
     
-    mid_price = (best_bid + best_ask) / 2
-
-    bid_sum_within_n_percent = sum(size for price, size in global_state.all_data[market]['bids'].items() if best_bid <= price <= mid_price * (1 + deviation_threshold))
-    ask_sum_within_n_percent = sum(size for price, size in global_state.all_data[market]['asks'].items() if mid_price * (1 - deviation_threshold) <= price <= best_ask)
+    # Handle None values in mid_price calculation
+    if best_bid is not None and best_ask is not None:
+        mid_price = (best_bid + best_ask) / 2
+        bid_sum_within_n_percent = sum(size for price, size in global_state.all_data[market]['bids'].items() if best_bid <= price <= mid_price * (1 + deviation_threshold))
+        ask_sum_within_n_percent = sum(size for price, size in global_state.all_data[market]['asks'].items() if mid_price * (1 - deviation_threshold) <= price <= best_ask)
+    else:
+        mid_price = None
+        bid_sum_within_n_percent = 0
+        ask_sum_within_n_percent = 0
 
     if name == 'token2':
-        best_bid, second_best_bid, top_bid, best_ask, second_best_ask, top_ask = 1 - best_ask, 1 - second_best_ask, 1 - top_ask, 1 - best_bid, 1 - second_best_bid, 1 - top_bid
-        best_bid_size, second_best_bid_size, best_ask_size, second_best_ask_size = best_ask_size, second_best_ask_size, best_bid_size, second_best_bid_size
-        bid_sum_within_n_percent, ask_sum_within_n_percent = ask_sum_within_n_percent, bid_sum_within_n_percent
+        # Handle None values before arithmetic operations
+        if all(x is not None for x in [best_bid, best_ask, second_best_bid, second_best_ask, top_bid, top_ask]):
+            best_bid, second_best_bid, top_bid, best_ask, second_best_ask, top_ask = 1 - best_ask, 1 - second_best_ask, 1 - top_ask, 1 - best_bid, 1 - second_best_bid, 1 - top_bid
+            best_bid_size, second_best_bid_size, best_ask_size, second_best_ask_size = best_ask_size, second_best_ask_size, best_bid_size, second_best_bid_size
+            bid_sum_within_n_percent, ask_sum_within_n_percent = ask_sum_within_n_percent, bid_sum_within_n_percent
+        else:
+            # Handle case where some prices are None - use available values or defaults
+            if best_bid is not None and best_ask is not None:
+                best_bid, best_ask = 1 - best_ask, 1 - best_bid
+                best_bid_size, best_ask_size = best_ask_size, best_bid_size
+            if second_best_bid is not None:
+                second_best_bid = 1 - second_best_bid
+            if second_best_ask is not None:
+                second_best_ask = 1 - second_best_ask
+            if top_bid is not None:
+                top_bid = 1 - top_bid
+            if top_ask is not None:
+                top_ask = 1 - top_ask
+            bid_sum_within_n_percent, ask_sum_within_n_percent = ask_sum_within_n_percent, bid_sum_within_n_percent
 
 
 
@@ -128,18 +149,45 @@ def round_up(number, decimals):
     factor = 10 ** decimals
     return math.ceil(number * factor) / factor
 
-def get_buy_sell_amount(position, bid_price, row):
+def get_buy_sell_amount(position, bid_price, row, other_token_position=0):
     buy_amount = 0
     sell_amount = 0
 
-    sell_amount = position
-    buy_amount = row['trade_size'] - position
+    # Get max_size, defaulting to trade_size if not specified
+    max_size = row.get('max_size', row['trade_size'])
+    trade_size = row['trade_size']
+    
+    # Calculate total exposure across both sides
+    total_exposure = position + other_token_position
+    
+    # If we haven't reached max_size on either side, continue building
+    if position < max_size:
+        # Continue quoting trade_size amounts until we reach max_size
+        remaining_to_max = max_size - position
+        buy_amount = min(trade_size, remaining_to_max)
+        
+        # Only sell if we have substantial position (to allow for exit when needed)
+        if position >= trade_size:
+            sell_amount = min(position, trade_size)
+        else:
+            sell_amount = 0
+    else:
+        # We've reached max_size, implement progressive exit strategy
+        # Always offer to sell trade_size amount when at max_size
+        sell_amount = min(position, trade_size)
+        
+        # Continue quoting to buy if total exposure warrants it
+        if total_exposure < max_size * 2:  # Allow some flexibility for market making
+            buy_amount = trade_size
+        else:
+            buy_amount = 0
 
+    # Ensure minimum order size compliance
     if buy_amount > 0.7 * row['min_size'] and buy_amount < row['min_size']:
         buy_amount = row['min_size']
 
-    if bid_price < 0.1:
-
+    # Apply multiplier for low-priced assets
+    if bid_price < 0.1 and buy_amount > 0:
         if row['multiplier'] != '':
             print(f"Multiplying buy amount by {int(row['multiplier'])}")
             buy_amount = buy_amount * int(row['multiplier'])

poly_data/utils.py

@@ -1,16 +1,33 @@
 import json
 from poly_utils.google_utils import get_spreadsheet
 import pandas as pd 
+import os
 
 def pretty_print(txt, dic):
     print("\n", txt, json.dumps(dic, indent=4))
 
-def get_sheet_df():
-
+def get_sheet_df(read_only=None):
+    """
+    Get sheet data with optional read-only mode
+    
+    Args:
+        read_only (bool): If None, auto-detects based on credentials availability
+    """
     all = 'All Markets'
     sel = 'Selected Markets'
 
-    spreadsheet = get_spreadsheet()
+    # Auto-detect read-only mode if not specified
+    if read_only is None:
+        creds_file = 'credentials.json' if os.path.exists('credentials.json') else '../credentials.json'
+        read_only = not os.path.exists(creds_file)
+        if read_only:
+            print("No credentials found, using read-only mode")
+
+    try:
+        spreadsheet = get_spreadsheet(read_only=read_only)
+    except FileNotFoundError:
+        print("No credentials found, falling back to read-only mode")
+        spreadsheet = get_spreadsheet(read_only=True)
 
     wk = spreadsheet.worksheet(sel)
     df = pd.DataFrame(wk.get_all_records())
@@ -20,7 +37,6 @@ def get_sheet_df():
     df2 = pd.DataFrame(wk2.get_all_records())
     df2 = df2[df2['question'] != ""].reset_index(drop=True)
 
-
     result = df.merge(df2, on='question', how='inner')
 
     wk_p = spreadsheet.worksheet('Hyperparameters')
@@ -28,7 +44,25 @@ def get_sheet_df():
     hyperparams, current_type = {}, None
 
     for r in records:
-        current_type = r['type'] or current_type
-        hyperparams.setdefault(current_type, {})[r['param']] = r['value']
+        # Update current_type only when we have a non-empty type value
+        # Handle both string and NaN values from pandas
+        type_value = r['type']
+        if type_value and str(type_value).strip() and str(type_value) != 'nan':
+            current_type = str(type_value).strip()
+        
+        # Skip rows where we don't have a current_type set
+        if current_type:
+            # Convert numeric values to appropriate types
+            value = r['value']
+            try:
+                # Try to convert to float if it's numeric
+                if isinstance(value, str) and value.replace('.', '').replace('-', '').isdigit():
+                    value = float(value)
+                elif isinstance(value, (int, float)):
+                    value = float(value)
+            except (ValueError, TypeError):
+                pass  # Keep as string if conversion fails
+            
+            hyperparams.setdefault(current_type, {})[r['param']] = value
 
     return result, hyperparams

poly_merger/README.md

@@ -20,10 +20,10 @@ node merge.js [amount_to_merge] [condition_id] [is_neg_risk_market]
 
 Example:
 ```
-node merge.js 1000000 1234567 true
+node merge.js 1000000 0xasdasda true
 ```
 
-This would merge 1 USDC worth of opposing positions in market 1234567, which is a negative risk market.
+This would merge 1 USDC worth of opposing positions in market 0xasdasda, which is a negative risk market. 0xasdasda should be condition_id
 
 ## Prerequisites
 

poly_merger/merge.js

@@ -15,14 +15,18 @@
 
 const { ethers } = require('ethers');
 const { resolve } = require('path');
+const { existsSync } = require('fs');
 const { signAndExecuteSafeTransaction } = require('./safe-helpers');
 const { safeAbi } = require('./safeAbi');
 
 // Load environment variables
-require('dotenv').config()
+const localEnvPath = resolve(__dirname, '.env');
+const parentEnvPath = resolve(__dirname, '../.env');
+const envPath = existsSync(localEnvPath) ? localEnvPath : parentEnvPath;
+require('dotenv').config({ path: envPath })
 
 // Connect to Polygon network
-const provider = new ethers.providers.JsonRpcProvider("https://polygon.llamarpc.com");
+const provider = new ethers.providers.JsonRpcProvider("https://polygon-rpc.com");
 const privateKey = process.env.PK;
 const wallet = new ethers.Wallet(privateKey, provider);
 

poly_utils/google_utils.py

@@ -1,21 +1,155 @@
 from google.oauth2.service_account import Credentials
 import gspread
 import os
+import pandas as pd
+import requests
+import re
 from dotenv import load_dotenv
 
 load_dotenv()
 
-def get_spreadsheet():
-    scope = ["https://spreadsheets.google.com/feeds", "https://www.googleapis.com/auth/drive"]
-
-    creds_file = 'credentials.json' if os.path.exists('credentials.json') else '../credentials.json'
-    credentials = Credentials.from_service_account_file(creds_file, scopes=scope)
-    client = gspread.authorize(credentials)
+def get_spreadsheet(read_only=False):
+    """
+    Get Google Spreadsheet with optional read-only mode.
+    
+    Args:
+        read_only (bool): If True, uses public CSV export when credentials are missing
+    
+    Returns:
+        Spreadsheet object or ReadOnlySpreadsheet wrapper for read-only mode
+    """
     spreadsheet_url = os.getenv("SPREADSHEET_URL")
     if not spreadsheet_url:
         raise ValueError("SPREADSHEET_URL environment variable is not set")
-
+    
+    # Check for credentials
+    creds_file = 'credentials.json' if os.path.exists('credentials.json') else '../credentials.json'
+    
+    if not os.path.exists(creds_file):
+        if read_only:
+            return ReadOnlySpreadsheet(spreadsheet_url)
+        else:
+            raise FileNotFoundError(f"Credentials file not found at {creds_file}. Use read_only=True for read-only access.")
+    
+    # Normal authenticated access
+    scope = ["https://spreadsheets.google.com/feeds", "https://www.googleapis.com/auth/drive"]
+    credentials = Credentials.from_service_account_file(creds_file, scopes=scope)
+    client = gspread.authorize(credentials)
     spreadsheet = client.open_by_url(spreadsheet_url)
     return spreadsheet
 
+class ReadOnlySpreadsheet:
+    """Read-only wrapper for Google Sheets using public CSV export"""
+    
+    def __init__(self, spreadsheet_url):
+        self.spreadsheet_url = spreadsheet_url
+        self.sheet_id = self._extract_sheet_id(spreadsheet_url)
+        
+    def _extract_sheet_id(self, url):
+        """Extract sheet ID from Google Sheets URL"""
+        match = re.search(r'/spreadsheets/d/([a-zA-Z0-9-_]+)', url)
+        if not match:
+            raise ValueError("Invalid Google Sheets URL")
+        return match.group(1)
+    
+    def worksheet(self, title):
+        """Return a read-only worksheet"""
+        return ReadOnlyWorksheet(self.sheet_id, title)
+
+class ReadOnlyWorksheet:
+    """Read-only worksheet that fetches data via CSV export"""
+    
+    def __init__(self, sheet_id, title):
+        self.sheet_id = sheet_id
+        self.title = title
+        
+    def get_all_records(self):
+        """Get all records from the worksheet as a list of dictionaries"""
+        try:
+            # URL encode the sheet title to handle spaces and special characters
+            import urllib.parse
+            encoded_title = urllib.parse.quote(self.title)
+            
+            # Map known sheet names to likely GID positions
+            # Based on the sheet order: Full Markets, All Markets, Volatility Markets, Selected Markets, Hyperparameters
+            sheet_gid_mapping = {
+                'Full Markets': 0,
+                'All Markets': 1, 
+                'Volatility Markets': 2,
+                'Selected Markets': 3,
+                'Hyperparameters': 4
+            }
+            
+            # Try multiple URL formats for accessing the sheet
+            urls_to_try = [
+                f"https://docs.google.com/spreadsheets/d/{self.sheet_id}/gviz/tq?tqx=out:csv&sheet={encoded_title}",
+                f"https://docs.google.com/spreadsheets/d/{self.sheet_id}/gviz/tq?tqx=out:csv&sheet={self.title}",
+            ]
+            
+            # Add GID-based URL if we know the likely position for this sheet
+            if self.title in sheet_gid_mapping:
+                gid = sheet_gid_mapping[self.title]
+                urls_to_try.append(f"https://docs.google.com/spreadsheets/d/{self.sheet_id}/export?format=csv&gid={gid}")
+            
+            # Also try a few common GID positions as fallback
+            for gid in [0, 1, 2, 3, 4]:
+                urls_to_try.append(f"https://docs.google.com/spreadsheets/d/{self.sheet_id}/export?format=csv&gid={gid}")
+            
+            for csv_url in urls_to_try:
+                try:
+                    print(f"Trying to fetch sheet '{self.title}' from: {csv_url}")
+                    response = requests.get(csv_url, timeout=30)
+                    response.raise_for_status()
+                    
+                    # Read CSV data into DataFrame
+                    from io import StringIO
+                    df = pd.read_csv(StringIO(response.text))
+                    
+                    # Check if we got meaningful data (not empty or error response)
+                    if not df.empty and len(df.columns) > 1:
+                        # For Hyperparameters sheet, verify it has the expected columns
+                        if self.title == 'Hyperparameters':
+                            expected_cols = ['type', 'param', 'value']
+                            if all(col in df.columns for col in expected_cols):
+                                print(f"Successfully fetched {len(df)} hyperparameter records")
+                                return df.to_dict('records')
+                            else:
+                                print(f"Sheet doesn't match Hyperparameters format. Columns: {list(df.columns)}")
+                                continue
+                        else:
+                            print(f"Successfully fetched {len(df)} records from sheet '{self.title}'")
+                            # Convert to list of dictionaries (same format as gspread)
+                            return df.to_dict('records')
+                    
+                except Exception as url_error:
+                    print(f"Failed with URL {csv_url}: {url_error}")
+                    continue
+            
+            print(f"All URL attempts failed for sheet '{self.title}'")
+            return []
+            
+        except Exception as e:
+            print(f"Warning: Could not fetch data from sheet '{self.title}': {e}")
+            return []
+    
+    def get_all_values(self):
+        """Get all values from the worksheet as a list of lists"""
+        try:
+            csv_url = f"https://docs.google.com/spreadsheets/d/{self.sheet_id}/gviz/tq?tqx=out:csv&sheet={self.title}"
+            response = requests.get(csv_url, timeout=30)
+            response.raise_for_status()
+            
+            # Read CSV and return as list of lists
+            from io import StringIO
+            df = pd.read_csv(StringIO(response.text))
+            
+            # Include headers and convert to list of lists
+            headers = [df.columns.tolist()]
+            data = df.values.tolist()
+            return headers + data
+            
+        except Exception as e:
+            print(f"Warning: Could not fetch data from sheet '{self.title}': {e}")
+            return []
+
 

requirements.txt

@@ -16,4 +16,5 @@ websockets==12.0
 cryptography==42.0.8
 asyncio
 requests
-google-auth
+google-auth
+web3==5.3.1

trading.py

@@ -31,9 +31,26 @@ def send_buy_order(order):
     """
     client = global_state.client
 
-    # Cancel existing orders for this token to avoid conflicts
-    if order['orders']['buy']['size'] > 0 or order['orders']['sell']['size'] > 0:
+    # Only cancel existing orders if we need to make significant changes
+    existing_buy_size = order['orders']['buy']['size']
+    existing_buy_price = order['orders']['buy']['price']
+    
+    # Cancel orders if price changed significantly or size needs major adjustment
+    price_diff = abs(existing_buy_price - order['price']) if existing_buy_price > 0 else float('inf')
+    size_diff = abs(existing_buy_size - order['size']) if existing_buy_size > 0 else float('inf')
+    
+    should_cancel = (
+        price_diff > 0.005 or  # Cancel if price diff > 0.5 cents
+        size_diff > order['size'] * 0.1 or  # Cancel if size diff > 10%
+        existing_buy_size == 0  # Cancel if no existing buy order
+    )
+    
+    if should_cancel and (existing_buy_size > 0 or order['orders']['sell']['size'] > 0):
+        print(f"Cancelling buy orders - price diff: {price_diff:.4f}, size diff: {size_diff:.1f}")
         client.cancel_all_asset(order['token'])
+    elif not should_cancel:
+        print(f"Keeping existing buy orders - minor changes: price diff: {price_diff:.4f}, size diff: {size_diff:.1f}")
+        return  # Don't place new order if existing one is fine
 
     # Calculate minimum acceptable price based on market spread
     incentive_start = order['mid_price'] - order['max_spread']/100
@@ -75,9 +92,26 @@ def send_sell_order(order):
     """
     client = global_state.client
 
-    # Cancel existing orders for this token to avoid conflicts
-    if order['orders']['buy']['size'] > 0 or order['orders']['sell']['size'] > 0:
+    # Only cancel existing orders if we need to make significant changes
+    existing_sell_size = order['orders']['sell']['size']
+    existing_sell_price = order['orders']['sell']['price']
+    
+    # Cancel orders if price changed significantly or size needs major adjustment
+    price_diff = abs(existing_sell_price - order['price']) if existing_sell_price > 0 else float('inf')
+    size_diff = abs(existing_sell_size - order['size']) if existing_sell_size > 0 else float('inf')
+    
+    should_cancel = (
+        price_diff > 0.005 or  # Cancel if price diff > 0.5 cents
+        size_diff > order['size'] * 0.1 or  # Cancel if size diff > 10%
+        existing_sell_size == 0  # Cancel if no existing sell order
+    )
+    
+    if should_cancel and (existing_sell_size > 0 or order['orders']['buy']['size'] > 0):
+        print(f"Cancelling sell orders - price diff: {price_diff:.4f}, size diff: {size_diff:.1f}")
         client.cancel_all_asset(order['token'])
+    elif not should_cancel:
+        print(f"Keeping existing sell orders - minor changes: price diff: {price_diff:.4f}, size diff: {size_diff:.1f}")
+        return  # Don't place new order if existing one is fine
 
     print(f'Creating new order for {order["size"]} at {order["price"]}')
     client.create_order(
@@ -161,6 +195,10 @@ async def perform_trade(market):
 
                 # Get market depth and price information
                 deets = get_best_bid_ask_deets(market, detail['name'], 100, 0.1)
+
+                #if deet has None for one these values below, call it with min size of 20
+                if deets['best_bid'] is None or deets['best_ask'] is None or deets['best_bid_size'] is None or deets['best_ask_size'] is None:
+                    deets = get_best_bid_ask_deets(market, detail['name'], 20, 0.1)
                 
                 # Extract all order book details
                 best_bid = deets['best_bid']
@@ -217,8 +255,15 @@ async def perform_trade(market):
                       f"avgPrice: {avgPrice}, Best Bid: {best_bid}, Best Ask: {best_ask}, "
                       f"Bid Price: {bid_price}, Ask Price: {ask_price}, Mid Price: {mid_price}")
 
+                # Get position for the opposite token to calculate total exposure
+                other_token = global_state.REVERSE_TOKENS[str(token)]
+                other_position = get_position(other_token)['size']
+                
                 # Calculate how much to buy or sell based on our position
-                buy_amount, sell_amount = get_buy_sell_amount(position, bid_price, row)
+                buy_amount, sell_amount = get_buy_sell_amount(position, bid_price, row, other_position)
+                
+                # Get max_size for logging (same logic as in get_buy_sell_amount)
+                max_size = row.get('max_size', row['trade_size'])
 
                 # Prepare order object with all necessary information
                 order = {
@@ -231,7 +276,8 @@ async def perform_trade(market):
                     'row': row
                 }
             
-                print(f"Position: {position}, Trade Size: {row['trade_size']}, "
+                print(f"Position: {position}, Other Position: {other_position}, "
+                      f"Trade Size: {row['trade_size']}, Max Size: {max_size}, "
                       f"buy_amount: {buy_amount}, sell_amount: {sell_amount}")
 
                 # File to store risk management information for this market
@@ -298,11 +344,14 @@ async def perform_trade(market):
                         continue
 
                 # ------- BUY ORDER LOGIC -------
+                # Get max_size, defaulting to trade_size if not specified
+                max_size = row.get('max_size', row['trade_size'])
+                
                 # Only buy if:
-                # 1. Position is less than 90% of target size
+                # 1. Position is less than max_size (new logic)
                 # 2. Position is less than absolute cap (250)
                 # 3. Buy amount is above minimum size
-                if position < 0.9 * row['trade_size'] and position < 250 and buy_amount > 0 and buy_amount >= row['min_size']:
+                if position < max_size and position < 250 and buy_amount > 0 and buy_amount >= row['min_size']:
                     # Get reference price from market data
                     sheet_value = row['best_bid']
 
@@ -366,8 +415,8 @@ async def perform_trade(market):
                                     print(f"Sending Buy Order for {token} because better price. "
                                           f"Orders look like this: {orders['buy']}. Best Bid: {best_bid}")
                                     send_buy_order(order)
-                                # 2. Current position + orders is not enough to reach target
-                                elif position + orders['buy']['size'] < 0.95 * row['trade_size']:
+                                # 2. Current position + orders is not enough to reach max_size
+                                elif position + orders['buy']['size'] < 0.95 * max_size:
                                     print(f"Sending Buy Order for {token} because not enough position + size")
                                     send_buy_order(order)
                                 # 3. Our current order is too large and needs to be resized
@@ -414,7 +463,7 @@ async def perform_trade(market):
                     #     send_sell_order(order)
 
         except Exception as ex:
-            print(f"Error performing trade for {market}")
+            print(f"Error performing trade for {market}: {ex}")
             traceback.print_exc()
 
         # Clean up memory and introduce a small delay

update_markets.py

@@ -88,8 +88,11 @@ def fetch_and_process_data():
 
 
     all_df = get_all_markets(client)
+    print("Got all Markets")
     all_results = get_all_results(all_df, client)
+    print("Got all Results")
     m_data, all_markets = get_markets(all_results, sel_df, maker_reward=0.75)
+    print("Got all orderbook")
 
     print(f'{pd.to_datetime("now")}: Fetched all markets data of length {len(all_markets)}.')
     new_df = add_volatility_to_df(all_markets)
@@ -124,7 +127,7 @@ if __name__ == "__main__":
     while True:
         try:
             fetch_and_process_data()
-            time.sleep(60 * 5)  # Sleep for 5 minutes (300 seconds)
+            time.sleep(60 * 60)  # Sleep for an hour
         except Exception as e:
             traceback.print_exc()
             print(str(e))