diff --git a/.env.example b/.env.example index 8f0f8b2..c0fbcc7 100644 --- a/.env.example +++ b/.env.example @@ -3,4 +3,5 @@ PK=your_private_key_here BROWSER_ADDRESS=your_wallet_address_here # Google Sheets (for data_updater) -SPREADSHEET_URL=your_google_spreadsheet_url_here \ No newline at end of file +SPREADSHEET_URL=https://docs.google.com/spreadsheets/d/1Kt6yGY7CZpB75cLJJAdWo7LSp9Oz7pjqfuVWwgtn7Ns/edit?gid=97507557#gid=97507557 +#replace with YOUR url diff --git a/README.md b/README.md index cbfc3af..a57e9bb 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ The repository consists of several interconnected modules: ## Requirements -- Python 3.8+ +- Python 3.9 with latest setuptools - Node.js (for poly_merger) - Google Sheets API credentials - Polymarket account and API credentials diff --git a/data_updater/google_utils.py b/data_updater/google_utils.py index 09d3de7..99562ac 100644 --- a/data_updater/google_utils.py +++ b/data_updater/google_utils.py @@ -1,13 +1,96 @@ from google.oauth2.service_account import Credentials import gspread import os +import pandas as pd +import requests +import re -def get_spreadsheet(): - """Get the main Google Spreadsheet using credentials and URL from environment variables""" +def get_spreadsheet(read_only=False): + """ + Get the main Google Spreadsheet using credentials and URL from environment variables + + Args: + read_only (bool): If True, uses public CSV export when credentials are missing + """ + spreadsheet_url = os.getenv("SPREADSHEET_URL") + if not spreadsheet_url: + raise ValueError("SPREADSHEET_URL environment variable is not set") + + # Check for credentials + if not os.path.exists('credentials.json'): + if read_only: + return ReadOnlySpreadsheet(spreadsheet_url) + else: + raise FileNotFoundError("credentials.json not found. Use read_only=True for read-only access.") + + # Normal authenticated access scope = ["https://spreadsheets.google.com/feeds", "https://www.googleapis.com/auth/drive"] - credentials = Credentials.from_service_account_file('credentials.json', scopes=scope) client = gspread.authorize(credentials) - spreadsheet = client.open_by_url(os.getenv("SPREADSHEET_URL")) - return spreadsheet \ No newline at end of file + spreadsheet = client.open_by_url(spreadsheet_url) + return spreadsheet + +class ReadOnlySpreadsheet: + """Read-only wrapper for Google Sheets using public CSV export""" + + def __init__(self, spreadsheet_url): + self.spreadsheet_url = spreadsheet_url + self.sheet_id = self._extract_sheet_id(spreadsheet_url) + + def _extract_sheet_id(self, url): + """Extract sheet ID from Google Sheets URL""" + match = re.search(r'/spreadsheets/d/([a-zA-Z0-9-_]+)', url) + if not match: + raise ValueError("Invalid Google Sheets URL") + return match.group(1) + + def worksheet(self, title): + """Return a read-only worksheet""" + return ReadOnlyWorksheet(self.sheet_id, title) + +class ReadOnlyWorksheet: + """Read-only worksheet that fetches data via CSV export""" + + def __init__(self, sheet_id, title): + self.sheet_id = sheet_id + self.title = title + + def get_all_records(self): + """Get all records from the worksheet as a list of dictionaries""" + try: + # Use the public CSV export URL + csv_url = f"https://docs.google.com/spreadsheets/d/{self.sheet_id}/gviz/tq?tqx=out:csv&sheet={self.title}" + response = requests.get(csv_url, timeout=30) + response.raise_for_status() + + # Read CSV data into DataFrame + from io import StringIO + df = pd.read_csv(StringIO(response.text)) + + # Convert to list of dictionaries (same format as gspread) + return df.to_dict('records') + + except Exception as e: + print(f"Warning: Could not fetch data from sheet '{self.title}': {e}") + return [] + + def get_all_values(self): + """Get all values from the worksheet as a list of lists""" + try: + csv_url = f"https://docs.google.com/spreadsheets/d/{self.sheet_id}/gviz/tq?tqx=out:csv&sheet={self.title}" + response = requests.get(csv_url, timeout=30) + response.raise_for_status() + + # Read CSV and return as list of lists + from io import StringIO + df = pd.read_csv(StringIO(response.text)) + + # Include headers and convert to list of lists + headers = [df.columns.tolist()] + data = df.values.tolist() + return headers + data + + except Exception as e: + print(f"Warning: Could not fetch data from sheet '{self.title}': {e}") + return [] \ No newline at end of file diff --git a/poly_data/utils.py b/poly_data/utils.py index a0a059a..3337c09 100644 --- a/poly_data/utils.py +++ b/poly_data/utils.py @@ -1,16 +1,33 @@ import json from poly_utils.google_utils import get_spreadsheet import pandas as pd +import os def pretty_print(txt, dic): print("\n", txt, json.dumps(dic, indent=4)) -def get_sheet_df(): - +def get_sheet_df(read_only=None): + """ + Get sheet data with optional read-only mode + + Args: + read_only (bool): If None, auto-detects based on credentials availability + """ all = 'All Markets' sel = 'Selected Markets' - spreadsheet = get_spreadsheet() + # Auto-detect read-only mode if not specified + if read_only is None: + creds_file = 'credentials.json' if os.path.exists('credentials.json') else '../credentials.json' + read_only = not os.path.exists(creds_file) + if read_only: + print("No credentials found, using read-only mode") + + try: + spreadsheet = get_spreadsheet(read_only=read_only) + except FileNotFoundError: + print("No credentials found, falling back to read-only mode") + spreadsheet = get_spreadsheet(read_only=True) wk = spreadsheet.worksheet(sel) df = pd.DataFrame(wk.get_all_records()) @@ -20,7 +37,6 @@ def get_sheet_df(): df2 = pd.DataFrame(wk2.get_all_records()) df2 = df2[df2['question'] != ""].reset_index(drop=True) - result = df.merge(df2, on='question', how='inner') wk_p = spreadsheet.worksheet('Hyperparameters') diff --git a/poly_utils/google_utils.py b/poly_utils/google_utils.py index 4ea1955..5020792 100644 --- a/poly_utils/google_utils.py +++ b/poly_utils/google_utils.py @@ -1,21 +1,105 @@ from google.oauth2.service_account import Credentials import gspread import os +import pandas as pd +import requests +import re from dotenv import load_dotenv load_dotenv() -def get_spreadsheet(): - scope = ["https://spreadsheets.google.com/feeds", "https://www.googleapis.com/auth/drive"] - - creds_file = 'credentials.json' if os.path.exists('credentials.json') else '../credentials.json' - credentials = Credentials.from_service_account_file(creds_file, scopes=scope) - client = gspread.authorize(credentials) +def get_spreadsheet(read_only=False): + """ + Get Google Spreadsheet with optional read-only mode. + + Args: + read_only (bool): If True, uses public CSV export when credentials are missing + + Returns: + Spreadsheet object or ReadOnlySpreadsheet wrapper for read-only mode + """ spreadsheet_url = os.getenv("SPREADSHEET_URL") if not spreadsheet_url: raise ValueError("SPREADSHEET_URL environment variable is not set") - + + # Check for credentials + creds_file = 'credentials.json' if os.path.exists('credentials.json') else '../credentials.json' + + if not os.path.exists(creds_file): + if read_only: + return ReadOnlySpreadsheet(spreadsheet_url) + else: + raise FileNotFoundError(f"Credentials file not found at {creds_file}. Use read_only=True for read-only access.") + + # Normal authenticated access + scope = ["https://spreadsheets.google.com/feeds", "https://www.googleapis.com/auth/drive"] + credentials = Credentials.from_service_account_file(creds_file, scopes=scope) + client = gspread.authorize(credentials) spreadsheet = client.open_by_url(spreadsheet_url) return spreadsheet +class ReadOnlySpreadsheet: + """Read-only wrapper for Google Sheets using public CSV export""" + + def __init__(self, spreadsheet_url): + self.spreadsheet_url = spreadsheet_url + self.sheet_id = self._extract_sheet_id(spreadsheet_url) + + def _extract_sheet_id(self, url): + """Extract sheet ID from Google Sheets URL""" + match = re.search(r'/spreadsheets/d/([a-zA-Z0-9-_]+)', url) + if not match: + raise ValueError("Invalid Google Sheets URL") + return match.group(1) + + def worksheet(self, title): + """Return a read-only worksheet""" + return ReadOnlyWorksheet(self.sheet_id, title) + +class ReadOnlyWorksheet: + """Read-only worksheet that fetches data via CSV export""" + + def __init__(self, sheet_id, title): + self.sheet_id = sheet_id + self.title = title + + def get_all_records(self): + """Get all records from the worksheet as a list of dictionaries""" + try: + # Use the public CSV export URL + csv_url = f"https://docs.google.com/spreadsheets/d/{self.sheet_id}/gviz/tq?tqx=out:csv&sheet={self.title}" + response = requests.get(csv_url, timeout=30) + response.raise_for_status() + + # Read CSV data into DataFrame + from io import StringIO + df = pd.read_csv(StringIO(response.text)) + + # Convert to list of dictionaries (same format as gspread) + return df.to_dict('records') + + except Exception as e: + print(f"Warning: Could not fetch data from sheet '{self.title}': {e}") + return [] + + def get_all_values(self): + """Get all values from the worksheet as a list of lists""" + try: + csv_url = f"https://docs.google.com/spreadsheets/d/{self.sheet_id}/gviz/tq?tqx=out:csv&sheet={self.title}" + response = requests.get(csv_url, timeout=30) + response.raise_for_status() + + # Read CSV and return as list of lists + from io import StringIO + df = pd.read_csv(StringIO(response.text)) + + # Include headers and convert to list of lists + headers = [df.columns.tolist()] + data = df.values.tolist() + return headers + data + + except Exception as e: + print(f"Warning: Could not fetch data from sheet '{self.title}': {e}") + return [] + diff --git a/requirements.txt b/requirements.txt index 0f227fa..854dc6d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,4 +16,5 @@ websockets==12.0 cryptography==42.0.8 asyncio requests -google-auth \ No newline at end of file +google-auth +web3==5.3.1 \ No newline at end of file