warproxxx
339 lines
10 KiB
Python
339 lines
10 KiB
Python
import pandas as pd
|
|
import numpy as np
|
|
import os
|
|
import requests
|
|
import time
|
|
import warnings
|
|
warnings.filterwarnings("ignore")
|
|
|
|
|
|
if not os.path.exists('data'):
|
|
os.makedirs('data')
|
|
|
|
def get_sel_df(spreadsheet, sheet_name='Selected Markets'):
|
|
try:
|
|
wk2 = spreadsheet.worksheet(sheet_name)
|
|
sel_df = pd.DataFrame(wk2.get_all_records())
|
|
sel_df = sel_df[sel_df['question'] != ""].reset_index(drop=True)
|
|
return sel_df
|
|
except:
|
|
return pd.DataFrame()
|
|
|
|
def get_all_markets(client):
|
|
cursor = ""
|
|
all_markets = []
|
|
|
|
while True:
|
|
try:
|
|
markets = client.get_sampling_markets(next_cursor = cursor)
|
|
markets_df = pd.DataFrame(markets['data'])
|
|
|
|
|
|
cursor = markets['next_cursor']
|
|
|
|
|
|
|
|
all_markets.append(markets_df)
|
|
|
|
if cursor is None:
|
|
break
|
|
except:
|
|
break
|
|
|
|
all_df = pd.concat(all_markets)
|
|
all_df = all_df.reset_index(drop=True)
|
|
|
|
return all_df
|
|
|
|
def get_bid_ask_range(ret, TICK_SIZE):
|
|
bid_from = ret['midpoint'] - ret['max_spread'] / 100
|
|
bid_to = ret['best_ask'] #Although bid to this high up will change bid_from because of changing midpoint, take optimistic approach
|
|
|
|
if bid_to == 0:
|
|
bid_to = ret['midpoint']
|
|
|
|
if bid_to - TICK_SIZE > ret['midpoint']:
|
|
bid_to = ret['best_bid'] + (TICK_SIZE + 0.1 * TICK_SIZE)
|
|
|
|
if bid_from > bid_to:
|
|
bid_from = bid_to - (TICK_SIZE + 0.1 * TICK_SIZE)
|
|
|
|
ask_to = ret['midpoint'] + ret['max_spread'] / 100
|
|
ask_from = ret['best_bid']
|
|
|
|
if ask_from == 0:
|
|
ask_from = ret['midpoint']
|
|
|
|
if ask_from + TICK_SIZE < ret['midpoint']:
|
|
ask_from = ret['best_ask'] - (TICK_SIZE + 0.1 * TICK_SIZE)
|
|
|
|
if ask_from > ask_to:
|
|
ask_to = ask_from + (TICK_SIZE + 0.1 * TICK_SIZE)
|
|
|
|
bid_from = round(bid_from, 3)
|
|
bid_to = round(bid_to, 3)
|
|
ask_from = round(ask_from, 3)
|
|
ask_to = round(ask_to, 3)
|
|
|
|
if bid_from < 0:
|
|
bid_from = 0
|
|
|
|
if ask_from < 0:
|
|
ask_from = 0
|
|
|
|
return bid_from, bid_to, ask_from, ask_to
|
|
|
|
|
|
def generate_numbers(start, end, TICK_SIZE):
|
|
# Calculate the starting point, rounding up to the next hundredth if not an exact multiple of TICK_SIZE
|
|
rounded_start = (int(start * 100) + 1) / 100 if start * 100 % 1 != 0 else start + TICK_SIZE
|
|
|
|
# Calculate the ending point, rounding down to the nearest hundredth
|
|
rounded_end = int(end * 100) / 100
|
|
|
|
# Generate numbers from rounded_start to rounded_end, ensuring they fall strictly within the original bounds
|
|
numbers = []
|
|
current = rounded_start
|
|
while current < end:
|
|
numbers.append(current)
|
|
current += TICK_SIZE
|
|
current = round(current, len(str(TICK_SIZE).split('.')[1])) # Rounding to avoid floating point imprecision
|
|
|
|
return numbers
|
|
|
|
def add_formula_params(curr_df, midpoint, v, daily_reward):
|
|
curr_df['s'] = (curr_df['price'] - midpoint).abs()
|
|
curr_df['S'] = ((v - curr_df['s']) / v) ** 2
|
|
curr_df['100'] = 1/curr_df['price'] * 100
|
|
|
|
curr_df['size'] = curr_df['size'] + curr_df['100']
|
|
|
|
curr_df['Q'] = curr_df['S'] * curr_df['size']
|
|
curr_df['reward_per_100'] = (curr_df['Q'] / curr_df['Q'].sum()) * daily_reward / 2 / curr_df['size'] * curr_df['100']
|
|
return curr_df
|
|
|
|
def process_single_row(row, client):
|
|
ret = {}
|
|
ret['question'] = row['question']
|
|
ret['neg_risk'] = row['neg_risk']
|
|
|
|
ret['answer1'] = row['tokens'][0]['outcome']
|
|
ret['answer2'] = row['tokens'][1]['outcome']
|
|
|
|
ret['min_size'] = row['rewards']['min_size']
|
|
ret['max_spread'] = row['rewards']['max_spread']
|
|
|
|
token1 = row['tokens'][0]['token_id']
|
|
token2 = row['tokens'][1]['token_id']
|
|
|
|
rate = 0
|
|
for rate_info in row['rewards']['rates']:
|
|
if rate_info['asset_address'].lower() == '0x2791Bca1f2de4661ED88A30C99A7a9449Aa84174'.lower():
|
|
rate = rate_info['rewards_daily_rate']
|
|
break
|
|
|
|
ret['rewards_daily_rate'] = rate
|
|
book = client.get_order_book(token1)
|
|
|
|
bids = pd.DataFrame()
|
|
asks = pd.DataFrame()
|
|
|
|
try:
|
|
bids = pd.DataFrame(book.bids).astype(float)
|
|
except:
|
|
pass
|
|
|
|
try:
|
|
asks = pd.DataFrame(book.asks).astype(float)
|
|
except:
|
|
pass
|
|
|
|
|
|
try:
|
|
ret['best_bid'] = bids.iloc[-1]['price']
|
|
except:
|
|
ret['best_bid'] = 0
|
|
|
|
try:
|
|
ret['best_ask'] = asks.iloc[-1]['price']
|
|
except:
|
|
ret['best_ask'] = 0
|
|
|
|
ret['midpoint'] = (ret['best_bid'] + ret['best_ask']) / 2
|
|
|
|
TICK_SIZE = row['minimum_tick_size']
|
|
ret['tick_size'] = TICK_SIZE
|
|
|
|
bid_from, bid_to, ask_from, ask_to = get_bid_ask_range(ret, TICK_SIZE)
|
|
v = round((ret['max_spread'] / 100), 2)
|
|
|
|
bids_df = pd.DataFrame()
|
|
bids_df['price'] = generate_numbers(bid_from, bid_to, TICK_SIZE)
|
|
|
|
asks_df = pd.DataFrame()
|
|
asks_df['price'] = generate_numbers(ask_from, ask_to, TICK_SIZE)
|
|
|
|
try:
|
|
bids_df = bids_df.merge(bids, on='price', how='left').fillna(0)
|
|
except:
|
|
bids_df = pd.DataFrame()
|
|
|
|
try:
|
|
asks_df = asks_df.merge(asks, on='price', how='left').fillna(0)
|
|
except:
|
|
asks_df = pd.DataFrame()
|
|
|
|
best_bid_reward = 0
|
|
ret_bid = pd.DataFrame()
|
|
|
|
try:
|
|
ret_bid = add_formula_params(bids_df, ret['midpoint'], v, rate)
|
|
best_bid_reward = round(ret_bid['reward_per_100'].max(), 2)
|
|
except:
|
|
pass
|
|
|
|
best_ask_reward = 0
|
|
ret_ask = pd.DataFrame()
|
|
|
|
try:
|
|
ret_ask = add_formula_params(asks_df, ret['midpoint'], v, rate)
|
|
best_ask_reward = round(ret_ask['reward_per_100'].max(), 2)
|
|
except:
|
|
pass
|
|
|
|
ret['bid_reward_per_100'] = best_bid_reward
|
|
ret['ask_reward_per_100'] = best_ask_reward
|
|
|
|
ret['sm_reward_per_100'] = round((best_bid_reward + best_ask_reward) / 2, 2)
|
|
ret['gm_reward_per_100'] = round((best_bid_reward * best_ask_reward) ** 0.5, 2)
|
|
|
|
ret['end_date_iso'] = row['end_date_iso']
|
|
ret['market_slug'] = row['market_slug']
|
|
ret['token1'] = token1
|
|
ret['token2'] = token2
|
|
ret['condition_id'] = row['condition_id']
|
|
|
|
return ret
|
|
|
|
|
|
def get_all_results(all_df, client, max_workers=5):
|
|
all_results = []
|
|
|
|
def process_with_progress(args):
|
|
idx, row = args
|
|
try:
|
|
return process_single_row(row, client)
|
|
except:
|
|
print("error fetching market")
|
|
return None
|
|
|
|
with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
|
|
futures = [executor.submit(process_with_progress, (idx, row)) for idx, row in all_df.iterrows()]
|
|
|
|
for future in concurrent.futures.as_completed(futures):
|
|
result = future.result()
|
|
if result is not None:
|
|
all_results.append(result)
|
|
|
|
if len(all_results) % (max_workers * 2) == 0:
|
|
print(f'{len(all_results)} of {len(all_df)}')
|
|
|
|
return all_results
|
|
|
|
def get_combined_markets(new_df, new_markets, sel_df):
|
|
|
|
if len(sel_df) > 0:
|
|
old_markets = new_df[new_df['question'].isin(sel_df['question'])]
|
|
all_markets = pd.concat([old_markets, new_markets])
|
|
else:
|
|
all_markets = new_markets
|
|
|
|
all_markets = all_markets.drop_duplicates('question')
|
|
|
|
all_markets = all_markets.sort_values('gm_reward_per_100', ascending=False)
|
|
return all_markets
|
|
|
|
import concurrent.futures
|
|
|
|
def calculate_annualized_volatility(df, hours):
|
|
end_time = df['t'].max()
|
|
start_time = end_time - pd.Timedelta(hours=hours)
|
|
window_df = df[df['t'] >= start_time]
|
|
volatility = window_df['log_return'].std()
|
|
annualized_volatility = volatility * np.sqrt(60 * 24 * 252)
|
|
return round(annualized_volatility, 2)
|
|
|
|
def add_volatility(row):
|
|
res = requests.get(f'https://clob.polymarket.com/prices-history?interval=1m&market={row["token1"]}&fidelity=10')
|
|
price_df = pd.DataFrame(res.json()['history'])
|
|
price_df['t'] = pd.to_datetime(price_df['t'], unit='s')
|
|
price_df['p'] = price_df['p'].round(2)
|
|
|
|
price_df.to_csv(f'data/{row["token1"]}.csv', index=False)
|
|
|
|
price_df['log_return'] = np.log(price_df['p'] / price_df['p'].shift(1))
|
|
|
|
row_dict = row.copy()
|
|
|
|
stats = {
|
|
'1_hour': calculate_annualized_volatility(price_df, 1),
|
|
'3_hour': calculate_annualized_volatility(price_df, 3),
|
|
'6_hour': calculate_annualized_volatility(price_df, 6),
|
|
'12_hour': calculate_annualized_volatility(price_df, 12),
|
|
'24_hour': calculate_annualized_volatility(price_df, 24),
|
|
'7_day': calculate_annualized_volatility(price_df, 24 * 7),
|
|
'14_day': calculate_annualized_volatility(price_df, 24 * 14),
|
|
'30_day': calculate_annualized_volatility(price_df, 24 * 30),
|
|
'volatility_price': price_df['p'].iloc[-1]
|
|
}
|
|
|
|
new_dict = {**row_dict, **stats}
|
|
return new_dict
|
|
|
|
def add_volatility_to_df(df, max_workers=2):
|
|
|
|
results = []
|
|
df = df.reset_index(drop=True)
|
|
|
|
def process_volatility_with_progress(args):
|
|
idx, row = args
|
|
try:
|
|
ret = add_volatility(row.to_dict())
|
|
return ret
|
|
except:
|
|
print("Error fetching volatility")
|
|
return None
|
|
|
|
with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
|
|
futures = [executor.submit(process_volatility_with_progress, (idx, row)) for idx, row in df.iterrows()]
|
|
|
|
for future in concurrent.futures.as_completed(futures):
|
|
result = future.result()
|
|
if result is not None:
|
|
results.append(result)
|
|
|
|
if len(results) % (max_workers * 2) == 0:
|
|
print(f'{len(results)} of {len(df)}')
|
|
|
|
return pd.DataFrame(results)
|
|
|
|
|
|
def get_markets(all_results, sel_df, maker_reward=1):
|
|
new_df = pd.DataFrame(all_results)
|
|
new_df['spread'] = abs(new_df['best_ask'] - new_df['best_bid'])
|
|
new_df = new_df.sort_values('rewards_daily_rate', ascending=False)
|
|
new_df[' '] = ''
|
|
|
|
new_df = new_df[['question', 'answer1', 'answer2', 'neg_risk', 'spread', 'best_bid', 'best_ask', 'rewards_daily_rate', 'bid_reward_per_100', 'ask_reward_per_100', 'gm_reward_per_100', 'sm_reward_per_100', 'min_size', 'max_spread', 'tick_size', 'market_slug', 'token1', 'token2', 'condition_id']]
|
|
new_df = new_df.replace([np.inf, -np.inf], 0)
|
|
all_data = new_df.copy()
|
|
s_df = new_df.copy()
|
|
|
|
|
|
making_markets = s_df[~new_df['question'].isin(sel_df['question'])]
|
|
making_markets = making_markets.sort_values('gm_reward_per_100', ascending=False)
|
|
making_markets = making_markets[making_markets['gm_reward_per_100'] >= maker_reward]
|
|
all_markets = get_combined_markets(new_df, making_markets, sel_df)
|
|
|
|
return all_data, all_markets
|