mirror of
https://github.com/firmai/financial-machine-learning.git
synced 2026-08-17 04:48:08 +00:00
refactor to split otu search and status run to reduce api hit
This commit is contained in:
Bin Yang
+282
@@ -0,0 +1,282 @@
|
||||
import datetime
|
||||
import os
|
||||
from typing import Dict, List
|
||||
|
||||
import pandas as pd
|
||||
|
||||
from conf import PROJECT_ROOT_DIR
|
||||
from git_status import get_repo_list
|
||||
from git_util import get_github_client, get_repo_attributes_dict
|
||||
|
||||
|
||||
def convert_repo_list_to_df(repo_list, category):
|
||||
df_list = []
|
||||
for repo in repo_list:
|
||||
attr_dict = get_repo_attributes_dict(repo)
|
||||
attr_dict['name'] = repo.name
|
||||
attr_dict['comment'] = 'NEW'
|
||||
attr_dict['category'] = category
|
||||
attr_dict['repo_path'] = repo.full_name
|
||||
attr_dict['url'] = 'https://github.com/{}'.format(repo.full_name)
|
||||
df_list.append(attr_dict)
|
||||
result_df = pd.DataFrame(df_list)
|
||||
return result_df
|
||||
|
||||
|
||||
# generic search functions
|
||||
def search_repo(search_term: str, qualifier_dict: Dict):
|
||||
g = get_github_client()
|
||||
qualifier_str = ' '.join(['{}:{}'.format(k, v) for k, v in iter(qualifier_dict.items())])
|
||||
if qualifier_str != '':
|
||||
final_search_term = '{} {}'.format(search_term, qualifier_str)
|
||||
else:
|
||||
final_search_term = search_term
|
||||
repo_result = g.search_repositories(final_search_term)
|
||||
return repo_result
|
||||
|
||||
|
||||
def search_repo_multiple_terms(term_list: List[str],
|
||||
category: str,
|
||||
min_stars_number: int = None,
|
||||
created_at: str = None,
|
||||
pushed_date: str = None,
|
||||
drop_duplicate: bool = True
|
||||
):
|
||||
"""
|
||||
|
||||
:param term_list:
|
||||
:param category:
|
||||
:param min_stars_number:
|
||||
:param created_at:
|
||||
:param pushed_date:
|
||||
:param drop_duplicate:
|
||||
:return:
|
||||
usage:
|
||||
>>> term_list = ['deep learning trading', 'deep learning finance', 'reinforcement learning trading',
|
||||
'reinforcement learning finance']
|
||||
>>> category = 'Deep Learning And Reinforcement Learning'
|
||||
>>> min_stars_number = 100
|
||||
>>> created_at = None
|
||||
>>> pushed_date = None
|
||||
>>> drop_duplicate = True
|
||||
"""
|
||||
repo_df_list = []
|
||||
for search_term in term_list:
|
||||
repo_list = search_repo_simple(search_term, min_stars_number, created_at=created_at, pushed_date=pushed_date)
|
||||
repo_df = convert_repo_list_to_df(repo_list, category)
|
||||
repo_df_list.append(repo_df)
|
||||
combined_df = pd.concat(repo_df_list).reset_index(drop=True)
|
||||
if drop_duplicate:
|
||||
combined_df = combined_df.drop_duplicates()
|
||||
combined_df['finml_added_date'] = datetime.datetime.now()
|
||||
return combined_df
|
||||
|
||||
|
||||
def search_repo_simple(search_term: str = None,
|
||||
min_stars_number: int = None,
|
||||
created_at: str = None,
|
||||
pushed_date: str = None
|
||||
):
|
||||
"""
|
||||
|
||||
:param search_term:
|
||||
:param min_stars_number:
|
||||
:param created_at:
|
||||
:param pushed_date:
|
||||
usage:
|
||||
>>> search_term = 'machine learning trading'
|
||||
>>> min_stars_number = 100
|
||||
>>> created_at = None
|
||||
>>> pushed_date = None
|
||||
"""
|
||||
if search_term is None:
|
||||
_search_term = ''
|
||||
else:
|
||||
_search_term = search_term
|
||||
|
||||
qualifier_dict = {}
|
||||
if min_stars_number is not None:
|
||||
qualifier_dict['stars'] = '>={}'.format(min_stars_number)
|
||||
|
||||
if created_at is not None:
|
||||
qualifier_dict['created'] = '>={}'.format(created_at)
|
||||
|
||||
if pushed_date is not None:
|
||||
qualifier_dict['pushed'] = '>={}'.format(pushed_date)
|
||||
search_result = search_repo(_search_term, qualifier_dict)
|
||||
return search_result
|
||||
|
||||
|
||||
def search_new_repo_by_category(category: str,
|
||||
min_stars_number: int = 100,
|
||||
existing_repo_df: pd.DataFrame = None):
|
||||
"""
|
||||
|
||||
:param category:
|
||||
:param min_stars_number:
|
||||
:param existing_repo_df:
|
||||
:return:
|
||||
usage:
|
||||
>>> category = 'Data Processing Techniques and Transformations'
|
||||
>>> min_stars_number = 100
|
||||
>>> existing_repo_df = get_repo_list()
|
||||
"""
|
||||
print('*** searching for category [{}] ***'.format(category))
|
||||
combined_df = None
|
||||
if category == 'Deep Learning And Reinforcement Learning':
|
||||
combined_df = search_repo_multiple_terms(['deep learning trading',
|
||||
'deep learning finance',
|
||||
'reinforcement learning trading',
|
||||
'reinforcement learning finance'],
|
||||
category,
|
||||
min_stars_number=min_stars_number
|
||||
)
|
||||
|
||||
elif category == 'Other Models':
|
||||
combined_df = search_repo_multiple_terms(['machine learning trading',
|
||||
'machine learning finance'],
|
||||
category,
|
||||
min_stars_number=min_stars_number
|
||||
)
|
||||
elif category == 'Data Processing Techniques and Transformations':
|
||||
combined_df = search_repo_multiple_terms(['data transformation trading',
|
||||
'data transformation finance',
|
||||
'data transformation time series',
|
||||
'data processing trading',
|
||||
'data processing finance',
|
||||
'power transform trading',
|
||||
'power transform finance',
|
||||
'standardization normalization trading',
|
||||
'standardization normalization finance',
|
||||
],
|
||||
category,
|
||||
min_stars_number=int(min_stars_number * 0.5)
|
||||
)
|
||||
elif category == 'Portfolio Selection and Optimisation':
|
||||
combined_df = search_repo_multiple_terms(['portfolio optimization machine learning finance',
|
||||
'portfolio optimization machine learning trading',
|
||||
'portfolio construction machine learning finance',
|
||||
'portfolio construction machine learning trading',
|
||||
'portfolio optimization finance',
|
||||
'portfolio optimization trading',
|
||||
'portfolio construction finance',
|
||||
'portfolio construction trading'
|
||||
],
|
||||
category,
|
||||
min_stars_number=min_stars_number
|
||||
)
|
||||
elif category == 'Factor and Risk Analysis':
|
||||
combined_df = search_repo_multiple_terms(['risk factor finance',
|
||||
'risk factor trading',
|
||||
'risk premia factor finance',
|
||||
'risk premia factor trading',
|
||||
'style factor finance',
|
||||
'style factor trading',
|
||||
'macro factor finance',
|
||||
'macro factor trading',
|
||||
],
|
||||
category,
|
||||
min_stars_number=int(min_stars_number * 0.05)
|
||||
)
|
||||
elif category == 'Unsupervised':
|
||||
combined_df = search_repo_multiple_terms(['unsupervised finance',
|
||||
'unsupervised trading'
|
||||
],
|
||||
category,
|
||||
min_stars_number=int(min_stars_number * 0.1)
|
||||
)
|
||||
elif category == 'Textual':
|
||||
combined_df = search_repo_multiple_terms(['NLP finance',
|
||||
'NLP trading'
|
||||
],
|
||||
category,
|
||||
min_stars_number=min_stars_number
|
||||
)
|
||||
elif category == 'Derivatives and Hedging':
|
||||
combined_df = search_repo_multiple_terms(['derivatives finance',
|
||||
'derivatives trading',
|
||||
'quantlib trading',
|
||||
'quantlib finance',
|
||||
'hedging finance',
|
||||
'hedging trading',
|
||||
'option trading',
|
||||
'option finance',
|
||||
'delta hedge trading',
|
||||
'delta hedge finance'
|
||||
],
|
||||
category,
|
||||
min_stars_number=min_stars_number
|
||||
)
|
||||
elif category == 'Fixed Income':
|
||||
combined_df = search_repo_multiple_terms(['corporate bond finance',
|
||||
'corporate bond trading',
|
||||
'muni bond trading',
|
||||
'muni bond finance',
|
||||
'investment grade finance',
|
||||
'investment grade trading',
|
||||
'high yield trading',
|
||||
'high yield finance',
|
||||
'credit rating trading',
|
||||
'credit rating finance',
|
||||
'fixed income trading',
|
||||
'fixed income finance',
|
||||
'corporate bond',
|
||||
'muni bond',
|
||||
'credit rating'
|
||||
],
|
||||
category,
|
||||
min_stars_number=int(min_stars_number * 0.2)
|
||||
)
|
||||
elif category == 'Alternative Finance':
|
||||
# don't include crypto here as it will skew the results, consider putting it as a seperate category
|
||||
combined_df = search_repo_multiple_terms(['private equity',
|
||||
'venture capital',
|
||||
'real estate trading',
|
||||
'real estate finance',
|
||||
'alternative asset trading',
|
||||
'alternative asset finance',
|
||||
'commodity trading',
|
||||
'commodity finance',
|
||||
'farmland finance',
|
||||
'farmland trading'
|
||||
],
|
||||
category,
|
||||
min_stars_number=int(min_stars_number * 0.5)
|
||||
)
|
||||
|
||||
# only find ones that need to be inserted
|
||||
if combined_df is not None and not combined_df.empty and existing_repo_df is not None:
|
||||
combined_df = combined_df[
|
||||
~combined_df['repo_path'].str.lower().isin(existing_repo_df['repo_path'].dropna().str.lower())]
|
||||
|
||||
return combined_df
|
||||
|
||||
|
||||
def search_new_repo_and_append(min_stars_number: int = 100, filter_list=None):
|
||||
"""
|
||||
|
||||
:param min_stars_number:
|
||||
:param filter_list:
|
||||
"""
|
||||
repo_df = get_repo_list()
|
||||
category_list = repo_df['category'].unique().tolist()
|
||||
if filter_list is not None:
|
||||
category_list = [x for x in category_list if x in filter_list]
|
||||
|
||||
new_repo_list = []
|
||||
for category in category_list:
|
||||
combined_df = search_new_repo_by_category(category, min_stars_number, repo_df)
|
||||
if combined_df is not None:
|
||||
new_repo_list.append(combined_df)
|
||||
new_repo_df = pd.concat(new_repo_list).reset_index(drop=True)
|
||||
# drop duplicate regardless of the category, keep first one for now
|
||||
new_repo_df = new_repo_df.drop_duplicates(subset='repo_path')
|
||||
|
||||
final_df = pd.concat([repo_df, new_repo_df]).reset_index(drop=True)
|
||||
|
||||
final_df = final_df.sort_values(by='category')
|
||||
final_df.to_csv(os.path.join(PROJECT_ROOT_DIR, 'raw_data', 'url_list.csv'), index=False)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
search_new_repo_and_append(min_stars_number=100)
|
||||
Reference in New Issue
Block a user