gavindiaz/financial-machine-learning
1
0
Fork 0
mirror of https://github.com/firmai/financial-machine-learning.git synced 2026-07-27 18:57:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor to split otu search and status run to reduce api hit

Browse Source
This commit is contained in:
Bin Yang
2021-04-13 11:01:47 -04:00
parent 0c50c4ef63
commit 90b8cb9b2d
6 changed files with 391 additions and 338 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/repo_search.yml
+38
View File
@@ -0,0 +1,38 @@
name: Repo-Search
on:
schedule:
- cron: '0 0 * * *'
jobs:
# Set the job key. The key is displayed as the job name
update-repo-status:
# Name the Job
name: Update repo status for all saved repo
# Set the type of machine to run on
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v2
- name: setup python
uses: actions/setup-python@v2
with:
python-version: 3.7
- name: Install dependencies
run: |
python -m pip install --upgrade pip
if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
- name: execute py script # run the run.py to get the latest data
run: |
python git_search.py
env:
GIT_TOKEN: ${{ secrets.GIT_TOKEN }}
- name: Commit & Push changes
uses: actions-js/push@master
with:
github_token: ${{ secrets.GIT_TOKEN }}
.github/workflows/repo_status.yml
+1 -1
View File
@@ -2,7 +2,7 @@ name: Repo-Updater
on:
schedule:
- cron: '0 0 * * *' # daily
- cron: '0 0 * * 0' # weekly
jobs:
.github/workflows/wiki_gen.yml
+5 -5
View File
@@ -25,11 +25,11 @@ jobs:
python -m pip install --upgrade pip
if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
# - name: execute status update script # run the run.py to get the latest data
# run: |
# python git_status.py
# env:
# GIT_TOKEN: ${{ secrets.GIT_TOKEN }}
- name: execute status update script # run the run.py to get the latest data
run: |
python git_search.py
env:
GIT_TOKEN: ${{ secrets.GIT_TOKEN }}
- name: execute wiki generation script # run the wiki_gen
run: |
git_search.py
+282
View File
@@ -0,0 +1,282 @@
import datetime
import os
from typing import Dict, List
import pandas as pd
from conf import PROJECT_ROOT_DIR
from git_status import get_repo_list
from git_util import get_github_client, get_repo_attributes_dict
def convert_repo_list_to_df(repo_list, category):
df_list = []
for repo in repo_list:
attr_dict = get_repo_attributes_dict(repo)
attr_dict['name'] = repo.name
attr_dict['comment'] = 'NEW'
attr_dict['category'] = category
attr_dict['repo_path'] = repo.full_name
attr_dict['url'] = 'https://github.com/{}'.format(repo.full_name)
df_list.append(attr_dict)
result_df = pd.DataFrame(df_list)
return result_df
# generic search functions
def search_repo(search_term: str, qualifier_dict: Dict):
g = get_github_client()
qualifier_str = ' '.join(['{}:{}'.format(k, v) for k, v in iter(qualifier_dict.items())])
if qualifier_str != '':
final_search_term = '{} {}'.format(search_term, qualifier_str)
else:
final_search_term = search_term
repo_result = g.search_repositories(final_search_term)
return repo_result
def search_repo_multiple_terms(term_list: List[str],
category: str,
min_stars_number: int = None,
created_at: str = None,
pushed_date: str = None,
drop_duplicate: bool = True
):
"""
:param term_list:
:param category:
:param min_stars_number:
:param created_at:
:param pushed_date:
:param drop_duplicate:
:return:
usage:
>>> term_list = ['deep learning trading', 'deep learning finance', 'reinforcement learning trading',
'reinforcement learning finance']
>>> category = 'Deep Learning And Reinforcement Learning'
>>> min_stars_number = 100
>>> created_at = None
>>> pushed_date = None
>>> drop_duplicate = True
"""
repo_df_list = []
for search_term in term_list:
repo_list = search_repo_simple(search_term, min_stars_number, created_at=created_at, pushed_date=pushed_date)
repo_df = convert_repo_list_to_df(repo_list, category)
repo_df_list.append(repo_df)
combined_df = pd.concat(repo_df_list).reset_index(drop=True)
if drop_duplicate:
combined_df = combined_df.drop_duplicates()
combined_df['finml_added_date'] = datetime.datetime.now()
return combined_df
def search_repo_simple(search_term: str = None,
min_stars_number: int = None,
created_at: str = None,
pushed_date: str = None
):
"""
:param search_term:
:param min_stars_number:
:param created_at:
:param pushed_date:
usage:
>>> search_term = 'machine learning trading'
>>> min_stars_number = 100
>>> created_at = None
>>> pushed_date = None
"""
if search_term is None:
_search_term = ''
else:
_search_term = search_term
qualifier_dict = {}
if min_stars_number is not None:
qualifier_dict['stars'] = '>={}'.format(min_stars_number)
if created_at is not None:
qualifier_dict['created'] = '>={}'.format(created_at)
if pushed_date is not None:
qualifier_dict['pushed'] = '>={}'.format(pushed_date)
search_result = search_repo(_search_term, qualifier_dict)
return search_result
def search_new_repo_by_category(category: str,
min_stars_number: int = 100,
existing_repo_df: pd.DataFrame = None):
"""
:param category:
:param min_stars_number:
:param existing_repo_df:
:return:
usage:
>>> category = 'Data Processing Techniques and Transformations'
>>> min_stars_number = 100
>>> existing_repo_df = get_repo_list()
"""
print('*** searching for category [{}] ***'.format(category))
combined_df = None
if category == 'Deep Learning And Reinforcement Learning':
combined_df = search_repo_multiple_terms(['deep learning trading',
'deep learning finance',
'reinforcement learning trading',
'reinforcement learning finance'],
category,
min_stars_number=min_stars_number
)
elif category == 'Other Models':
combined_df = search_repo_multiple_terms(['machine learning trading',
'machine learning finance'],
category,
min_stars_number=min_stars_number
)
elif category == 'Data Processing Techniques and Transformations':
combined_df = search_repo_multiple_terms(['data transformation trading',
'data transformation finance',
'data transformation time series',
'data processing trading',
'data processing finance',
'power transform trading',
'power transform finance',
'standardization normalization trading',
'standardization normalization finance',
],
category,
min_stars_number=int(min_stars_number * 0.5)
)
elif category == 'Portfolio Selection and Optimisation':
combined_df = search_repo_multiple_terms(['portfolio optimization machine learning finance',
'portfolio optimization machine learning trading',
'portfolio construction machine learning finance',
'portfolio construction machine learning trading',
'portfolio optimization finance',
'portfolio optimization trading',
'portfolio construction finance',
'portfolio construction trading'
],
category,
min_stars_number=min_stars_number
)
elif category == 'Factor and Risk Analysis':
combined_df = search_repo_multiple_terms(['risk factor finance',
'risk factor trading',
'risk premia factor finance',
'risk premia factor trading',
'style factor finance',
'style factor trading',
'macro factor finance',
'macro factor trading',
],
category,
min_stars_number=int(min_stars_number * 0.05)
)
elif category == 'Unsupervised':
combined_df = search_repo_multiple_terms(['unsupervised finance',
'unsupervised trading'
],
category,
min_stars_number=int(min_stars_number * 0.1)
)
elif category == 'Textual':
combined_df = search_repo_multiple_terms(['NLP finance',
'NLP trading'
],
category,
min_stars_number=min_stars_number
)
elif category == 'Derivatives and Hedging':
combined_df = search_repo_multiple_terms(['derivatives finance',
'derivatives trading',
'quantlib trading',
'quantlib finance',
'hedging finance',
'hedging trading',
'option trading',
'option finance',
'delta hedge trading',
'delta hedge finance'
],
category,
min_stars_number=min_stars_number
)
elif category == 'Fixed Income':
combined_df = search_repo_multiple_terms(['corporate bond finance',
'corporate bond trading',
'muni bond trading',
'muni bond finance',
'investment grade finance',
'investment grade trading',
'high yield trading',
'high yield finance',
'credit rating trading',
'credit rating finance',
'fixed income trading',
'fixed income finance',
'corporate bond',
'muni bond',
'credit rating'
],
category,
min_stars_number=int(min_stars_number * 0.2)
)
elif category == 'Alternative Finance':
# don't include crypto here as it will skew the results, consider putting it as a seperate category
combined_df = search_repo_multiple_terms(['private equity',
'venture capital',
'real estate trading',
'real estate finance',
'alternative asset trading',
'alternative asset finance',
'commodity trading',
'commodity finance',
'farmland finance',
'farmland trading'
],
category,
min_stars_number=int(min_stars_number * 0.5)
)
# only find ones that need to be inserted
if combined_df is not None and not combined_df.empty and existing_repo_df is not None:
combined_df = combined_df[
~combined_df['repo_path'].str.lower().isin(existing_repo_df['repo_path'].dropna().str.lower())]
return combined_df
def search_new_repo_and_append(min_stars_number: int = 100, filter_list=None):
"""
:param min_stars_number:
:param filter_list:
"""
repo_df = get_repo_list()
category_list = repo_df['category'].unique().tolist()
if filter_list is not None:
category_list = [x for x in category_list if x in filter_list]
new_repo_list = []
for category in category_list:
combined_df = search_new_repo_by_category(category, min_stars_number, repo_df)
if combined_df is not None:
new_repo_list.append(combined_df)
new_repo_df = pd.concat(new_repo_list).reset_index(drop=True)
# drop duplicate regardless of the category, keep first one for now
new_repo_df = new_repo_df.drop_duplicates(subset='repo_path')
final_df = pd.concat([repo_df, new_repo_df]).reset_index(drop=True)
final_df = final_df.sort_values(by='category')
final_df.to_csv(os.path.join(PROJECT_ROOT_DIR, 'raw_data', 'url_list.csv'), index=False)
if __name__ == '__main__':
search_new_repo_and_append(min_stars_number=100)
git_status.py
+2 -332
View File
@@ -1,294 +1,11 @@
import os
from typing import Dict, List
import datetime
from conf import PROJECT_ROOT_DIR
import re
import pandas as pd
from github import Github, Repository
from git_util import get_repo_attributes_dict, get_github_client, get_repo_path
def get_github_client():
# search for app_client and client secrets first, since this allow higher api request limit
github_app = os.environ.get('GIT_APP_ID')
if github_app is None:
github_token = os.environ.get('GIT_TOKEN')
g = Github(github_token)
else:
github_app_secret = os.environ.get('GIT_APP_SECRET')
g = Github(
client_id=github_app,
client_secret=github_app_secret)
return g
# generic search functions
def search_repo(search_term: str, qualifier_dict: Dict):
g = get_github_client()
qualifier_str = ' '.join(['{}:{}'.format(k, v) for k, v in iter(qualifier_dict.items())])
if qualifier_str != '':
final_search_term = '{} {}'.format(search_term, qualifier_str)
else:
final_search_term = search_term
repo_result = g.search_repositories(final_search_term)
return repo_result
def search_repo_multiple_terms(term_list: List[str],
category: str,
min_stars_number: int = None,
created_at: str = None,
pushed_date: str = None,
drop_duplicate: bool = True
):
"""
:param term_list:
:param category:
:param min_stars_number:
:param created_at:
:param pushed_date:
:param drop_duplicate:
:return:
usage:
>>> term_list = ['deep learning trading', 'deep learning finance', 'reinforcement learning trading',
'reinforcement learning finance']
>>> category = 'Deep Learning And Reinforcement Learning'
>>> min_stars_number = 100
>>> created_at = None
>>> pushed_date = None
>>> drop_duplicate = True
"""
repo_df_list = []
for search_term in term_list:
repo_list = search_repo_simple(search_term, min_stars_number, created_at=created_at, pushed_date=pushed_date)
repo_df = convert_repo_list_to_df(repo_list, category)
repo_df_list.append(repo_df)
combined_df = pd.concat(repo_df_list).reset_index(drop=True)
if drop_duplicate:
combined_df = combined_df.drop_duplicates()
combined_df['finml_added_date'] = datetime.datetime.now()
return combined_df
def search_repo_simple(search_term: str = None,
min_stars_number: int = None,
created_at: str = None,
pushed_date: str = None
):
"""
:param search_term:
:param min_stars_number:
:param created_at:
:param pushed_date:
usage:
>>> search_term = 'machine learning trading'
>>> min_stars_number = 100
>>> created_at = None
>>> pushed_date = None
"""
if search_term is None:
_search_term = ''
else:
_search_term = search_term
qualifier_dict = {}
if min_stars_number is not None:
qualifier_dict['stars'] = '>={}'.format(min_stars_number)
if created_at is not None:
qualifier_dict['created'] = '>={}'.format(created_at)
if pushed_date is not None:
qualifier_dict['pushed'] = '>={}'.format(pushed_date)
search_result = search_repo(_search_term, qualifier_dict)
return search_result
# *******
# topic specific search functions
# *******
def convert_repo_list_to_df(repo_list, category):
df_list = []
for repo in repo_list:
attr_dict = get_repo_attributes_dict(repo)
attr_dict['name'] = repo.name
attr_dict['comment'] = 'NEW'
attr_dict['category'] = category
attr_dict['repo_path'] = repo.full_name
attr_dict['url'] = 'https://github.com/{}'.format(repo.full_name)
df_list.append(attr_dict)
result_df = pd.DataFrame(df_list)
return result_df
def search_new_repo_by_category(category: str,
min_stars_number: int = 100,
existing_repo_df: pd.DataFrame = None):
"""
:param category:
:param min_stars_number:
:param existing_repo_df:
:return:
usage:
>>> category = 'Other Models'
>>> min_stars_number = 100
>>> existing_repo_df = get_repo_list()
"""
print('*** searching for category [{}] ***'.format(category))
combined_df = None
if category == 'Deep Learning And Reinforcement Learning':
combined_df = search_repo_multiple_terms(['deep learning trading',
'deep learning finance',
'reinforcement learning trading',
'reinforcement learning finance'],
category,
min_stars_number=min_stars_number
)
elif category == 'Other Models':
combined_df = search_repo_multiple_terms(['machine learning trading',
'machine learning finance'],
category,
min_stars_number=min_stars_number
)
elif category == 'Data Processing Techniques and Transformations':
combined_df = search_repo_multiple_terms(['data transformation trading',
'data transformation finance',
'data transformation time series'
'data processing trading',
'data processing finance'
],
category,
min_stars_number=int(min_stars_number)
)
elif category == 'Portfolio Selection and Optimisation':
combined_df = search_repo_multiple_terms(['portfolio optimization machine learning finance',
'portfolio optimization machine learning trading',
'portfolio construction machine learning finance',
'portfolio construction machine learning trading',
'portfolio optimization finance',
'portfolio optimization trading',
'portfolio construction finance',
'portfolio construction trading'
],
category,
min_stars_number=int(min_stars_number)
)
elif category == 'Factor and Risk Analysis':
combined_df = search_repo_multiple_terms(['risk factor finance',
'risk factor trading',
'risk premia factor finance',
'risk premia factor trading',
'style factor finance',
'style factor trading',
'macro factor finance',
'macro factor trading'
],
category,
min_stars_number=int(min_stars_number)
)
elif category == 'Unsupervised':
combined_df = search_repo_multiple_terms(['unsupervised learning finance',
'unsupervised learning trading'
],
category,
min_stars_number=int(min_stars_number)
)
elif category == 'Textual':
combined_df = search_repo_multiple_terms(['NLP finance',
'NLP trading'
],
category,
min_stars_number=int(min_stars_number)
)
elif category == 'Derivatives and Hedging':
combined_df = search_repo_multiple_terms(['derivatives finance',
'derivatives trading',
'quantlib trading',
'quantlib finance',
'hedging finance',
'hedging trading',
'option trading',
'option finance',
'delta hedge trading',
'delta hedge finance'
],
category,
min_stars_number=int(min_stars_number)
)
elif category == 'Fixed Income':
combined_df = search_repo_multiple_terms(['corporate bond finance',
'corporate bond trading',
'muni bond trading',
'muni bond finance',
'investment grade finance',
'investment grade trading',
'high yield trading',
'high yield finance',
'credit rating trading',
'credit rating finance',
'fixed income trading',
'fixed income finance'
],
category,
min_stars_number=int(min_stars_number)
)
elif category == 'Alternative Finance':
# don't include crypto here as it will skew the results, consider putting it as a seperate category
combined_df = search_repo_multiple_terms(['private equity',
'venture capital',
'real estate trading',
'real estate finance',
'alternative asset trading',
'alternative asset finance',
'commodity trading',
'commodity finance',
'farmland finance',
'farmland trading'
],
category,
min_stars_number=int(min_stars_number)
)
# only find ones that need to be inserted
if combined_df is not None and not combined_df.empty and existing_repo_df is not None:
combined_df = combined_df[
~combined_df['repo_path'].str.lower().isin(existing_repo_df['repo_path'].dropna().str.lower())]
return combined_df
def search_new_repo_and_append(min_stars_number: int = 100, filter_list=None):
"""
:param min_stars_number:
:param filter_list:
"""
repo_df = get_repo_list()
category_list = repo_df['category'].unique().tolist()
if filter_list is not None:
category_list = [x for x in category_list if x in filter_list]
new_repo_list = []
for category in category_list:
combined_df = search_new_repo_by_category(category, min_stars_number, repo_df)
if combined_df is not None:
new_repo_list.append(combined_df)
new_repo_df = pd.concat(new_repo_list).reset_index(drop=True)
# drop duplicate regardless of the category, keep first one for now
new_repo_df = new_repo_df.drop_duplicates(subset='repo_path')
final_df = pd.concat([repo_df, new_repo_df]).reset_index(drop=True)
final_df = final_df.sort_values(by='category')
final_df.to_csv(os.path.join(PROJECT_ROOT_DIR, 'raw_data', 'url_list.csv'), index=False)
# *******
# saved repo list, treat it as database for now
# *******
def get_repo_list():
repo_df = pd.read_csv(os.path.join(PROJECT_ROOT_DIR, 'raw_data', 'url_list.csv'))
if 'repo_path' not in repo_df.columns:
@@ -296,52 +13,6 @@ def get_repo_list():
return repo_df
# *******
# repo specific information
# *******
def get_repo_path(in_url):
repo_path = None
if 'https://github.com/' in in_url:
url_query = in_url.replace('https://github.com/', '')
repo_path = '/'.join(url_query.split('/')[:2])
return repo_path
def get_last_commit_date(input_repo: Repository):
"""
get latest commit from repo
:param input_repo:
:return:
"""
page = input_repo.get_commits().get_page(0)[0]
return page.commit.author.date
def get_repo_attributes_dict(input_repo: Repository, last_commit_within_years: int = 2):
result_dict = {
'repo_path': input_repo.full_name,
'created_at': input_repo.created_at,
'last_commit': get_last_commit_date(input_repo),
'last_update': input_repo.updated_at,
'star_count': input_repo.stargazers_count,
'fork_count': input_repo.forks_count,
'contributors_count': input_repo.get_contributors().totalCount
}
today = datetime.datetime.today()
check_start_date = datetime.datetime(today.year - last_commit_within_years,
today.month,
today.day)
if result_dict['last_commit'] >= check_start_date:
repo_status = 'active'
else:
repo_status = 'inactive'
result_dict['repo_status'] = repo_status
return result_dict
def get_repo_status():
g = get_github_client()
repo_df = get_repo_list()
@@ -417,4 +88,3 @@ def parse_readme_md():
if __name__ == '__main__':
get_repo_status()
search_new_repo_and_append(min_stars_number=100)
git_util.py
+63
View File
@@ -0,0 +1,63 @@
from github import Github, Repository
import os
import datetime
def get_github_client():
# search for app_client and client secrets first, since this allow higher api request limit
github_app = os.environ.get('GIT_APP_ID')
if github_app is None:
github_token = os.environ.get('GIT_TOKEN')
g = Github(github_token)
else:
github_app_secret = os.environ.get('GIT_APP_SECRET')
g = Github(
client_id=github_app,
client_secret=github_app_secret)
return g
# *******
# repo specific information
# *******
def get_repo_path(in_url):
repo_path = None
if 'https://github.com/' in in_url:
url_query = in_url.replace('https://github.com/', '')
repo_path = '/'.join(url_query.split('/')[:2])
return repo_path
def get_last_commit_date(input_repo: Repository):
"""
get latest commit from repo
:param input_repo:
:return:
"""
page = input_repo.get_commits().get_page(0)[0]
return page.commit.author.date
def get_repo_attributes_dict(input_repo: Repository, last_commit_within_years: int = 2):
result_dict = {
'repo_path': input_repo.full_name,
'created_at': input_repo.created_at,
'last_commit': get_last_commit_date(input_repo),
'last_update': input_repo.updated_at,
'star_count': input_repo.stargazers_count,
'fork_count': input_repo.forks_count,
'contributors_count': input_repo.get_contributors().totalCount
}
today = datetime.datetime.today()
check_start_date = datetime.datetime(today.year - last_commit_within_years,
today.month,
today.day)
if result_dict['last_commit'] >= check_start_date:
repo_status = 'active'
else:
repo_status = 'inactive'
result_dict['repo_status'] = repo_status
return result_dict