diff --git a/.github/workflows/repo_search.yml b/.github/workflows/repo_search.yml new file mode 100644 index 0000000..cfa41be --- /dev/null +++ b/.github/workflows/repo_search.yml @@ -0,0 +1,38 @@ +name: Repo-Search + +on: + schedule: + - cron: '0 0 * * *' + +jobs: + # Set the job key. The key is displayed as the job name + update-repo-status: + # Name the Job + name: Update repo status for all saved repo + # Set the type of machine to run on + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v2 + + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: 3.7 + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + + - name: execute py script # run the run.py to get the latest data + run: | + python git_search.py + env: + GIT_TOKEN: ${{ secrets.GIT_TOKEN }} + + - name: Commit & Push changes + uses: actions-js/push@master + with: + github_token: ${{ secrets.GIT_TOKEN }} diff --git a/.github/workflows/repo_status.yml b/.github/workflows/repo_status.yml index ab10d78..6f5456f 100644 --- a/.github/workflows/repo_status.yml +++ b/.github/workflows/repo_status.yml @@ -2,7 +2,7 @@ name: Repo-Updater on: schedule: - - cron: '0 0 * * *' # daily + - cron: '0 0 * * 0' # weekly jobs: diff --git a/.github/workflows/wiki_gen.yml b/.github/workflows/wiki_gen.yml index 0e18498..ce25c57 100644 --- a/.github/workflows/wiki_gen.yml +++ b/.github/workflows/wiki_gen.yml @@ -25,11 +25,11 @@ jobs: python -m pip install --upgrade pip if [ -f requirements.txt ]; then pip install -r requirements.txt; fi -# - name: execute status update script # run the run.py to get the latest data -# run: | -# python git_status.py -# env: -# GIT_TOKEN: ${{ secrets.GIT_TOKEN }} + - name: execute status update script # run the run.py to get the latest data + run: | + python git_search.py + env: + GIT_TOKEN: ${{ secrets.GIT_TOKEN }} - name: execute wiki generation script # run the wiki_gen run: | diff --git a/git_search.py b/git_search.py new file mode 100644 index 0000000..85d94b5 --- /dev/null +++ b/git_search.py @@ -0,0 +1,282 @@ +import datetime +import os +from typing import Dict, List + +import pandas as pd + +from conf import PROJECT_ROOT_DIR +from git_status import get_repo_list +from git_util import get_github_client, get_repo_attributes_dict + + +def convert_repo_list_to_df(repo_list, category): + df_list = [] + for repo in repo_list: + attr_dict = get_repo_attributes_dict(repo) + attr_dict['name'] = repo.name + attr_dict['comment'] = 'NEW' + attr_dict['category'] = category + attr_dict['repo_path'] = repo.full_name + attr_dict['url'] = 'https://github.com/{}'.format(repo.full_name) + df_list.append(attr_dict) + result_df = pd.DataFrame(df_list) + return result_df + + +# generic search functions +def search_repo(search_term: str, qualifier_dict: Dict): + g = get_github_client() + qualifier_str = ' '.join(['{}:{}'.format(k, v) for k, v in iter(qualifier_dict.items())]) + if qualifier_str != '': + final_search_term = '{} {}'.format(search_term, qualifier_str) + else: + final_search_term = search_term + repo_result = g.search_repositories(final_search_term) + return repo_result + + +def search_repo_multiple_terms(term_list: List[str], + category: str, + min_stars_number: int = None, + created_at: str = None, + pushed_date: str = None, + drop_duplicate: bool = True + ): + """ + + :param term_list: + :param category: + :param min_stars_number: + :param created_at: + :param pushed_date: + :param drop_duplicate: + :return: + usage: + >>> term_list = ['deep learning trading', 'deep learning finance', 'reinforcement learning trading', + 'reinforcement learning finance'] + >>> category = 'Deep Learning And Reinforcement Learning' + >>> min_stars_number = 100 + >>> created_at = None + >>> pushed_date = None + >>> drop_duplicate = True + """ + repo_df_list = [] + for search_term in term_list: + repo_list = search_repo_simple(search_term, min_stars_number, created_at=created_at, pushed_date=pushed_date) + repo_df = convert_repo_list_to_df(repo_list, category) + repo_df_list.append(repo_df) + combined_df = pd.concat(repo_df_list).reset_index(drop=True) + if drop_duplicate: + combined_df = combined_df.drop_duplicates() + combined_df['finml_added_date'] = datetime.datetime.now() + return combined_df + + +def search_repo_simple(search_term: str = None, + min_stars_number: int = None, + created_at: str = None, + pushed_date: str = None + ): + """ + + :param search_term: + :param min_stars_number: + :param created_at: + :param pushed_date: + usage: + >>> search_term = 'machine learning trading' + >>> min_stars_number = 100 + >>> created_at = None + >>> pushed_date = None + """ + if search_term is None: + _search_term = '' + else: + _search_term = search_term + + qualifier_dict = {} + if min_stars_number is not None: + qualifier_dict['stars'] = '>={}'.format(min_stars_number) + + if created_at is not None: + qualifier_dict['created'] = '>={}'.format(created_at) + + if pushed_date is not None: + qualifier_dict['pushed'] = '>={}'.format(pushed_date) + search_result = search_repo(_search_term, qualifier_dict) + return search_result + + +def search_new_repo_by_category(category: str, + min_stars_number: int = 100, + existing_repo_df: pd.DataFrame = None): + """ + + :param category: + :param min_stars_number: + :param existing_repo_df: + :return: + usage: + >>> category = 'Data Processing Techniques and Transformations' + >>> min_stars_number = 100 + >>> existing_repo_df = get_repo_list() + """ + print('*** searching for category [{}] ***'.format(category)) + combined_df = None + if category == 'Deep Learning And Reinforcement Learning': + combined_df = search_repo_multiple_terms(['deep learning trading', + 'deep learning finance', + 'reinforcement learning trading', + 'reinforcement learning finance'], + category, + min_stars_number=min_stars_number + ) + + elif category == 'Other Models': + combined_df = search_repo_multiple_terms(['machine learning trading', + 'machine learning finance'], + category, + min_stars_number=min_stars_number + ) + elif category == 'Data Processing Techniques and Transformations': + combined_df = search_repo_multiple_terms(['data transformation trading', + 'data transformation finance', + 'data transformation time series', + 'data processing trading', + 'data processing finance', + 'power transform trading', + 'power transform finance', + 'standardization normalization trading', + 'standardization normalization finance', + ], + category, + min_stars_number=int(min_stars_number * 0.5) + ) + elif category == 'Portfolio Selection and Optimisation': + combined_df = search_repo_multiple_terms(['portfolio optimization machine learning finance', + 'portfolio optimization machine learning trading', + 'portfolio construction machine learning finance', + 'portfolio construction machine learning trading', + 'portfolio optimization finance', + 'portfolio optimization trading', + 'portfolio construction finance', + 'portfolio construction trading' + ], + category, + min_stars_number=min_stars_number + ) + elif category == 'Factor and Risk Analysis': + combined_df = search_repo_multiple_terms(['risk factor finance', + 'risk factor trading', + 'risk premia factor finance', + 'risk premia factor trading', + 'style factor finance', + 'style factor trading', + 'macro factor finance', + 'macro factor trading', + ], + category, + min_stars_number=int(min_stars_number * 0.05) + ) + elif category == 'Unsupervised': + combined_df = search_repo_multiple_terms(['unsupervised finance', + 'unsupervised trading' + ], + category, + min_stars_number=int(min_stars_number * 0.1) + ) + elif category == 'Textual': + combined_df = search_repo_multiple_terms(['NLP finance', + 'NLP trading' + ], + category, + min_stars_number=min_stars_number + ) + elif category == 'Derivatives and Hedging': + combined_df = search_repo_multiple_terms(['derivatives finance', + 'derivatives trading', + 'quantlib trading', + 'quantlib finance', + 'hedging finance', + 'hedging trading', + 'option trading', + 'option finance', + 'delta hedge trading', + 'delta hedge finance' + ], + category, + min_stars_number=min_stars_number + ) + elif category == 'Fixed Income': + combined_df = search_repo_multiple_terms(['corporate bond finance', + 'corporate bond trading', + 'muni bond trading', + 'muni bond finance', + 'investment grade finance', + 'investment grade trading', + 'high yield trading', + 'high yield finance', + 'credit rating trading', + 'credit rating finance', + 'fixed income trading', + 'fixed income finance', + 'corporate bond', + 'muni bond', + 'credit rating' + ], + category, + min_stars_number=int(min_stars_number * 0.2) + ) + elif category == 'Alternative Finance': + # don't include crypto here as it will skew the results, consider putting it as a seperate category + combined_df = search_repo_multiple_terms(['private equity', + 'venture capital', + 'real estate trading', + 'real estate finance', + 'alternative asset trading', + 'alternative asset finance', + 'commodity trading', + 'commodity finance', + 'farmland finance', + 'farmland trading' + ], + category, + min_stars_number=int(min_stars_number * 0.5) + ) + + # only find ones that need to be inserted + if combined_df is not None and not combined_df.empty and existing_repo_df is not None: + combined_df = combined_df[ + ~combined_df['repo_path'].str.lower().isin(existing_repo_df['repo_path'].dropna().str.lower())] + + return combined_df + + +def search_new_repo_and_append(min_stars_number: int = 100, filter_list=None): + """ + + :param min_stars_number: + :param filter_list: + """ + repo_df = get_repo_list() + category_list = repo_df['category'].unique().tolist() + if filter_list is not None: + category_list = [x for x in category_list if x in filter_list] + + new_repo_list = [] + for category in category_list: + combined_df = search_new_repo_by_category(category, min_stars_number, repo_df) + if combined_df is not None: + new_repo_list.append(combined_df) + new_repo_df = pd.concat(new_repo_list).reset_index(drop=True) + # drop duplicate regardless of the category, keep first one for now + new_repo_df = new_repo_df.drop_duplicates(subset='repo_path') + + final_df = pd.concat([repo_df, new_repo_df]).reset_index(drop=True) + + final_df = final_df.sort_values(by='category') + final_df.to_csv(os.path.join(PROJECT_ROOT_DIR, 'raw_data', 'url_list.csv'), index=False) + + +if __name__ == '__main__': + search_new_repo_and_append(min_stars_number=100) diff --git a/git_status.py b/git_status.py index 2753015..c78267c 100644 --- a/git_status.py +++ b/git_status.py @@ -1,294 +1,11 @@ import os -from typing import Dict, List -import datetime from conf import PROJECT_ROOT_DIR import re import pandas as pd -from github import Github, Repository + +from git_util import get_repo_attributes_dict, get_github_client, get_repo_path -def get_github_client(): - # search for app_client and client secrets first, since this allow higher api request limit - github_app = os.environ.get('GIT_APP_ID') - if github_app is None: - github_token = os.environ.get('GIT_TOKEN') - g = Github(github_token) - else: - github_app_secret = os.environ.get('GIT_APP_SECRET') - g = Github( - client_id=github_app, - client_secret=github_app_secret) - return g - - -# generic search functions -def search_repo(search_term: str, qualifier_dict: Dict): - g = get_github_client() - qualifier_str = ' '.join(['{}:{}'.format(k, v) for k, v in iter(qualifier_dict.items())]) - if qualifier_str != '': - final_search_term = '{} {}'.format(search_term, qualifier_str) - else: - final_search_term = search_term - repo_result = g.search_repositories(final_search_term) - return repo_result - - -def search_repo_multiple_terms(term_list: List[str], - category: str, - min_stars_number: int = None, - created_at: str = None, - pushed_date: str = None, - drop_duplicate: bool = True - ): - """ - - :param term_list: - :param category: - :param min_stars_number: - :param created_at: - :param pushed_date: - :param drop_duplicate: - :return: - usage: - >>> term_list = ['deep learning trading', 'deep learning finance', 'reinforcement learning trading', - 'reinforcement learning finance'] - >>> category = 'Deep Learning And Reinforcement Learning' - >>> min_stars_number = 100 - >>> created_at = None - >>> pushed_date = None - >>> drop_duplicate = True - """ - repo_df_list = [] - for search_term in term_list: - repo_list = search_repo_simple(search_term, min_stars_number, created_at=created_at, pushed_date=pushed_date) - repo_df = convert_repo_list_to_df(repo_list, category) - repo_df_list.append(repo_df) - combined_df = pd.concat(repo_df_list).reset_index(drop=True) - if drop_duplicate: - combined_df = combined_df.drop_duplicates() - combined_df['finml_added_date'] = datetime.datetime.now() - return combined_df - - -def search_repo_simple(search_term: str = None, - min_stars_number: int = None, - created_at: str = None, - pushed_date: str = None - ): - """ - - :param search_term: - :param min_stars_number: - :param created_at: - :param pushed_date: - usage: - >>> search_term = 'machine learning trading' - >>> min_stars_number = 100 - >>> created_at = None - >>> pushed_date = None - """ - if search_term is None: - _search_term = '' - else: - _search_term = search_term - - qualifier_dict = {} - if min_stars_number is not None: - qualifier_dict['stars'] = '>={}'.format(min_stars_number) - - if created_at is not None: - qualifier_dict['created'] = '>={}'.format(created_at) - - if pushed_date is not None: - qualifier_dict['pushed'] = '>={}'.format(pushed_date) - search_result = search_repo(_search_term, qualifier_dict) - return search_result - - -# ******* -# topic specific search functions -# ******* -def convert_repo_list_to_df(repo_list, category): - df_list = [] - for repo in repo_list: - attr_dict = get_repo_attributes_dict(repo) - attr_dict['name'] = repo.name - attr_dict['comment'] = 'NEW' - attr_dict['category'] = category - attr_dict['repo_path'] = repo.full_name - attr_dict['url'] = 'https://github.com/{}'.format(repo.full_name) - df_list.append(attr_dict) - result_df = pd.DataFrame(df_list) - return result_df - - -def search_new_repo_by_category(category: str, - min_stars_number: int = 100, - existing_repo_df: pd.DataFrame = None): - """ - - :param category: - :param min_stars_number: - :param existing_repo_df: - :return: - usage: - >>> category = 'Other Models' - >>> min_stars_number = 100 - >>> existing_repo_df = get_repo_list() - """ - print('*** searching for category [{}] ***'.format(category)) - combined_df = None - if category == 'Deep Learning And Reinforcement Learning': - combined_df = search_repo_multiple_terms(['deep learning trading', - 'deep learning finance', - 'reinforcement learning trading', - 'reinforcement learning finance'], - category, - min_stars_number=min_stars_number - ) - - elif category == 'Other Models': - combined_df = search_repo_multiple_terms(['machine learning trading', - 'machine learning finance'], - category, - min_stars_number=min_stars_number - ) - elif category == 'Data Processing Techniques and Transformations': - combined_df = search_repo_multiple_terms(['data transformation trading', - 'data transformation finance', - 'data transformation time series' - 'data processing trading', - 'data processing finance' - ], - category, - min_stars_number=int(min_stars_number) - ) - elif category == 'Portfolio Selection and Optimisation': - combined_df = search_repo_multiple_terms(['portfolio optimization machine learning finance', - 'portfolio optimization machine learning trading', - 'portfolio construction machine learning finance', - 'portfolio construction machine learning trading', - 'portfolio optimization finance', - 'portfolio optimization trading', - 'portfolio construction finance', - 'portfolio construction trading' - ], - category, - min_stars_number=int(min_stars_number) - ) - elif category == 'Factor and Risk Analysis': - combined_df = search_repo_multiple_terms(['risk factor finance', - 'risk factor trading', - 'risk premia factor finance', - 'risk premia factor trading', - 'style factor finance', - 'style factor trading', - 'macro factor finance', - 'macro factor trading' - ], - category, - min_stars_number=int(min_stars_number) - ) - elif category == 'Unsupervised': - combined_df = search_repo_multiple_terms(['unsupervised learning finance', - 'unsupervised learning trading' - ], - category, - min_stars_number=int(min_stars_number) - ) - elif category == 'Textual': - combined_df = search_repo_multiple_terms(['NLP finance', - 'NLP trading' - ], - category, - min_stars_number=int(min_stars_number) - ) - elif category == 'Derivatives and Hedging': - combined_df = search_repo_multiple_terms(['derivatives finance', - 'derivatives trading', - 'quantlib trading', - 'quantlib finance', - 'hedging finance', - 'hedging trading', - 'option trading', - 'option finance', - 'delta hedge trading', - 'delta hedge finance' - ], - category, - min_stars_number=int(min_stars_number) - ) - elif category == 'Fixed Income': - combined_df = search_repo_multiple_terms(['corporate bond finance', - 'corporate bond trading', - 'muni bond trading', - 'muni bond finance', - 'investment grade finance', - 'investment grade trading', - 'high yield trading', - 'high yield finance', - 'credit rating trading', - 'credit rating finance', - 'fixed income trading', - 'fixed income finance' - ], - category, - min_stars_number=int(min_stars_number) - ) - elif category == 'Alternative Finance': - # don't include crypto here as it will skew the results, consider putting it as a seperate category - combined_df = search_repo_multiple_terms(['private equity', - 'venture capital', - 'real estate trading', - 'real estate finance', - 'alternative asset trading', - 'alternative asset finance', - 'commodity trading', - 'commodity finance', - 'farmland finance', - 'farmland trading' - ], - category, - min_stars_number=int(min_stars_number) - ) - - # only find ones that need to be inserted - if combined_df is not None and not combined_df.empty and existing_repo_df is not None: - combined_df = combined_df[ - ~combined_df['repo_path'].str.lower().isin(existing_repo_df['repo_path'].dropna().str.lower())] - - return combined_df - - -def search_new_repo_and_append(min_stars_number: int = 100, filter_list=None): - """ - - :param min_stars_number: - :param filter_list: - """ - repo_df = get_repo_list() - category_list = repo_df['category'].unique().tolist() - if filter_list is not None: - category_list = [x for x in category_list if x in filter_list] - - new_repo_list = [] - for category in category_list: - combined_df = search_new_repo_by_category(category, min_stars_number, repo_df) - if combined_df is not None: - new_repo_list.append(combined_df) - new_repo_df = pd.concat(new_repo_list).reset_index(drop=True) - # drop duplicate regardless of the category, keep first one for now - new_repo_df = new_repo_df.drop_duplicates(subset='repo_path') - - final_df = pd.concat([repo_df, new_repo_df]).reset_index(drop=True) - - final_df = final_df.sort_values(by='category') - final_df.to_csv(os.path.join(PROJECT_ROOT_DIR, 'raw_data', 'url_list.csv'), index=False) - - -# ******* -# saved repo list, treat it as database for now -# ******* def get_repo_list(): repo_df = pd.read_csv(os.path.join(PROJECT_ROOT_DIR, 'raw_data', 'url_list.csv')) if 'repo_path' not in repo_df.columns: @@ -296,52 +13,6 @@ def get_repo_list(): return repo_df -# ******* -# repo specific information -# ******* -def get_repo_path(in_url): - repo_path = None - if 'https://github.com/' in in_url: - url_query = in_url.replace('https://github.com/', '') - repo_path = '/'.join(url_query.split('/')[:2]) - return repo_path - - -def get_last_commit_date(input_repo: Repository): - """ - get latest commit from repo - :param input_repo: - :return: - """ - page = input_repo.get_commits().get_page(0)[0] - return page.commit.author.date - - -def get_repo_attributes_dict(input_repo: Repository, last_commit_within_years: int = 2): - result_dict = { - 'repo_path': input_repo.full_name, - 'created_at': input_repo.created_at, - 'last_commit': get_last_commit_date(input_repo), - 'last_update': input_repo.updated_at, - 'star_count': input_repo.stargazers_count, - 'fork_count': input_repo.forks_count, - 'contributors_count': input_repo.get_contributors().totalCount - - } - today = datetime.datetime.today() - check_start_date = datetime.datetime(today.year - last_commit_within_years, - today.month, - today.day) - - if result_dict['last_commit'] >= check_start_date: - repo_status = 'active' - else: - repo_status = 'inactive' - result_dict['repo_status'] = repo_status - - return result_dict - - def get_repo_status(): g = get_github_client() repo_df = get_repo_list() @@ -417,4 +88,3 @@ def parse_readme_md(): if __name__ == '__main__': get_repo_status() - search_new_repo_and_append(min_stars_number=100) diff --git a/git_util.py b/git_util.py new file mode 100644 index 0000000..2f1c793 --- /dev/null +++ b/git_util.py @@ -0,0 +1,63 @@ +from github import Github, Repository +import os +import datetime + + +def get_github_client(): + # search for app_client and client secrets first, since this allow higher api request limit + github_app = os.environ.get('GIT_APP_ID') + if github_app is None: + github_token = os.environ.get('GIT_TOKEN') + g = Github(github_token) + else: + github_app_secret = os.environ.get('GIT_APP_SECRET') + g = Github( + client_id=github_app, + client_secret=github_app_secret) + return g + + +# ******* +# repo specific information +# ******* +def get_repo_path(in_url): + repo_path = None + if 'https://github.com/' in in_url: + url_query = in_url.replace('https://github.com/', '') + repo_path = '/'.join(url_query.split('/')[:2]) + return repo_path + + +def get_last_commit_date(input_repo: Repository): + """ + get latest commit from repo + :param input_repo: + :return: + """ + page = input_repo.get_commits().get_page(0)[0] + return page.commit.author.date + + +def get_repo_attributes_dict(input_repo: Repository, last_commit_within_years: int = 2): + result_dict = { + 'repo_path': input_repo.full_name, + 'created_at': input_repo.created_at, + 'last_commit': get_last_commit_date(input_repo), + 'last_update': input_repo.updated_at, + 'star_count': input_repo.stargazers_count, + 'fork_count': input_repo.forks_count, + 'contributors_count': input_repo.get_contributors().totalCount + + } + today = datetime.datetime.today() + check_start_date = datetime.datetime(today.year - last_commit_within_years, + today.month, + today.day) + + if result_dict['last_commit'] >= check_start_date: + repo_status = 'active' + else: + repo_status = 'inactive' + result_dict['repo_status'] = repo_status + + return result_dict