diff --git a/data_loader/load.py b/data_loader/load.py index 7a107bf..5529e75 100644 --- a/data_loader/load.py +++ b/data_loader/load.py @@ -7,7 +7,7 @@ from feature_extractors.types import FeatureExtractor from utils.helpers import drop_columns_if_exist from data_loader.collections import DataCollection from typing import Literal -import ray + import os from config.hashing import hash_data_config from .types import XDataFrame, ReturnSeries, ForwardReturnSeries @@ -53,8 +53,8 @@ def __load_data( ] files = other_files + other_assets - target_asset_future = [ - __load_df.remote( + target_asset_df = [ + __load_df( data_source=data_source, prefix=data_source[1], returns="log_returns", @@ -62,18 +62,16 @@ def __load_data( ) for data_source in target_file ] - target_asset_df = ray.get(target_asset_future) - target_asset_only_returns_future = __load_df.remote( + df_target_asset_only_returns = __load_df( data_source=target_file[0], prefix=target_file[0][1], returns="returns", feature_extractors=[], ) - df_target_asset_only_returns = ray.get(target_asset_only_returns_future) - asset_futures = [ - __load_df.remote( + asset_dfs = [ + __load_df( data_source=data_source, prefix=data_source[1], returns="log_returns", @@ -81,10 +79,9 @@ def __load_data( ) for data_source in files ] - asset_dfs = ray.get(asset_futures) - exogenous_futures = [ - __load_df.remote( + exogenous_dfs = [ + __load_df( data_source=data_source, prefix=data_source[1], returns="none", @@ -92,7 +89,6 @@ def __load_data( ) for data_source in exogenous_data ] - exogenous_dfs = ray.get(exogenous_futures) X = target_asset_df + asset_dfs + exogenous_dfs X = pd.concat([df.sort_index().reindex(X[0].index) for df in X], axis=1).fillna(0.0) @@ -107,7 +103,6 @@ def __load_data( return X, returns -@ray.remote def __load_df( data_source: DataSource, prefix: str, @@ -162,7 +157,7 @@ def load_only_returns( ) -> pd.DataFrame: assets_future = [ - __load_df.remote( + __load_df( data_source=data_source, prefix=data_source[1], returns=returns, @@ -170,7 +165,7 @@ def load_only_returns( ) for data_source in assets ] - dfs = ray.get(assets_future) + dfs = assets_future dfs = pd.concat(dfs, axis=1) dfs.index = pd.DatetimeIndex(dfs.index) diff --git a/environment.yml b/environment.yml index 76af740..d7879c1 100644 --- a/environment.yml +++ b/environment.yml @@ -7,19 +7,21 @@ channels: - ranaroussi dependencies: - python=3.9 - - scikit-learn-intelex=2021.4.0 + - numba=0.53.1 + - xgboost + - scikit-learn + - lightgbm + - pydantic + - seaborn + - black + - ipython + - ipykernel + - scipy + - pip - pip: - black - - skorch - fracdiff - - ray - diskcache - - seaborn - - ipython - - ipykernel - - scipy - - scikit-learn - - numba - matplotlib - numpy - quantstats @@ -29,11 +31,8 @@ dependencies: - tscv - tqdm - pandas-ta - - xgboost - - lightgbm - alphalens-reloaded - vectorbt - - pydantic - pandera[mypy] - binance_historical_data prefix: /usr/local/anaconda3/envs/quant diff --git a/models/model_map.py b/models/model_map.py index d60eec3..ad96621 100644 --- a/models/model_map.py +++ b/models/model_map.py @@ -1,6 +1,5 @@ from models.sklearn import SKLearnModel -from sklearnex.ensemble import RandomForestClassifier -from sklearnex.ensemble import RandomForestRegressor +from sklearn.ensemble import RandomForestClassifier, RandomForestRegressor from .base import Model default_feature_selector_classification = SKLearnModel( @@ -24,7 +23,7 @@ def get_model(model_name: str) -> Model: ) ) elif model_name == "LogisticRegression_three_class": - from sklearnex.linear_model import LogisticRegression as LogisticRegression_EX + from sklearn.linear_model import LogisticRegression as LogisticRegression_EX return set_name( SKLearnModel( diff --git a/reporting/saving.py b/reporting/saving.py index ce414ab..048357d 100644 --- a/reporting/saving.py +++ b/reporting/saving.py @@ -15,7 +15,6 @@ def save_models(pipeline_outcome: PipelineOutcome, config: Config) -> None: date_string = datetime.datetime.now().strftime("%Y-%m-%d-%H-%M") if not os.path.exists("output/models"): - warnings.warn("No folder exists, creating one.") os.makedirs("output/models") pickle.dump(dict_for_pickle, open("output/models/{}.p".format(date_string), "wb")) diff --git a/run_pipeline.py b/run_pipeline.py index 170dbab..b7db9c2 100644 --- a/run_pipeline.py +++ b/run_pipeline.py @@ -17,10 +17,6 @@ from training.directional_training import train_directional_model from training.bet_sizing import bet_sizing_with_meta_model from training.types import PipelineOutcome -import ray - -ray.init() - def run_pipeline( project_name: str, with_wandb: bool, sweep: bool, raw_config: RawConfig