mirror of
https://github.com/webclinic017/drift.git
synced 2026-08-08 16:47:50 +00:00
chore(Linter): reformatted code with black (#211)
* chore(Linter): reformatted code with black * Create black.yaml
This commit is contained in:
Mark Aron Szulyovszky
committed by
GitHub
GitHub
parent
f3fee4a4e1
commit
8dd2d88740
+92
-64
@@ -10,8 +10,8 @@ import os
|
||||
from config.hashing import hash_data_config
|
||||
from .types import XDataFrame, ReturnSeries, ForwardReturnSeries
|
||||
from diskcache import Cache
|
||||
cache = Cache(".cachedir/data")
|
||||
|
||||
cache = Cache(".cachedir/data")
|
||||
|
||||
|
||||
def load_data(**kwargs) -> tuple[XDataFrame, ReturnSeries]:
|
||||
@@ -23,15 +23,17 @@ def load_data(**kwargs) -> tuple[XDataFrame, ReturnSeries]:
|
||||
cache[hashed] = return_value
|
||||
return return_value
|
||||
|
||||
def __load_data(assets: DataCollection,
|
||||
other_assets: DataCollection,
|
||||
exogenous_data: DataCollection,
|
||||
target_asset: DataSource,
|
||||
load_non_target_asset: bool,
|
||||
own_features: list[tuple[str, FeatureExtractor, list[int]]],
|
||||
other_features: list[tuple[str, FeatureExtractor, list[int]]],
|
||||
exogenous_features: list[tuple[str, FeatureExtractor, list[int]]],
|
||||
) -> tuple[XDataFrame, ReturnSeries]:
|
||||
|
||||
def __load_data(
|
||||
assets: DataCollection,
|
||||
other_assets: DataCollection,
|
||||
exogenous_data: DataCollection,
|
||||
target_asset: DataSource,
|
||||
load_non_target_asset: bool,
|
||||
own_features: list[tuple[str, FeatureExtractor, list[int]]],
|
||||
other_features: list[tuple[str, FeatureExtractor, list[int]]],
|
||||
exogenous_features: list[tuple[str, FeatureExtractor, list[int]]],
|
||||
) -> tuple[XDataFrame, ReturnSeries]:
|
||||
"""
|
||||
Loads asset data from the specified path.
|
||||
Returns:
|
||||
@@ -42,77 +44,96 @@ def __load_data(assets: DataCollection,
|
||||
|
||||
target_file = [f for f in assets if f[1].startswith(target_asset[1])]
|
||||
assert len(target_file) == 1, "There should be exactly one target file"
|
||||
other_files = [f for f in assets if load_non_target_asset == True and f[1].startswith(target_asset[1]) == False]
|
||||
other_files = [
|
||||
f
|
||||
for f in assets
|
||||
if load_non_target_asset == True and f[1].startswith(target_asset[1]) == False
|
||||
]
|
||||
files = other_files + other_assets
|
||||
|
||||
target_asset_future = [__load_df.remote(
|
||||
data_source=data_source,
|
||||
prefix=data_source[1],
|
||||
returns='log_returns',
|
||||
feature_extractors=own_features,
|
||||
) for data_source in target_file]
|
||||
|
||||
target_asset_future = [
|
||||
__load_df.remote(
|
||||
data_source=data_source,
|
||||
prefix=data_source[1],
|
||||
returns="log_returns",
|
||||
feature_extractors=own_features,
|
||||
)
|
||||
for data_source in target_file
|
||||
]
|
||||
target_asset_df = ray.get(target_asset_future)
|
||||
|
||||
|
||||
target_asset_only_returns_future = __load_df.remote(
|
||||
data_source=target_file[0],
|
||||
prefix=target_file[0][1],
|
||||
returns='returns',
|
||||
returns="returns",
|
||||
feature_extractors=[],
|
||||
)
|
||||
df_target_asset_only_returns = ray.get(target_asset_only_returns_future)
|
||||
|
||||
asset_futures = [__load_df.remote(
|
||||
data_source=data_source,
|
||||
prefix=data_source[1],
|
||||
returns='log_returns',
|
||||
feature_extractors=other_features,
|
||||
) for data_source in files]
|
||||
asset_futures = [
|
||||
__load_df.remote(
|
||||
data_source=data_source,
|
||||
prefix=data_source[1],
|
||||
returns="log_returns",
|
||||
feature_extractors=other_features,
|
||||
)
|
||||
for data_source in files
|
||||
]
|
||||
asset_dfs = ray.get(asset_futures)
|
||||
|
||||
exogenous_futures = [__load_df.remote(
|
||||
data_source=data_source,
|
||||
prefix=data_source[1],
|
||||
returns='none',
|
||||
feature_extractors=exogenous_features,
|
||||
) for data_source in exogenous_data]
|
||||
exogenous_futures = [
|
||||
__load_df.remote(
|
||||
data_source=data_source,
|
||||
prefix=data_source[1],
|
||||
returns="none",
|
||||
feature_extractors=exogenous_features,
|
||||
)
|
||||
for data_source in exogenous_data
|
||||
]
|
||||
exogenous_dfs = ray.get(exogenous_futures)
|
||||
|
||||
X = target_asset_df + asset_dfs + exogenous_dfs
|
||||
X = pd.concat([df.sort_index().reindex(X[0].index) for df in X], axis=1).fillna(0.)
|
||||
X = pd.concat([df.sort_index().reindex(X[0].index) for df in X], axis=1).fillna(0.0)
|
||||
|
||||
X.index = pd.DatetimeIndex(X.index)
|
||||
|
||||
## Create target
|
||||
returns = df_target_asset_only_returns[target_asset[1] + '_returns']
|
||||
|
||||
## Create target
|
||||
returns = df_target_asset_only_returns[target_asset[1] + "_returns"]
|
||||
returns.index = pd.DatetimeIndex(X.index)
|
||||
|
||||
return X, returns
|
||||
|
||||
return X, returns
|
||||
|
||||
|
||||
@ray.remote
|
||||
def __load_df(data_source: DataSource,
|
||||
prefix: str,
|
||||
returns: Literal['none', 'price', 'returns', 'log_returns'],
|
||||
feature_extractors: list[tuple[str, FeatureExtractor, list[int]]]) -> pd.DataFrame:
|
||||
df = pd.read_csv(os.path.join(data_source[0], data_source[1] + '.csv'), header=0, index_col=0).fillna(0)
|
||||
def __load_df(
|
||||
data_source: DataSource,
|
||||
prefix: str,
|
||||
returns: Literal["none", "price", "returns", "log_returns"],
|
||||
feature_extractors: list[tuple[str, FeatureExtractor, list[int]]],
|
||||
) -> pd.DataFrame:
|
||||
df = pd.read_csv(
|
||||
os.path.join(data_source[0], data_source[1] + ".csv"), header=0, index_col=0
|
||||
).fillna(0)
|
||||
|
||||
if returns == 'log_returns':
|
||||
df['returns'] = np.log(df['close']).diff(1)
|
||||
elif returns == 'price':
|
||||
df['returns'] = df['close']
|
||||
elif returns == 'returns':
|
||||
df['returns'] = df['close'].pct_change()
|
||||
if returns == "log_returns":
|
||||
df["returns"] = np.log(df["close"]).diff(1)
|
||||
elif returns == "price":
|
||||
df["returns"] = df["close"]
|
||||
elif returns == "returns":
|
||||
df["returns"] = df["close"].pct_change()
|
||||
|
||||
df = __apply_feature_extractors(df, feature_extractors = feature_extractors)
|
||||
df = __apply_feature_extractors(df, feature_extractors=feature_extractors)
|
||||
|
||||
df = df.replace([np.inf, -np.inf], 0.)
|
||||
df = drop_columns_if_exist(df, ['open', 'high', 'low', 'close', 'volume'])
|
||||
|
||||
df.columns = [prefix + "_" + c if 'date' not in c else c for c in df.columns]
|
||||
df = df.replace([np.inf, -np.inf], 0.0)
|
||||
df = drop_columns_if_exist(df, ["open", "high", "low", "close", "volume"])
|
||||
|
||||
df.columns = [prefix + "_" + c if "date" not in c else c for c in df.columns]
|
||||
return df
|
||||
|
||||
|
||||
def __apply_feature_extractors(df: pd.DataFrame, feature_extractors: list[tuple[str, FeatureExtractor, list[int]]]) -> pd.DataFrame:
|
||||
def __apply_feature_extractors(
|
||||
df: pd.DataFrame, feature_extractors: list[tuple[str, FeatureExtractor, list[int]]]
|
||||
) -> pd.DataFrame:
|
||||
|
||||
for name, extractor, periods in feature_extractors:
|
||||
for period in periods:
|
||||
@@ -120,20 +141,27 @@ def __apply_feature_extractors(df: pd.DataFrame, feature_extractors: list[tuple[
|
||||
if type(features) == pd.DataFrame:
|
||||
df = pd.concat([df, features], axis=1)
|
||||
elif type(features) == pd.Series:
|
||||
df[name + '_' + str(period)] = features
|
||||
df[name + "_" + str(period)] = features
|
||||
else:
|
||||
assert False, "Feature extractor must return a pd.DataFrame or pd.Series"
|
||||
assert (
|
||||
False
|
||||
), "Feature extractor must return a pd.DataFrame or pd.Series"
|
||||
return df
|
||||
|
||||
|
||||
def load_only_returns(assets: DataCollection, returns: Literal['price', 'returns']) -> pd.DataFrame:
|
||||
def load_only_returns(
|
||||
assets: DataCollection, returns: Literal["price", "returns"]
|
||||
) -> pd.DataFrame:
|
||||
|
||||
assets_future = [__load_df.remote(
|
||||
data_source=data_source,
|
||||
prefix=data_source[1],
|
||||
returns=returns,
|
||||
feature_extractors=[],
|
||||
) for data_source in assets]
|
||||
assets_future = [
|
||||
__load_df.remote(
|
||||
data_source=data_source,
|
||||
prefix=data_source[1],
|
||||
returns=returns,
|
||||
feature_extractors=[],
|
||||
)
|
||||
for data_source in assets
|
||||
]
|
||||
dfs = ray.get(assets_future)
|
||||
|
||||
dfs = pd.concat(dfs, axis=1)
|
||||
|
||||
Reference in New Issue
Block a user