gavindiaz/NexQuant
1
0
Fork 0
mirror of https://github.com/NicolasBohn/NexQuant.git synced 2026-08-02 09:57:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

align local data folder and kaggle data folder to store mid result (#314)

Browse Source
This commit is contained in:
Xu Yang
2024-09-24 17:01:33 +08:00
committed by GitHub
parent 6855dd81f8
commit 804e353549
9 changed files with 50 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rdagent/components/coder/factor_coder/factor.py
+1 -1
View File
@@ -138,7 +138,7 @@ class FactorFBWorkspace(FBWorkspace):
)
elif self.target_task.version == 2:
# TODO you can change the name of the data folder for a better understanding
source_data_path = Path(FACTOR_IMPLEMENT_SETTINGS.data_folder) / KAGGLE_IMPLEMENT_SETTING.competition
source_data_path = Path(KAGGLE_IMPLEMENT_SETTING.local_data_path) / KAGGLE_IMPLEMENT_SETTING.competition
source_data_path.mkdir(exist_ok=True, parents=True)
code_path = self.workspace_path / f"factor.py"
rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/fea_share_preprocess.py
+7 -7
View File
@@ -27,13 +27,13 @@ def preprocess_script():
"""
This method applies the preprocessing steps to the training, validation, and test datasets.
"""
if os.path.exists("X_train.pkl"):
X_train = pd.read_pickle("X_train.pkl")
X_valid = pd.read_pickle("X_valid.pkl")
y_train = pd.read_pickle("y_train.pkl")
y_valid = pd.read_pickle("y_valid.pkl")
X_test = pd.read_pickle("X_test.pkl")
others = pd.read_pickle("others.pkl")
if os.path.exists("/kaggle/input/X_train.pkl"):
X_train = pd.read_pickle("/kaggle/input/X_train.pkl")
X_valid = pd.read_pickle("/kaggle/input/X_valid.pkl")
y_train = pd.read_pickle("/kaggle/input/y_train.pkl")
y_valid = pd.read_pickle("/kaggle/input/y_valid.pkl")
X_test = pd.read_pickle("/kaggle/input/X_test.pkl")
others = pd.read_pickle("/kaggle/input/others.pkl")
return X_train, X_valid, y_train, y_valid, X_test, *others
rdagent/scenarios/kaggle/experiment/meta_tpl/fea_share_preprocess.py
+7 -7
View File
@@ -84,13 +84,13 @@ def preprocess_script():
"""
This method applies the preprocessing steps to the training, validation, and test datasets.
"""
if os.path.exists("/kaggle/preprocessed_data/X_train.pkl"):
X_train = pd.read_pickle("/kaggle/preprocessed_data/X_train.pkl")
X_valid = pd.read_pickle("/kaggle/preprocessed_data/X_valid.pkl")
y_train = pd.read_pickle("/kaggle/preprocessed_data/y_train.pkl")
y_valid = pd.read_pickle("/kaggle/preprocessed_data/y_valid.pkl")
X_test = pd.read_pickle("/kaggle/preprocessed_data/X_test.pkl")
others = pd.read_pickle("/kaggle/preprocessed_data/others.pkl")
if os.path.exists("/kaggle/input/X_train.pkl"):
X_train = pd.read_pickle("/kaggle/input/X_train.pkl")
X_valid = pd.read_pickle("/kaggle/input/X_valid.pkl")
y_train = pd.read_pickle("/kaggle/input/y_train.pkl")
y_valid = pd.read_pickle("/kaggle/input/y_valid.pkl")
X_test = pd.read_pickle("/kaggle/input/X_test.pkl")
others = pd.read_pickle("/kaggle/input/others.pkl")
return X_train, X_valid, y_train, y_valid, X_test, *others
X_train, X_valid, y_train, y_valid = prepreprocess()
rdagent/scenarios/kaggle/experiment/playground-series-s4e8_template/fea_share_preprocess.py
+7 -7
View File
@@ -84,13 +84,13 @@ def preprocess_script():
"""
This method applies the preprocessing steps to the training, validation, and test datasets.
"""
if os.path.exists("X_train.pkl"):
X_train = pd.read_pickle("X_train.pkl")
X_valid = pd.read_pickle("X_valid.pkl")
y_train = pd.read_pickle("y_train.pkl")
y_valid = pd.read_pickle("y_valid.pkl")
X_test = pd.read_pickle("X_test.pkl")
others = pd.read_pickle("others.pkl")
if os.path.exists("/kaggle/input/X_train.pkl"):
X_train = pd.read_pickle("/kaggle/input/X_train.pkl")
X_valid = pd.read_pickle("/kaggle/input/X_valid.pkl")
y_train = pd.read_pickle("/kaggle/input/y_train.pkl")
y_valid = pd.read_pickle("/kaggle/input/y_valid.pkl")
X_test = pd.read_pickle("/kaggle/input/X_test.pkl")
others = pd.read_pickle("/kaggle/input/others.pkl")
return X_train, X_valid, y_train, y_valid, X_test, *others
X_train, X_valid, y_train, y_valid = prepreprocess()
rdagent/scenarios/kaggle/experiment/playground-series-s4e9_template/fea_share_preprocess.py
+7 -7
View File
@@ -81,13 +81,13 @@ def preprocess_script():
"""
This method applies the preprocessing steps to the training, validation, and test datasets.
"""
if os.path.exists("X_train.pkl"):
X_train = pd.read_pickle("X_train.pkl")
X_valid = pd.read_pickle("X_valid.pkl")
y_train = pd.read_pickle("y_train.pkl")
y_valid = pd.read_pickle("y_valid.pkl")
X_test = pd.read_pickle("X_test.pkl")
others = pd.read_pickle("others.pkl")
if os.path.exists("/kaggle/input/X_train.pkl"):
X_train = pd.read_pickle("/kaggle/input/X_train.pkl")
X_valid = pd.read_pickle("/kaggle/input/X_valid.pkl")
y_train = pd.read_pickle("/kaggle/input/y_train.pkl")
y_valid = pd.read_pickle("/kaggle/input/y_valid.pkl")
X_test = pd.read_pickle("/kaggle/input/X_test.pkl")
others = pd.read_pickle("/kaggle/input/others.pkl")
return X_train, X_valid, y_train, y_valid, X_test, *others
X_train, X_valid, y_train, y_valid = prepreprocess()
rdagent/scenarios/kaggle/experiment/scenario.py
+1 -1
View File
@@ -97,7 +97,7 @@ Competition Features: {self.competition_features}
@property
def source_data(self) -> str:
data_folder = Path(FACTOR_IMPLEMENT_SETTINGS.data_folder) / self.competition
data_folder = Path(KAGGLE_IMPLEMENT_SETTING.local_data_path) / self.competition
if (data_folder / "X_valid.pkl").exists():
X_valid = pd.read_pickle(data_folder / "X_valid.pkl")
rdagent/scenarios/kaggle/experiment/sf-crime_template/fea_share_preprocess.py
+7 -7
View File
@@ -102,13 +102,13 @@ def preprocess_script():
"""
This method applies the preprocessing steps to the training, validation, and test datasets.
"""
if os.path.exists("X_train.pkl"):
X_train = pd.read_pickle("X_train.pkl")
X_valid = pd.read_pickle("X_valid.pkl")
y_train = pd.read_pickle("y_train.pkl")
y_valid = pd.read_pickle("y_valid.pkl")
X_test = pd.read_pickle("X_test.pkl")
others = pd.read_pickle("others.pkl")
if os.path.exists("/kaggle/input/X_train.pkl"):
X_train = pd.read_pickle("/kaggle/input/X_train.pkl")
X_valid = pd.read_pickle("/kaggle/input/X_valid.pkl")
y_train = pd.read_pickle("/kaggle/input/y_train.pkl")
y_valid = pd.read_pickle("/kaggle/input/y_valid.pkl")
X_test = pd.read_pickle("/kaggle/input/X_test.pkl")
others = pd.read_pickle("/kaggle/input/others.pkl")
return X_train, X_valid, y_train, y_valid, X_test, *others
X_train, X_valid, y_train, y_valid, test, category_encoder, test_ids = prepreprocess()
rdagent/scenarios/kaggle/experiment/spaceship-titanic_template/fea_share_preprocess.py
+7 -7
View File
@@ -78,13 +78,13 @@ def preprocess_script():
"""
This method applies the preprocessing steps to the training, validation, and test datasets.
"""
if os.path.exists("X_train.pkl"):
X_train = pd.read_pickle("X_train.pkl")
X_valid = pd.read_pickle("X_valid.pkl")
y_train = pd.read_pickle("y_train.pkl")
y_valid = pd.read_pickle("y_valid.pkl")
X_test = pd.read_pickle("X_test.pkl")
others = pd.read_pickle("others.pkl")
if os.path.exists("/kaggle/input/X_train.pkl"):
X_train = pd.read_pickle("/kaggle/input/X_train.pkl")
X_valid = pd.read_pickle("/kaggle/input/X_valid.pkl")
y_train = pd.read_pickle("/kaggle/input/y_train.pkl")
y_valid = pd.read_pickle("/kaggle/input/y_valid.pkl")
X_test = pd.read_pickle("/kaggle/input/X_test.pkl")
others = pd.read_pickle("/kaggle/input/others.pkl")
return X_train, X_valid, y_train, y_valid, X_test, *others
X_train, X_valid, y_train, y_valid = prepreprocess()
rdagent/scenarios/kaggle/experiment/workspace.py
+6 -8
View File
@@ -69,15 +69,13 @@ class KGFBWorkspace(FBWorkspace):
kgde = KGDockerEnv(KAGGLE_IMPLEMENT_SETTING.competition)
kgde.prepare()
running_extra_volume = {
(
Path(FACTOR_IMPLEMENT_SETTINGS.data_folder) / KAGGLE_IMPLEMENT_SETTING.competition
).absolute(): "/kaggle/preprocessed_data"
}
running_extra_volume = {}
if KAGGLE_IMPLEMENT_SETTING.competition:
running_extra_volume[
KAGGLE_IMPLEMENT_SETTING.local_data_path + "/" + KAGGLE_IMPLEMENT_SETTING.competition
] = "/kaggle/input"
running_extra_volume = {
KAGGLE_IMPLEMENT_SETTING.local_data_path + "/" + KAGGLE_IMPLEMENT_SETTING.competition: "/kaggle/input"
}
else:
running_extra_volume = {}
execute_log = kgde.run(
local_path=str(self.workspace_path),