diff --git a/rdagent/scenarios/kaggle/developer/runner.py b/rdagent/scenarios/kaggle/developer/runner.py index 0603d82c..42e5541d 100644 --- a/rdagent/scenarios/kaggle/developer/runner.py +++ b/rdagent/scenarios/kaggle/developer/runner.py @@ -41,9 +41,60 @@ class KGCachedRunner(CachedRunner[ASpecificExp]): codes = "\n".join(codes) return md5_hash(codes) + def init_develop(self, exp: KGFactorExperiment | KGModelExperiment) -> KGFactorExperiment | KGModelExperiment: + """ + For the initial development, the experiment serves as a benchmark for feature engineering. + """ + self.build_from_SOTA(exp) + if RUNNER_SETTINGS.cache_result: + cache_hit, result = self.get_cache_result(exp) + if cache_hit: + exp.result = result + return exp + + env_to_use = {"PYTHONPATH": "./"} + + result = exp.experiment_workspace.execute(run_env=env_to_use) + + exp.result = result + sub_task = FactorTask( + factor_name="original features", factor_description="here is the original features", factor_formulation="" + ) + + org_data_path = ( + Path(FACTOR_IMPLEMENT_SETTINGS.data_folder) / KAGGLE_IMPLEMENT_SETTING.competition / "X_valid.pkl" + ) + with open(org_data_path, "rb") as f: + org_data = pickle.load(f) + feature_shape = org_data.shape[-1] + exp.experiment_workspace.data_description.append((sub_task.get_task_information(), feature_shape)) + + sub_model_1_description = ( + self.extract_model_task_from_code( + (exp.experiment_workspace.workspace_path / "model" / "model_randomforest.py").read_text() + ) + + f"""code: { (exp.experiment_workspace.workspace_path / "model" / "model_randomforest.py").read_text()}""" + ) + sub_model_2_description = ( + self.extract_model_task_from_code( + (exp.experiment_workspace.workspace_path / "model" / "model_xgboost.py").read_text() + ) + + f"""code: { (exp.experiment_workspace.workspace_path / "model" / "model_xgboost.py").read_text()}""" + ) + + exp.experiment_workspace.model_description["XGBoost"] = sub_model_1_description + exp.experiment_workspace.model_description["RandomForest"] = sub_model_2_description + + if RUNNER_SETTINGS.cache_result: + self.dump_cache_result(exp, result) + + return exp + class KGModelRunner(KGCachedRunner[KGModelExperiment]): def develop(self, exp: KGModelExperiment) -> KGModelExperiment: + if exp.based_experiments and exp.based_experiments[-1].result is None: + exp.based_experiments[-1] = self.init_develop(exp.based_experiments[-1]) self.build_from_SOTA(exp) sub_ws = exp.sub_workspace_list[0] @@ -118,55 +169,6 @@ class KGFactorRunner(KGCachedRunner[KGFactorExperiment]): return task_desc - def init_develop(self, exp: KGFactorExperiment) -> KGFactorExperiment: - """ - For the initial development, the experiment serves as a benchmark for feature engineering. - """ - self.build_from_SOTA(exp) - if RUNNER_SETTINGS.cache_result: - cache_hit, result = self.get_cache_result(exp) - if cache_hit: - exp.result = result - return exp - - env_to_use = {"PYTHONPATH": "./"} - - result = exp.experiment_workspace.execute(run_env=env_to_use) - - exp.result = result - sub_task = FactorTask( - factor_name="original features", factor_description="here is the original features", factor_formulation="" - ) - - org_data_path = ( - Path(FACTOR_IMPLEMENT_SETTINGS.data_folder) / KAGGLE_IMPLEMENT_SETTING.competition / "X_valid.pkl" - ) - with open(org_data_path, "rb") as f: - org_data = pickle.load(f) - feature_shape = org_data.shape[-1] - exp.experiment_workspace.data_description.append((sub_task.get_task_information(), feature_shape)) - - sub_model_1_description = ( - self.extract_model_task_from_code( - (exp.experiment_workspace.workspace_path / "model" / "model_randomforest.py").read_text() - ) - + f"""code: { (exp.experiment_workspace.workspace_path / "model" / "model_randomforest.py").read_text()}""" - ) - sub_model_2_description = ( - self.extract_model_task_from_code( - (exp.experiment_workspace.workspace_path / "model" / "model_xgboost.py").read_text() - ) - + f"""code: { (exp.experiment_workspace.workspace_path / "model" / "model_xgboost.py").read_text()}""" - ) - - exp.experiment_workspace.model_description["XGBoost"] = sub_model_1_description - exp.experiment_workspace.model_description["RandomForest"] = sub_model_2_description - - if RUNNER_SETTINGS.cache_result: - self.dump_cache_result(exp, result) - - return exp - def develop(self, exp: KGFactorExperiment) -> KGFactorExperiment: if exp.based_experiments and exp.based_experiments[-1].result is None: exp.based_experiments[-1] = self.init_develop(exp.based_experiments[-1]) diff --git a/rdagent/scenarios/kaggle/proposal/proposal.py b/rdagent/scenarios/kaggle/proposal/proposal.py index 0765c0a9..1899b530 100644 --- a/rdagent/scenarios/kaggle/proposal/proposal.py +++ b/rdagent/scenarios/kaggle/proposal/proposal.py @@ -334,7 +334,7 @@ class KGHypothesis2Experiment(ModelHypothesis2Experiment): ) ) exp = KGModelExperiment(tasks) - exp.based_experiments = [t[1] for t in trace.hist if t[2]] + exp.based_experiments = [KGModelExperiment(sub_tasks=[])] + [t[1] for t in trace.hist if t[2]] return exp def convert_response(self, response: str, trace: Trace) -> ModelExperiment: