From 6809154ed7735908e7b3370a9fc8b90755d5a831 Mon Sep 17 00:00:00 2001 From: Xu Yang Date: Fri, 15 Nov 2024 15:40:22 +0800 Subject: [PATCH] feat: align mlebench data and evaluation & several fix on kaggle workflow (#477) * several improvement on kaggle loop * small refinement on prompt * fix bugs * add the score of each model in every experiment * fix ci error * fix error in ventilator tpl * fix CI --------- Co-authored-by: Xu Yang Co-authored-by: Bowen Xian Co-authored-by: WinstonLiye <1957922024@qq.com> Co-authored-by: TPLin22 --- rdagent/app/kaggle/loop.py | 5 +- .../coder/model_coder/CoSTEER/evaluators.py | 5 +- .../components/knowledge_management/graph.py | 1 + rdagent/components/proposal/__init__.py | 2 +- rdagent/components/runner/__init__.py | 7 - rdagent/core/experiment.py | 1 + rdagent/core/utils.py | 2 +- rdagent/log/ui/app.py | 4 +- .../scenarios/kaggle/developer/feedback.py | 107 +++++++------ rdagent/scenarios/kaggle/developer/runner.py | 34 ++++- .../docker/{ => kaggle_docker}/Dockerfile | 3 +- .../kaggle/docker/mle_bench_docker/Dockerfile | 17 +++ .../scenarios/kaggle/experiment/prompts.yaml | 16 +- .../scenarios/kaggle/experiment/scenario.py | 5 +- .../experiment/sf-crime_template/train.py | 48 +++++- .../spaceship-titanic_template/train.py | 39 ++++- .../model/model_xgboost.py | 2 +- .../model/select_lightgbm.py | 12 ++ .../train.py | 9 +- .../model/select_lightgbm.py | 12 ++ .../train.py | 5 +- rdagent/scenarios/kaggle/kaggle_crawler.py | 37 +++-- rdagent/scenarios/kaggle/prompts.yaml | 140 ++++++------------ rdagent/scenarios/kaggle/proposal/proposal.py | 27 ++-- rdagent/utils/env.py | 33 ++++- 25 files changed, 364 insertions(+), 209 deletions(-) rename rdagent/scenarios/kaggle/docker/{ => kaggle_docker}/Dockerfile (92%) create mode 100644 rdagent/scenarios/kaggle/docker/mle_bench_docker/Dockerfile create mode 100644 rdagent/scenarios/kaggle/experiment/tabular-playground-series-may-2022_template/model/select_lightgbm.py create mode 100644 rdagent/scenarios/kaggle/experiment/ventilator-pressure-prediction_template/model/select_lightgbm.py diff --git a/rdagent/app/kaggle/loop.py b/rdagent/app/kaggle/loop.py index ffa6001e..2c66d668 100644 --- a/rdagent/app/kaggle/loop.py +++ b/rdagent/app/kaggle/loop.py @@ -1,6 +1,4 @@ import subprocess -from collections import defaultdict -from concurrent.futures import TimeoutError from typing import Any import fire @@ -14,7 +12,6 @@ from rdagent.core.proposal import ( Hypothesis2Experiment, HypothesisExperiment2Feedback, HypothesisGen, - Trace, ) from rdagent.core.scenario import Scenario from rdagent.core.utils import import_class @@ -116,7 +113,7 @@ class KaggleRDLoop(RDLoop): return exp - skip_loop_error = (ModelEmptyError, FactorEmptyError, TimeoutError) + skip_loop_error = (ModelEmptyError, FactorEmptyError) def main(path=None, step_n=None, competition=None): diff --git a/rdagent/components/coder/model_coder/CoSTEER/evaluators.py b/rdagent/components/coder/model_coder/CoSTEER/evaluators.py index 136049d9..3634f807 100644 --- a/rdagent/components/coder/model_coder/CoSTEER/evaluators.py +++ b/rdagent/components/coder/model_coder/CoSTEER/evaluators.py @@ -283,7 +283,10 @@ class ModelCoderEvaluator(Evaluator): else: gt_np_array = None - shape_feedback, shape_decision = shape_evaluator(gen_np_array, (batch_size, 1)) + shape_feedback, shape_decision = shape_evaluator( + gen_np_array, + (batch_size, self.scen.model_output_channel if hasattr(self.scen, "model_output_channel") else 1), + ) value_feedback, value_decision = value_evaluator(gen_np_array, gt_np_array) code_feedback, _ = ModelCodeEvaluator(scen=self.scen).evaluate( target_task=target_task, diff --git a/rdagent/components/knowledge_management/graph.py b/rdagent/components/knowledge_management/graph.py index 50b408ac..9ea5b7a5 100644 --- a/rdagent/components/knowledge_management/graph.py +++ b/rdagent/components/knowledge_management/graph.py @@ -22,6 +22,7 @@ class UndirectedNode(Node): def __init__(self, content: str = "", label: str = "", embedding: Any = None) -> None: super().__init__(content, label, embedding) self.neighbors: set[UndirectedNode] = set() + assert isinstance(content, str), "content must be a string" def add_neighbor(self, node: UndirectedNode) -> None: self.neighbors.add(node) diff --git a/rdagent/components/proposal/__init__.py b/rdagent/components/proposal/__init__.py index f2e2462c..846f93cf 100644 --- a/rdagent/components/proposal/__init__.py +++ b/rdagent/components/proposal/__init__.py @@ -72,7 +72,7 @@ class ModelHypothesisGen(LLMHypothesisGen): self.targets = "model tuning" -class FactorAndModelHypothesisGen(FactorHypothesisGen): +class FactorAndModelHypothesisGen(LLMHypothesisGen): def __init__(self, scen: Scenario): super().__init__(scen) self.targets = "feature engineering and model building" diff --git a/rdagent/components/runner/__init__.py b/rdagent/components/runner/__init__.py index 0f62bfce..18a53886 100644 --- a/rdagent/components/runner/__init__.py +++ b/rdagent/components/runner/__init__.py @@ -21,12 +21,5 @@ class CachedRunner(Developer[ASpecificExp]): def assign_cached_result(self, exp: Experiment, cached_res: Experiment) -> Experiment: if exp.based_experiments and exp.based_experiments[-1].result is None: exp.based_experiments[-1].result = cached_res.based_experiments[-1].result - if cached_res.experiment_workspace.workspace_path.exists(): - for csv_file in cached_res.experiment_workspace.workspace_path.glob("*.csv"): - shutil.copy(csv_file, exp.experiment_workspace.workspace_path) - for py_file in (cached_res.experiment_workspace.workspace_path / "feature").glob("*.py"): - shutil.copy(py_file, exp.experiment_workspace.workspace_path / "feature") - for py_file in (cached_res.experiment_workspace.workspace_path / "model").glob("*.py"): - shutil.copy(py_file, exp.experiment_workspace.workspace_path / "model") exp.result = cached_res.result return exp diff --git a/rdagent/core/experiment.py b/rdagent/core/experiment.py index 20b42bd7..0cf49c90 100644 --- a/rdagent/core/experiment.py +++ b/rdagent/core/experiment.py @@ -210,6 +210,7 @@ class Experiment( self.sub_workspace_list: list[ASpecificWSForSubTasks | None] = [None] * len(self.sub_tasks) self.based_experiments: Sequence[ASpecificWSForExperiment] = based_experiments self.result: object = None # The result of the experiment, can be different types in different scenarios. + self.sub_results: dict[str, float] = {} self.experiment_workspace: ASpecificWSForExperiment | None = None diff --git a/rdagent/core/utils.py b/rdagent/core/utils.py index 3967f507..be0a15cf 100644 --- a/rdagent/core/utils.py +++ b/rdagent/core/utils.py @@ -142,7 +142,7 @@ def multiprocessing_wrapper(func_calls: list[tuple[Callable, tuple]], n: int) -> list """ - if n == 1: + if n == 1 or max(1, min(n, len(func_calls))) == 1: return [f(*args) for f, args in func_calls] with mp.Pool(processes=max(1, min(n, len(func_calls)))) as pool: diff --git a/rdagent/log/ui/app.py b/rdagent/log/ui/app.py index 8fe2d47a..57f13327 100644 --- a/rdagent/log/ui/app.py +++ b/rdagent/log/ui/app.py @@ -409,7 +409,7 @@ def summary_window(): if state.alpha158_metrics is not None: selected = ["alpha158"] + [i for i in df.index if state.h_decisions[int(i[6:])]] else: - selected = [i for i in df.index if state.h_decisions[int(i[6:])]] + selected = [i for i in df.index if i == "Baseline" or state.h_decisions[int(i[6:])]] df = df.loc[selected] if df.shape[0] == 1: st.table(df.iloc[0]) @@ -637,6 +637,7 @@ def evolving_window(): for j, w in enumerate(ws): with wtabs[j]: # Evolving Code + st.markdown(f"**Workspace Path**: {w.workspace_path}") for k, v in w.code_dict.items(): with st.expander(f":green[`{k}`]", expanded=True): st.code(v, language="python") @@ -681,6 +682,7 @@ with st.sidebar: st.text_input("log path", key="log_path", on_change=refresh, label_visibility="collapsed") else: folders = [folder.relative_to(main_log_path) for folder in main_log_path.iterdir() if folder.is_dir()] + folders = sorted(folders, key=lambda x: x.name) st.selectbox(f"**Select from `{main_log_path}`**", folders, key="log_path", on_change=refresh) else: st.text_input(":blue[**log path**]", key="log_path", on_change=refresh) diff --git a/rdagent/scenarios/kaggle/developer/feedback.py b/rdagent/scenarios/kaggle/developer/feedback.py index 9fe68957..708c74e7 100644 --- a/rdagent/scenarios/kaggle/developer/feedback.py +++ b/rdagent/scenarios/kaggle/developer/feedback.py @@ -4,6 +4,7 @@ from pathlib import Path import pandas as pd from jinja2 import Environment, StrictUndefined +from rdagent.components.knowledge_management.graph import UndirectedNode from rdagent.core.experiment import Experiment from rdagent.core.prompts import Prompts from rdagent.core.proposal import ( @@ -14,6 +15,7 @@ from rdagent.core.proposal import ( ) from rdagent.log import rdagent_logger as logger from rdagent.oai.llm_utils import APIBackend +from rdagent.scenarios.kaggle.experiment.kaggle_experiment import KG_SELECT_MAPPING from rdagent.utils import convert2bool prompt_dict = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml") @@ -59,17 +61,7 @@ class KGHypothesisExperiment2Feedback(HypothesisExperiment2Feedback): Any: The feedback generated for the given experiment and hypothesis. """ logger.info("Generating feedback...") - hypothesis_text = hypothesis.hypothesis current_result = exp.result - tasks_factors = [] - if exp.sub_tasks: - tasks_factors = [] - for task in exp.sub_tasks: - try: - task_info = task.get_task_information_and_implementation_result() - tasks_factors.append(task_info) - except AttributeError: - print(f"Warning: Task {task} does not have get_task_information_and_implementation_result method") evaluation_description = None # Check if there are any based experiments @@ -84,11 +76,6 @@ class KGHypothesisExperiment2Feedback(HypothesisExperiment2Feedback): ) # Compare with itself print("Warning: No previous experiments to compare against. Using current result as baseline.") - available_features = { - task_info: feature_shape for task_info, feature_shape in exp.experiment_workspace.data_description - } - model_code = exp.experiment_workspace.model_description - # Generate the user prompt based on the action type if hypothesis.action == "Model tuning": prompt_key = "model_tuning_feedback_generation" @@ -104,35 +91,56 @@ class KGHypothesisExperiment2Feedback(HypothesisExperiment2Feedback): .render(scenario=self.scen.get_scenario_all_desc(filtered_tag="feedback")) ) - last_task_and_code = None - if trace.hist: - last_task_and_code = ( - trace.hist[-1][1].experiment_workspace.data_description - if trace.hist[-1][0].action == "Feature engineering" or trace.hist[-1][0].action == "Feature processing" - else trace.hist[-1][1].experiment_workspace.model_description - ) + sota_exp = exp.based_experiments[-1] if exp.based_experiments else None + assert sota_exp is not None + sota_features = str(exp.based_experiments[-1].experiment_workspace.data_description) + sota_models = json.dumps(exp.based_experiments[-1].experiment_workspace.model_description, indent=2) + sota_result = exp.based_experiments[-1].result + sota_sub_results = exp.based_experiments[-1].sub_results + + current_hypothesis = hypothesis.hypothesis + current_hypothesis_reason = hypothesis.reason + current_target_action = hypothesis.action + current_sub_exps_to_code = {} + if hypothesis.action == "Model tuning": + current_sub_exps_to_code[exp.sub_tasks[0].get_task_information()] = exp.sub_workspace_list[0].code + elif hypothesis.action == "Model feature selection": + current_sub_exps_to_code[exp.sub_tasks[0].get_task_information()] = exp.experiment_workspace.code_dict[ + KG_SELECT_MAPPING[exp.sub_tasks[0].model_type] + ] + else: + current_sub_exps_to_code = { + sub_ws.target_task.get_task_information(): sub_ws.code for sub_ws in exp.sub_workspace_list + } + current_sub_exps_to_code_str = json.dumps(current_sub_exps_to_code, indent=2) + current_result = exp.result + current_sub_results = exp.sub_results + + last_hypothesis_and_feedback = None + if trace.hist and len(trace.hist) > 0: + last_hypothesis_and_feedback = (trace.hist[-1][0], trace.hist[-1][2]) # Prepare render dictionary render_dict = { - "last_hypothesis": trace.hist[-1][0] if trace.hist else None, - "last_task_and_code": last_task_and_code, - "last_result": trace.hist[-1][1].result if trace.hist else None, - "sota_task_and_code": ( - exp.based_experiments[-1].experiment_workspace.data_description if exp.based_experiments else None - ), - "sota_result": exp.based_experiments[-1].result if exp.based_experiments else None, - "hypothesis": hypothesis, - "exp": exp, - "model_code": model_code, # This turn - "available_features": available_features, # This turn - "combined_result": combined_result, # This turn and sota - "hypothesis_text": hypothesis_text, # This turn - "task_details": tasks_factors, # This turn + "sota_features": sota_features, + "sota_models": sota_models, + "sota_result": sota_result, + "sota_sub_results": sota_sub_results, + "current_hypothesis": current_hypothesis, + "current_hypothesis_reason": current_hypothesis_reason, + "current_target_action": current_target_action, + "current_sub_exps_to_code": current_sub_exps_to_code_str, + "current_result": current_result, + "current_sub_results": current_sub_results, + "combined_result": combined_result, "evaluation_description": evaluation_description, + "last_hypothesis_and_feedback": last_hypothesis_and_feedback, } usr_prompt = ( - Environment(undefined=StrictUndefined).from_string(prompt_dict[prompt_key]["user"]).render(**render_dict) + Environment(undefined=StrictUndefined) + .from_string(prompt_dict["kg_feedback_generation_user"]) + .render(**render_dict) ) response = APIBackend().build_messages_and_create_chat_completion( @@ -160,22 +168,29 @@ class KGHypothesisExperiment2Feedback(HypothesisExperiment2Feedback): percentile_ranking = (insert_position) / (len(sorted_scores)) * 100 experiment_feedback = { - "current_competition": self.scen.get_competition_full_desc(), - "hypothesis_text": hypothesis_text, + "hypothesis_text": current_hypothesis, + "tasks_factors": current_sub_exps_to_code, "current_result": current_result, - "model_code": model_code, - "available_features": available_features, - "observations": observations, - "hypothesis_evaluation": hypothesis_evaluation, - "reason": reason, - "percentile_ranking": percentile_ranking, } if self.scen.if_using_vector_rag: + raise NotImplementedError("Vector RAG is not implemented yet since there are plenty bugs!") self.scen.vector_base.add_experience_to_vector_base(experiment_feedback) self.scen.vector_base.dump() elif self.scen.if_using_graph_rag: - trace.knowledge_base.add_document(experiment_feedback, self.scen) + competition_node = UndirectedNode(content=self.scen.get_competition_full_desc(), label="competition") + hypothesis_node = UndirectedNode(content=hypothesis.hypothesis, label=hypothesis.action) + exp_code_nodes = [] + for exp, code in current_sub_exps_to_code.items(): + exp_code_nodes.append(UndirectedNode(content=exp, label="experiments")) + if code != "": + exp_code_nodes.append(UndirectedNode(content=code, label="code")) + conclusion_node = UndirectedNode(content=response, label="conclusion") + all_nodes = [competition_node, hypothesis_node, *exp_code_nodes, conclusion_node] + all_nodes = trace.knowledge_base.batch_embedding(all_nodes) + for node in all_nodes: + if node is not competition_node: + trace.knowledge_base.add_node(node, competition_node) if self.scen.if_action_choosing_based_on_UCB: self.scen.action_counts[hypothesis.action] += 1 diff --git a/rdagent/scenarios/kaggle/developer/runner.py b/rdagent/scenarios/kaggle/developer/runner.py index ba6b0144..b7cc4815 100644 --- a/rdagent/scenarios/kaggle/developer/runner.py +++ b/rdagent/scenarios/kaggle/developer/runner.py @@ -3,9 +3,11 @@ import pickle import shutil from pathlib import Path +import pandas as pd + from rdagent.components.runner import CachedRunner from rdagent.core.exception import CoderError, FactorEmptyError, ModelEmptyError -from rdagent.core.experiment import ASpecificExp +from rdagent.core.experiment import ASpecificExp, Experiment from rdagent.core.prompts import Prompts from rdagent.core.utils import cache_with_pickle from rdagent.oai.llm_utils import md5_hash @@ -28,6 +30,18 @@ class KGCachedRunner(CachedRunner[ASpecificExp]): cached_key_from_exp = CachedRunner.get_cache_key(self, exp) return md5_hash(codes + cached_key_from_exp) + def assign_cached_result(self, exp: Experiment, cached_res: Experiment) -> Experiment: + exp = CachedRunner.assign_cached_result(self, exp, cached_res) + if cached_res.experiment_workspace.workspace_path.exists(): + for csv_file in cached_res.experiment_workspace.workspace_path.glob("*.csv"): + shutil.copy(csv_file, exp.experiment_workspace.workspace_path) + for py_file in (cached_res.experiment_workspace.workspace_path / "feature").glob("*.py"): + shutil.copy(py_file, exp.experiment_workspace.workspace_path / "feature") + for py_file in (cached_res.experiment_workspace.workspace_path / "model").glob("*.py"): + shutil.copy(py_file, exp.experiment_workspace.workspace_path / "model") + exp.experiment_workspace.data_description = cached_res.experiment_workspace.data_description + return exp + @cache_with_pickle(get_cache_key, CachedRunner.assign_cached_result) def init_develop(self, exp: KGFactorExperiment | KGModelExperiment) -> KGFactorExperiment | KGModelExperiment: """ @@ -40,6 +54,11 @@ class KGCachedRunner(CachedRunner[ASpecificExp]): exp.result = result + sub_result_score_path = Path(exp.experiment_workspace.workspace_path) / "sub_submission_score.csv" + if sub_result_score_path.exists(): + sub_submission_df = pd.read_csv(sub_result_score_path) + exp.sub_results = sub_submission_df.set_index("Model")["score"].to_dict() + return exp @@ -67,6 +86,10 @@ class KGModelRunner(KGCachedRunner[KGModelExperiment]): raise CoderError("No result is returned from the experiment workspace") exp.result = result + sub_result_score_path = Path(exp.experiment_workspace.workspace_path) / "sub_submission_score.csv" + if sub_result_score_path.exists(): + sub_submission_df = pd.read_csv(sub_result_score_path) + exp.sub_results = sub_submission_df.set_index("Model")["score"].to_dict() return exp @@ -79,10 +102,13 @@ class KGFactorRunner(KGCachedRunner[KGFactorExperiment]): for sub_ws in exp.sub_workspace_list: if sub_ws.code_dict == {}: continue + execued_df = sub_ws.execute()[1] + if execued_df is None: + continue implemented_factor_count += 1 target_feature_file_name = f"feature/feature_{current_feature_file_count:05d}.py" exp.experiment_workspace.inject_code(**{target_feature_file_name: sub_ws.code_dict["factor.py"]}) - feature_shape = sub_ws.execute()[1].shape[-1] + feature_shape = execued_df.shape[-1] exp.experiment_workspace.data_description.append((sub_ws.target_task.get_task_information(), feature_shape)) current_feature_file_count += 1 if implemented_factor_count == 0: @@ -100,5 +126,9 @@ class KGFactorRunner(KGCachedRunner[KGFactorExperiment]): raise CoderError("No result is returned from the experiment workspace") exp.result = result + sub_result_score_path = Path(exp.experiment_workspace.workspace_path) / "sub_submission_score.csv" + if sub_result_score_path.exists(): + sub_submission_df = pd.read_csv(sub_result_score_path) + exp.sub_results = sub_submission_df.set_index("Model")["score"].to_dict() return exp diff --git a/rdagent/scenarios/kaggle/docker/Dockerfile b/rdagent/scenarios/kaggle/docker/kaggle_docker/Dockerfile similarity index 92% rename from rdagent/scenarios/kaggle/docker/Dockerfile rename to rdagent/scenarios/kaggle/docker/kaggle_docker/Dockerfile index 41eadcd7..3515a922 100644 --- a/rdagent/scenarios/kaggle/docker/Dockerfile +++ b/rdagent/scenarios/kaggle/docker/kaggle_docker/Dockerfile @@ -17,13 +17,14 @@ RUN python -m pip install numpy RUN python -m pip install pandas # RUN pip install pyg_lib torch_scatter torch_sparse torch_cluster -f https://data.pyg.org/whl/torch-2.3.0%2Bcu121.html RUN pip install torch_geometric +RUN pip install pytorch_lightning RUN pip install ogb RUN pip install networkx RUN pip install scikit-learn RUN pip install catboost RUN pip install xgboost RUN pip install sparse -RUN pip install lightgbm +RUN pip install lightgbm==3.3.5 RUN pip install pyarrow RUN pip install fastparquet RUN pip install optuna \ No newline at end of file diff --git a/rdagent/scenarios/kaggle/docker/mle_bench_docker/Dockerfile b/rdagent/scenarios/kaggle/docker/mle_bench_docker/Dockerfile new file mode 100644 index 00000000..22796935 --- /dev/null +++ b/rdagent/scenarios/kaggle/docker/mle_bench_docker/Dockerfile @@ -0,0 +1,17 @@ +FROM pytorch/pytorch:2.4.0-cuda12.4-cudnn9-runtime +# For GPU support, please choose the proper tag from https://hub.docker.com/r/pytorch/pytorch/tags + +RUN apt-get clean && apt-get update && apt-get install -y \ + curl \ + vim \ + git \ + build-essential \ + git-lfs \ + && rm -rf /var/lib/apt/lists/* + +RUN git clone https://github.com/openai/mle-bench.git +RUN cd mle-bench && git lfs fetch --all +RUN cd mle-bench && git lfs pull +RUN cd mle-bench && python -m pip install -e . + +WORKDIR /workspace diff --git a/rdagent/scenarios/kaggle/experiment/prompts.yaml b/rdagent/scenarios/kaggle/experiment/prompts.yaml index 92ec9b51..bead19f7 100644 --- a/rdagent/scenarios/kaggle/experiment/prompts.yaml +++ b/rdagent/scenarios/kaggle/experiment/prompts.yaml @@ -37,6 +37,7 @@ kg_background: |- The final output of our pipeline is from a ensemble of up to four models. Each model is trained on a different subset of the data. The four model types are: XGBoost, RandomForest, LightGBM and Neural Network (A Pytorch model). + About the Neural Network model, You can try different architectures and hyperparameters to improve the performance. You can even use a pytorch model to ensemble the other three types of models. Try to open your mind on the NN model. The data is extracted from the competition dataset, focusing on relevant attributes in {{ competition_features }}. @@ -65,7 +66,10 @@ kg_background: |- - "Feature processing": The user will design a new task to process the feature book like normalization or one hot encoding to improve the model performance. Any processing with help of a deep model is not included in this task. - Model related: - "Model feature selection": The user will modify one model to select the part of the features from the feature book to improve the model performance. - - "Model tuning": The user will tune the hyperparameters of XGBoost, RandomForest or LightGBM or build or improve the NN model to improve the model performance. + - "Model tuning": The user will tune the hyperparameters of XGBoost, RandomForest or LightGBM or build or improve the NN model to improve the model performance. + Notice: You can automatically optimize the hyperparameters of the model using some library when training the model. Since we don't have a lot of time to train the model, please use a small number of trials to optimize the hyperparameters. + Our validation set split is not deterministic, so when you are using hyperparameter tuning, you can merge training and validation and use cross validation method to tune the hyperparameters. + One you have determine the best model parameter, you should retrain the model on all training and validation set to get the final model. For each loop, you need to help user decide which action item to choose and provide the corresponding code to implement the action item. @@ -81,6 +85,8 @@ kg_feature_interface: |- The input to 'fit' is the training data in pandas dataframe, and the input to 'transform' is the data to be transformed in pandas dataframe. The original columns should be excluded from the returned DataFrame. + Notice: Since we have a very big dataset, the feature engineering should be efficient and fast. Otherwise, please sufficiently exploit the multiprocessing or parallel computing to speed up the feature engineering process! + Exception handling will be managed externally, so avoid using try-except blocks in your code. The user will handle any exceptions that arise and provide feedback as needed. The feat_eng function can be one of the following: @@ -138,8 +144,8 @@ kg_model_interface: |- - model: The trained model. - X: The features as a pandas DataFrame. The function should return the predicted probabilities or boolean predictions in numpy.ndarray format. - """ - + Please refer to the train.py script to verify whether the output should be a class label or a probability! + Here are some examples of how your Python code should be structured: {% if tag == "XGBoost" or tag is none %} @@ -302,8 +308,8 @@ kg_feature_output_format: |- memory usage: {Memory usage of the output DataFrame} kg_model_output_format: |- - For feature related tasks, the output should be a pandas DataFrame with the new features. The columns should be the new features, and the rows should correspond to the number of samples in the input DataFrame. - For model related tasks, the output should be an np.ndarray with the appropriate number of predictions. + For model related tasks, the output should be an np.ndarray with the appropriate number of predictions. + Please refer to the train.py script to verify whether the output should be a class label or a probability! {% if channel == 1 %} For each sample, the output should be a single value (e.g., (8, 1) if there are 8 samples). {% else %} diff --git a/rdagent/scenarios/kaggle/experiment/scenario.py b/rdagent/scenarios/kaggle/experiment/scenario.py index 6c8039ce..b5a7f84e 100644 --- a/rdagent/scenarios/kaggle/experiment/scenario.py +++ b/rdagent/scenarios/kaggle/experiment/scenario.py @@ -285,8 +285,9 @@ To automatically optimize performance metrics within the validation set or Kaggl {self.simulator(tag)} """ - assert filtered_tag is not None, "filtered_tag should not be None in Kaggle scenario" - if filtered_tag == "hypothesis_and_experiment" or filtered_tag == "feedback": + if filtered_tag is None: + return common_description() + interface(None) + output(None) + simulator(None) + elif filtered_tag == "hypothesis_and_experiment" or filtered_tag == "feedback": return common_description() + simulator(None) elif filtered_tag == "feature": return common_description() + interface("feature") + output("feature") + simulator("feature") diff --git a/rdagent/scenarios/kaggle/experiment/sf-crime_template/train.py b/rdagent/scenarios/kaggle/experiment/sf-crime_template/train.py index 8a5e08fd..a935e3e4 100644 --- a/rdagent/scenarios/kaggle/experiment/sf-crime_template/train.py +++ b/rdagent/scenarios/kaggle/experiment/sf-crime_template/train.py @@ -17,7 +17,7 @@ DIRNAME = Path(__file__).absolute().resolve().parent # Support various method for metrics calculation def compute_metrics_for_classification(y_true, y_pred): """Compute log loss for classification.""" - all_classes = np.unique(y_true) + all_classes = np.arange(39) logloss = log_loss(y_true, y_pred, labels=all_classes) return logloss @@ -32,6 +32,11 @@ def import_module_from_path(module_name, module_path): # 1) Preprocess the data X_train, X_valid, y_train, y_valid, X_test, category_encoder, test_ids = preprocess_script() +X_train = X_train.iloc[: X_train.shape[0] // 10] +y_train = y_train.iloc[: y_train.shape[0] // 10] +X_valid = X_valid.iloc[: X_valid.shape[0] // 10] +y_valid = y_valid.iloc[: y_valid.shape[0] // 10] +X_test = X_test.iloc[: X_test.shape[0] // 10] # 2) Auto feature engineering X_train_l, X_valid_l = [], [] @@ -85,21 +90,54 @@ for f in DIRNAME.glob("model/model*.py"): model_l.append((m.fit(X_train_selected, y_train, X_valid_selected, y_valid), m.predict, select_m)) # 4) Evaluate the model on the validation set -metrics_all = [] +# metrics_all = [] +# for model, predict_func, select_m in model_l: +# X_valid_selected = select_m.select(X_valid.copy()) +# y_valid_pred = predict_func(model, X_valid_selected) +# metrics = compute_metrics_for_classification(y_valid, y_valid_pred) +# print(f"log_loss on valid set: {metrics}") +# metrics_all.append(metrics) +# 4) Use grid search to find the best ensemble model +valid_pred_list = [] for model, predict_func, select_m in model_l: X_valid_selected = select_m.select(X_valid.copy()) y_valid_pred = predict_func(model, X_valid_selected) + valid_pred_list.append(y_valid_pred) + +metrics_all = [] +weight_list = [] +searched_set = set() +for i in range(100): + weight = np.random.randint(0, high=10, size=(len(valid_pred_list),), dtype="i") + if str(weight.tolist()) in searched_set or weight.sum() == 0: + continue + weight = weight / weight.sum() + searched_set.add(str(weight.tolist())) + y_valid_pred = np.zeros_like(valid_pred_list[0]) + for j in range(len(valid_pred_list)): + y_valid_pred += valid_pred_list[j] * weight[j] + # normalize y_valid_pred each row to sum 1 + y_valid_pred = y_valid_pred / y_valid_pred.sum(axis=1)[:, np.newaxis] metrics = compute_metrics_for_classification(y_valid, y_valid_pred) - print(f"log_loss on valid set: {metrics}") metrics_all.append(metrics) + weight_list.append(weight) + # 5) Save the validation accuracy min_index = np.argmin(metrics_all) pd.Series(data=[metrics_all[min_index]], index=["log_loss"]).to_csv("submission_score.csv") +print(f"Accuracy on valid set: {metrics_all[min_index]}") # 6) Make predictions on the test set and save them -X_test_selected = model_l[min_index][2].select(X_test.copy()) -y_test_pred = model_l[min_index][1](model_l[min_index][0], X_test_selected) +test_pred_list = [] +for model, predict_func, select_m in model_l: + X_test_selected = select_m.select(X_test.copy()) + y_test_pred = predict_func(model, X_test_selected) + test_pred_list.append(y_test_pred) +y_test_pred = np.zeros_like(test_pred_list[0]) +for j in range(len(test_pred_list)): + y_test_pred += test_pred_list[j] * weight_list[min_index][j] +y_test_pred = y_test_pred / y_test_pred.sum(axis=1)[:, np.newaxis] # 7) Submit predictions for the test set diff --git a/rdagent/scenarios/kaggle/experiment/spaceship-titanic_template/train.py b/rdagent/scenarios/kaggle/experiment/spaceship-titanic_template/train.py index 3e5f6913..6f696e25 100644 --- a/rdagent/scenarios/kaggle/experiment/spaceship-titanic_template/train.py +++ b/rdagent/scenarios/kaggle/experiment/spaceship-titanic_template/train.py @@ -84,23 +84,54 @@ for f in DIRNAME.glob("model/model*.py"): model_l.append((m.fit(X_train_selected, y_train, X_valid_selected, y_valid), m.predict, select_m)) # 4) Evaluate the model on the validation set -metrics_all = [] +# metrics_all = [] +# for model, predict_func, select_m in model_l: +# X_valid_selected = select_m.select(X_valid.copy()) +# y_valid_pred = predict_func(model, X_valid_selected) +# y_valid_pred = (y_valid_pred > 0.5).astype(int) +# metrics = compute_metrics_for_classification(y_valid, y_valid_pred) +# print(f"Accuracy on valid set: {metrics}") +# metrics_all.append(metrics) + +# 4) Use grid search to find the best ensemble model +valid_pred_list = [] for model, predict_func, select_m in model_l: X_valid_selected = select_m.select(X_valid.copy()) y_valid_pred = predict_func(model, X_valid_selected) + valid_pred_list.append(y_valid_pred) + +metrics_all = [] +weight_list = [] +searched_set = set() +for i in range(1000): + weight = np.random.randint(0, high=10, size=(len(valid_pred_list),), dtype="i") + if str(weight.tolist()) in searched_set or weight.sum() == 0: + continue + weight = weight / weight.sum() + searched_set.add(str(weight.tolist())) + y_valid_pred = np.zeros_like(valid_pred_list[0]) + for j in range(len(valid_pred_list)): + y_valid_pred += valid_pred_list[j] * weight[j] y_valid_pred = (y_valid_pred > 0.5).astype(int) metrics = compute_metrics_for_classification(y_valid, y_valid_pred) - print(f"Accuracy on valid set: {metrics}") metrics_all.append(metrics) + weight_list.append(weight) # 5) Save the validation accuracy max_index = np.argmax(metrics_all) pd.Series(data=[metrics_all[max_index]], index=["MCC"]).to_csv("submission_score.csv") +print(f"Accuracy on valid set: {metrics_all[max_index]}") # 6) Make predictions on the test set and save them -X_test_selected = model_l[max_index][2].select(X_test.copy()) -y_test_pred = model_l[max_index][1](model_l[max_index][0], X_test_selected) +test_pred_list = [] +for model, predict_func, select_m in model_l: + X_test_selected = select_m.select(X_test.copy()) + y_test_pred = predict_func(model, X_test_selected) + test_pred_list.append(y_test_pred) +y_test_pred = np.zeros_like(test_pred_list[0]) +for j in range(len(test_pred_list)): + y_test_pred += test_pred_list[j] * weight_list[max_index][j] y_test_pred = (y_test_pred > 0.5).astype(bool) y_test_pred = y_test_pred.ravel() diff --git a/rdagent/scenarios/kaggle/experiment/tabular-playground-series-may-2022_template/model/model_xgboost.py b/rdagent/scenarios/kaggle/experiment/tabular-playground-series-may-2022_template/model/model_xgboost.py index 30d85b84..d035bf74 100644 --- a/rdagent/scenarios/kaggle/experiment/tabular-playground-series-may-2022_template/model/model_xgboost.py +++ b/rdagent/scenarios/kaggle/experiment/tabular-playground-series-may-2022_template/model/model_xgboost.py @@ -31,5 +31,5 @@ def predict(model: xgb.Booster, X): Keep feature select's consistency. """ dtest = xgb.DMatrix(X) - y_pred = pd.Series([round(v) for v in model.predict(dtest)]) + y_pred = model.predict(dtest).reshape(-1, 1) return y_pred diff --git a/rdagent/scenarios/kaggle/experiment/tabular-playground-series-may-2022_template/model/select_lightgbm.py b/rdagent/scenarios/kaggle/experiment/tabular-playground-series-may-2022_template/model/select_lightgbm.py new file mode 100644 index 00000000..f230f130 --- /dev/null +++ b/rdagent/scenarios/kaggle/experiment/tabular-playground-series-may-2022_template/model/select_lightgbm.py @@ -0,0 +1,12 @@ +import pandas as pd + + +def select(X: pd.DataFrame) -> pd.DataFrame: + """ + Select relevant features. To be used in fit & predict function. + """ + # For now, we assume all features are relevant. This can be expanded to feature selection logic. + if X.columns.nlevels == 1: + return X + X.columns = ["_".join(str(i) for i in col).strip() for col in X.columns.values] + return X diff --git a/rdagent/scenarios/kaggle/experiment/tabular-playground-series-may-2022_template/train.py b/rdagent/scenarios/kaggle/experiment/tabular-playground-series-may-2022_template/train.py index 08b757ed..7fee449a 100644 --- a/rdagent/scenarios/kaggle/experiment/tabular-playground-series-may-2022_template/train.py +++ b/rdagent/scenarios/kaggle/experiment/tabular-playground-series-may-2022_template/train.py @@ -54,17 +54,20 @@ for f in DIRNAME.glob("model/model*.py"): X_valid_selected = select_m.select(X_valid.copy()) m = import_module_from_path(f.stem, f) - model_l.append((m.fit(X_train_selected, y_train, X_valid_selected, y_valid), m.predict, select_m)) + model_l.append((m.fit(X_train_selected, y_train, X_valid_selected, y_valid), m.predict, select_m, f.stem)) print(f"Model [{f.stem}] has been trained") # 4) Evaluate the model on the validation set +sub_submission = pd.DataFrame(columns=["Model", "score"]) metrics_all = [] -for model, predict_func, select_m in model_l: +for model, predict_func, select_m, model_name in model_l: X_valid_selected = select_m.select(X_valid.copy()) y_valid_pred = predict_func(model, X_valid_selected) auroc = roc_auc_score(y_valid, y_valid_pred) print(f"[{type(model).__name__}] AUROC on valid set: {auroc}") metrics_all.append(auroc) + sub_submission = sub_submission._append({"Model": model_name, "score": auroc}, ignore_index=True) +sub_submission.to_csv("sub_submission_score.csv") # 5) Save the validation accuracy max_index = np.argmax(metrics_all) @@ -72,7 +75,7 @@ pd.Series(data=[metrics_all[max_index]], index=["AUROC"]).to_csv("submission_sco # 6) Make predictions on the test set and save them X_test_selected = model_l[max_index][2].select(X_test.copy()) -y_test_pred = model_l[max_index][1](model_l[max_index][0], X_test_selected).flatten() + 1 +y_test_pred = model_l[max_index][1](model_l[max_index][0], X_test_selected).flatten() # 7) Submit predictions for the test set diff --git a/rdagent/scenarios/kaggle/experiment/ventilator-pressure-prediction_template/model/select_lightgbm.py b/rdagent/scenarios/kaggle/experiment/ventilator-pressure-prediction_template/model/select_lightgbm.py new file mode 100644 index 00000000..f230f130 --- /dev/null +++ b/rdagent/scenarios/kaggle/experiment/ventilator-pressure-prediction_template/model/select_lightgbm.py @@ -0,0 +1,12 @@ +import pandas as pd + + +def select(X: pd.DataFrame) -> pd.DataFrame: + """ + Select relevant features. To be used in fit & predict function. + """ + # For now, we assume all features are relevant. This can be expanded to feature selection logic. + if X.columns.nlevels == 1: + return X + X.columns = ["_".join(str(i) for i in col).strip() for col in X.columns.values] + return X diff --git a/rdagent/scenarios/kaggle/experiment/ventilator-pressure-prediction_template/train.py b/rdagent/scenarios/kaggle/experiment/ventilator-pressure-prediction_template/train.py index 35a0e2a5..8fec0e8f 100644 --- a/rdagent/scenarios/kaggle/experiment/ventilator-pressure-prediction_template/train.py +++ b/rdagent/scenarios/kaggle/experiment/ventilator-pressure-prediction_template/train.py @@ -23,6 +23,7 @@ def import_module_from_path(module_name, module_path): # 1) Preprocess the data X_train, X_valid, y_train, y_valid, X_test, ids = preprocess_script() +mask = X_valid["u_out"] == 0 # 2) Auto feature engineering X_train_l, X_valid_l = [], [] @@ -62,7 +63,9 @@ metrics_all = [] for model, predict_func, select_m in model_l: X_valid_selected = select_m.select(X_valid.copy()) y_valid_pred = predict_func(model, X_valid_selected) - mae = mean_absolute_error(y_valid, y_valid_pred) + y_valid_filtered = y_valid[mask] + y_valid_pred_filtered = y_valid_pred[mask] + mae = mean_absolute_error(y_valid_filtered, y_valid_pred_filtered) print(f"[{type(model).__name__}] MAE on valid set: {mae}") metrics_all.append(mae) diff --git a/rdagent/scenarios/kaggle/kaggle_crawler.py b/rdagent/scenarios/kaggle/kaggle_crawler.py index ab331c25..569cf53e 100644 --- a/rdagent/scenarios/kaggle/kaggle_crawler.py +++ b/rdagent/scenarios/kaggle/kaggle_crawler.py @@ -19,6 +19,7 @@ from rdagent.core.exception import KaggleError from rdagent.core.prompts import Prompts from rdagent.log import rdagent_logger as logger from rdagent.oai.llm_utils import APIBackend +from rdagent.utils.env import MLEBDockerEnv # %% options = webdriver.ChromeOptions() @@ -101,22 +102,28 @@ def download_data(competition: str, local_path: str = KAGGLE_IMPLEMENT_SETTING.l if KAGGLE_IMPLEMENT_SETTING.if_using_mle_data: zipfile_path = f"{local_path}/zip_files" zip_competition_path = Path(zipfile_path) / competition - if not zip_competition_path.exists(): - try: - subprocess.run( - ["mlebench", "prepare", "-c", competition, "--data-dir", zipfile_path], - check=True, - stderr=subprocess.PIPE, - stdout=subprocess.PIPE, - ) - except subprocess.CalledProcessError as e: - logger.error(f"Download failed: {e}, stderr: {e.stderr}, stdout: {e.stdout}") - raise KaggleError(f"Download failed: {e}, stderr: {e.stderr}, stdout: {e.stdout}") + if ( + not zip_competition_path.exists() + or not (Path(local_path) / competition).exists() + or list((Path(local_path) / competition).iterdir()) == [] + ): + mleb_env = MLEBDockerEnv() + mleb_env.prepare() + (Path(local_path) / "zip_files").mkdir(parents=True, exist_ok=True) + (Path(local_path) / competition).mkdir(parents=True, exist_ok=True) - competition_path = Path(local_path) / competition - competition_path.mkdir(parents=True, exist_ok=True) - processed_data_folder_path = zip_competition_path / "prepared/public" - subprocess.run(f"cp -r {processed_data_folder_path}/* {competition_path}", shell=True) + mleb_env.run( + f"mlebench prepare -c {competition} --data-dir ./zip_files", + local_path=local_path, + running_extra_volume={str(Path("~/.kaggle").expanduser().absolute()): "/root/.kaggle"}, + ) + mleb_env.run( + f"/bin/sh -c 'cp -r ./zip_files/{competition}/prepared/public/* ./{competition}'", local_path=local_path + ) + mleb_env.run( + f"/bin/sh -c 'cp -r ./zip_files/{competition}/prepared/private/test.csv ./{competition}/valid.csv'", + local_path=local_path, + ) else: zipfile_path = f"{local_path}/zip_files" diff --git a/rdagent/scenarios/kaggle/prompts.yaml b/rdagent/scenarios/kaggle/prompts.yaml index 2c2fad3e..c73ae099 100644 --- a/rdagent/scenarios/kaggle/prompts.yaml +++ b/rdagent/scenarios/kaggle/prompts.yaml @@ -29,8 +29,6 @@ hypothesis_and_feedback: |- Hypothesis {{ loop.index }}: {{ hypothesis }} Observation on the result with the hypothesis: {{ feedback.observations }} Feedback on the original hypothesis: {{ feedback.hypothesis_evaluation }} - New Feedback for Context (For your reference): {{ feedback.new_hypothesis }} - Reasoning for new hypothesis: {{ feedback.reason }} Did changing to this hypothesis work? (focus on the change): {{ feedback.decision }} {% endfor %} @@ -177,6 +175,46 @@ model_experiment_output_format: |- } Usually, a larger model works better than a smaller one. Hence, the parameters should be larger. +kg_feedback_generation_user: |- + We are in a process of finding and validating hypotheses to build a powerful model. Each round aims to confirm or reject hypotheses based on results. + + The SOTA solution for the task is as follows: + Features and its corresponding channel: {{ sota_features }} + Models and its corresponding code: {{ sota_models }} + Final result of the SOTA solution (we select the best-performing model's metric as the final result): {{ sota_result }} + {% if sota_sub_results %} + Sub-results of all sub-models: {{ sota_sub_results }} + {% endif %} + + Current solution to be evaluated: + Hypothesis: {{ current_hypothesis }} + Reasoning: {{ current_hypothesis_reason }} + Current target action: {{ current_target_action }} + Experiments conducted and their code: {{ current_sub_exps_to_code }} + Final result of the current solution (we select the best-performing model's metric as the final result): {{ current_result }} + {% if current_sub_results %} + Sub-results of all sub-models: {{ current_sub_results }} + {% endif %} + + A more detailed comparison between the current solution and the SOTA solution: + {{ combined_result }} + + Some information about comparing the current solution with the SOTA solution: + {{ evaluation_description }} + + {% if last_hypothesis_and_feedback %} + The user has made some hypothesis and conducted experiments to validate them, and the results are as follows: + hypothesis: {{ last_hypothesis_and_feedback[0].hypothesis }} + feedback decision: {{ last_hypothesis_and_feedback[1].decision }} + reason: {{ last_hypothesis_and_feedback[1].reason }} + {% endif %} + Please refer to these hypothesis and feedback to help you recommend new hypothesis + + Consider Changing Direction for Significant Gaps with the Best Result and the last round: + - If the new results significantly differ from SOTA, consider a new direction. + - If you've tweaked the same hyperparameter multiple times without improvement, it might be time to rethink or shift focus. + - If it is model tuning, focus on comparing the SOTA's Sub-results of all sub-models: {{ sota_sub_results }} with the current experiment's Sub-results of all sub-models: {{ current_sub_results }}. For example, identify which model is currently the best, which model was adjusted in this experiment, and whether the adjustment was effective. Determine if there is potential to continue with this model or if another model shows more promise. + model_tuning_feedback_generation: system: |- You are an advanced assistant for analyzing results in data-driven R&D, in the context of designing machine learning models. @@ -212,41 +250,6 @@ model_tuning_feedback_generation: - Further Growth (if successful): Add dropout regularization (0.5 rate), retain L1 features. - Adjust (if unsuccessful): Use 5 layers, Leaky ReLU, dropout 0.3 rate. - user: |- - We are in a process of finding and validating hypotheses to build a powerful model. Each round aims to confirm or reject hypotheses based on results. - Target hypothesis: - {{ hypothesis_text }} - - {% if last_hypothesis %} - Previous Round (if applicable): - - Last Hypothesis: {{last_hypothesis.hypothesis}} - - Last Task and Code: {{last_task_and_code}} - - Last Result: {{last_result}} - {% else %} - - This is the first round. No previous information available. - {% endif %} - - SOTA (State of the Art) Round: - - SOTA Task and Code: {{sota_task_and_code}} - - SOTA Result: {{sota_result}} - - Current Round: - - Current Hypothesis: {{hypothesis.hypothesis}} - - Current Reasoning: {{hypothesis.reason}} - - Current Model_code: {{model_code}} - - Current Result: {{ exp.result }} - - Combined Results (Compared with SOTA): - {{ combined_result }} - - Analyze the combined result in the context of its ability to: - 1. Result Comparison: How does the result compare to the best? {{ evaluation_description }} - 2. To a large extent, the experiment with better metrics is the better one. - - Consider Changing Direction for Significant Gaps with the Best Result and the last round: - - If the new results significantly differ from SOTA, consider a new direction. - - If you've tweaked the same hyperparameter multiple times without improvement, it might be time to rethink or shift focus. - factor_feedback_generation: system: |- You are a professional data feature engineering assistant in data-driven R&D. @@ -284,41 +287,18 @@ factor_feedback_generation: "Reasoning": "Reasoning for the new hypothesis", "Replace Best Result": "yes or no" } - user: |- - Target hypothesis: - {{ hypothesis_text }} - Tasks and Features: - {% for task in task_details %} - - {{ task.factor_name }}: {{ task.factor_description }} - - Feature Formulation: {{ task.factor_formulation }} - - Variables: {{ task.variables }} - - Feature Implementation: {{ task.factor_implementation }} - {% if task.factor_implementation == "False" %} - **Note: This feature was not implemented in the current experiment. Only the hypothesis for implemented features can be verified.** - {% endif %} - {% endfor %} - Combined Results (Compared with SOTA): - {{ combined_result }} - - Analyze the combined result in the context of its ability to: - 1. Result Comparison: How does the result compare to the best? {{ evaluation_description }} - 2. To a large extent, the experiment with better metrics is the better one. - - Consider Changing Direction for Significant Gaps with the Best Result: - - If the new results significantly differ from the best, consider exploring a new direction. - - Avoid re-implementing previous features as those that surpassed the best are already included in the feature library and will be used in each run. - Note: Only features with 'Feature Implementation' as True are implemented and tested in this experiment. If 'Feature Implementation' is False, the hypothesis for that feature cannot be verified in this run. - feature_selection_feedback_generation: system: |- You are a professional feature selection assistant for machine learning models. Your task is to analyze the current feature selection strategy, evaluate its effectiveness, and suggest improvements. + The task is described in the following scenario: + {{ scenario }} - Consider the following when analyzing: - 1. How well does the current feature selection support the hypothesis? - 2. Which features seem to contribute most to the model's performance? - 3. Are there any features that might be redundant or noisy? - 4. What new feature selection strategies might improve the model? + In your feedback, consider: + 1. How effective is the current feature selection strategy? + 2. Are there any patterns in the selected or discarded features that might inform future selections? + 3. How might we refine or change the feature selection approach to improve model performance? + 4. Are there any domain-specific considerations that should inform our feature selection? Provide detailed and constructive feedback, focusing on actionable insights for feature selection improvement. @@ -331,32 +311,6 @@ feature_selection_feedback_generation: "Replace Best Result": "yes or no" } - user: |- - We are in an experiment of finding hypotheses for feature selection and validating or rejecting them to optimize our model's performance. - Target hypothesis: - {{ hypothesis_text }} - Tasks and Features: - {% for task in task_details %} - - {{ task.factor_name }}: {{ task.factor_description }} - - Feature Formulation: {{ task.factor_formulation }} - - Variables: {{ task.variables }} - - Feature Implementation: {{ task.factor_implementation }} - {% if task.factor_implementation == "False" %} - **Note: This feature was not implemented in the current experiment. Only the hypothesis for implemented features can be verified.** - {% endif %} - {% endfor %} - Combined Results (Compared with SOTA): - {{ combined_result }} - - Analyze the combined result in the context of its ability to: - 1. Result Comparison: How does the result compare to the best? {{ evaluation_description }} - 2. To a large extent, the experiment with better metrics is the better one. - - In your feedback, consider: - 1. How effective is the current feature selection strategy? - 2. Are there any patterns in the selected or discarded features that might inform future selections? - 3. How might we refine or change the feature selection approach to improve model performance? - 4. Are there any domain-specific considerations that should inform our feature selection? model_feature_selection: system: |- diff --git a/rdagent/scenarios/kaggle/proposal/proposal.py b/rdagent/scenarios/kaggle/proposal/proposal.py index d59ddaef..1628c77b 100644 --- a/rdagent/scenarios/kaggle/proposal/proposal.py +++ b/rdagent/scenarios/kaggle/proposal/proposal.py @@ -273,6 +273,22 @@ class KGHypothesisGen(FactorAndModelHypothesisGen): if self.scen.if_action_choosing_based_on_UCB: action = self.execute_next_action(trace) + hypothesis_specification = f"Hypothesis should avoid being too general and vague, and should be specific and actionable. For example, hypothesis like 'tune a model' is too general, while hypothesis like 'increase the learning rate to 0.1 of the lightgbm model will improve the performance' is specific and actionable." + if len(trace.hist) > 0: + sota_features = str(trace.hist[-1][1].based_experiments[-1].experiment_workspace.data_description) + sota_models = json.dumps( + trace.hist[-1][1].based_experiments[-1].experiment_workspace.model_description, indent=2 + ) + sota_result = trace.hist[-1][1].based_experiments[-1].result + hypothesis_specification += f"\nYour hypothesis should based on current SOTA solution. The user will conduct experiments based on the SOTA solution to test whether your hypothesis is right on this specific ecompetition. \n\nSOTA Features: {sota_features}\n\nSOTA Models: {sota_models}\n\nSOTA Result: {sota_result}" + if self.scen.if_action_choosing_based_on_UCB: + hypothesis_specification += ( + "\n\nNext experiment action is " + + action + + "\nspecification: " + + prompt_dict["hypothesis_specification"][action] + ) + context_dict = { "hypothesis_and_feedback": hypothesis_and_feedback, "RAG": generate_RAG_content( @@ -282,14 +298,7 @@ class KGHypothesisGen(FactorAndModelHypothesisGen): target=action if self.scen.if_action_choosing_based_on_UCB else None, ), "hypothesis_output_format": prompt_dict["hypothesis_output_format"], - "hypothesis_specification": ( - { - "next_experiment_action": f"next experiment action is {action}", - "specification": prompt_dict["hypothesis_specification"][action], - } - if self.scen.if_action_choosing_based_on_UCB - else None - ), + "hypothesis_specification": hypothesis_specification, } return context_dict, True @@ -383,7 +392,7 @@ class KGHypothesis2Experiment(FactorAndModelHypothesis2Experiment): response_dict = json.loads(response) tasks = [] model_type = response_dict.get("model_type", "Model type not provided") - if model_type not in KG_SELECT_MAPPING: + if not isinstance(model_type, str) or model_type not in KG_SELECT_MAPPING: raise ModelEmptyError( f"Invalid model type '{model_type}'. Allowed model types are: {', '.join(KG_SELECT_MAPPING)}." ) diff --git a/rdagent/utils/env.py b/rdagent/utils/env.py index 20e15915..47b202d2 100644 --- a/rdagent/utils/env.py +++ b/rdagent/utils/env.py @@ -179,7 +179,7 @@ class KGDockerConf(DockerConf): env_prefix = "KG_DOCKER_" build_from_dockerfile: bool = True - dockerfile_folder_path: Path = Path(__file__).parent.parent / "scenarios" / "kaggle" / "docker" + dockerfile_folder_path: Path = Path(__file__).parent.parent / "scenarios" / "kaggle" / "docker" / "kaggle_docker" image: str = "local_kg:latest" # image: str = "gcr.io/kaggle-gpu-images/python:latest" mount_path: str = "/workspace/kg_workspace/" @@ -192,6 +192,22 @@ class KGDockerConf(DockerConf): running_timeout_period: int = 600 +class MLEBDockerConf(DockerConf): + class Config: + env_prefix = "MLEB_DOCKER_" + + build_from_dockerfile: bool = True + dockerfile_folder_path: Path = Path(__file__).parent.parent / "scenarios" / "kaggle" / "docker" / "mle_bench_docker" + image: str = "local_mle:latest" + # image: str = "gcr.io/kaggle-gpu-images/python:latest" + mount_path: str = "/workspace/data_folder/" + default_entry: str = "mlebench prepare --all" + # extra_volumes: dict = { + # # TODO connect to the place where the data is stored + # Path("git_ignore_folder/data").resolve(): "/root/.data/" + # } + + # physionet.org/files/mimic-eicu-fiddle-feature/1.0.0/FIDDLE_mimic3 class DockerEnv(Env[DockerConf]): # TODO: Save the output into a specific file @@ -346,12 +362,8 @@ class DockerEnv(Env[DockerConf]): env: dict | None = None, running_extra_volume: dict | None = None, ): - with ThreadPoolExecutor() as executor: - future = executor.submit(self.__run, entry, local_path, env, running_extra_volume) - try: - return future.result(timeout=self.conf.running_timeout_period) - except TimeoutError: - raise TimeoutError(f"Timeout while running the container: {self.conf.running_timeout_period} seconds") + entry_add_timeout = f"timeout {self.conf.running_timeout_period} {entry}" + return self.__run(entry_add_timeout, local_path, env, running_extra_volume) def dump_python_code_run_and_get_results( self, @@ -428,3 +440,10 @@ class KGDockerEnv(DockerEnv): def __init__(self, competition: str = None, conf: DockerConf = KGDockerConf()): super().__init__(conf) + + +class MLEBDockerEnv(DockerEnv): + """MLEBench Docker""" + + def __init__(self, conf: DockerConf = MLEBDockerConf()): + super().__init__(conf)