diff --git a/rdagent/app/benchmark/factor/analysis.py b/rdagent/app/benchmark/factor/analysis.py index 3ef10fe2..f3412b40 100644 --- a/rdagent/app/benchmark/factor/analysis.py +++ b/rdagent/app/benchmark/factor/analysis.py @@ -82,12 +82,12 @@ class BenchmarkAnalyzer: for i in x: order_v.append( { - "avg. Run successful rate": 0, - "avg. Format successful rate": 1, - "avg. Correlation (value only)": 2, - "max. Correlation": 3, - "max. accuracy": 4, - "avg. accuracy": 5, + "Avg Run SR": 0, + "Avg Format SR": 1, + "Avg Correlation": 2, + "Max Correlation": 3, + "Max Accuracy": 4, + "Avg Accuracy": 5, }.get(i, i), ) return order_v @@ -140,12 +140,12 @@ class BenchmarkAnalyzer: result_all = pd.concat( { - "avg. Correlation (value only)": corr_res.iloc[:, 0], - "avg. Format successful rate": format_succ_rate_f.iloc[:, 0], - "avg. Run successful rate": succ_rate_f.iloc[:, 0], - "max. Correlation": corr_max_res.iloc[:, 0], - "max. accuracy": value_max_res.iloc[:, 0], - "avg. accuracy": value_avg_res.iloc[:, 0], + "Avg Correlation": corr_res.iloc[:, 0], + "Avg Format SR": format_succ_rate_f.iloc[:, 0], + "Avg Run SR": succ_rate_f.iloc[:, 0], + "Max Correlation": corr_max_res.iloc[:, 0], + "Max Accuracy": value_max_res.iloc[:, 0], + "Avg Accuracy": value_avg_res.iloc[:, 0], }, axis=1, ) @@ -179,11 +179,16 @@ class Plotter: @staticmethod def plot_data(data, file_name, title): - plt.figure(figsize=(10, 6)) - sns.barplot(x="index", y="b", hue="a", data=data) - plt.xlabel("Method") + plt.figure(figsize=(10, 10)) plt.ylabel("Value") - plt.title(title) + colors = ["#3274A1", "#E1812C", "#3A923A", "#C03D3E"] + plt.bar(data["a"], data["b"], color=colors, capsize=5) + for idx, row in data.iterrows(): + plt.text(idx, row["b"] + 0.01, f"{row['b']:.2f}", ha="center", va="bottom") + plt.suptitle(title, y=0.98) + plt.xticks(rotation=45) + plt.ylim(0, 1) + plt.tight_layout() plt.savefig(file_name) @@ -201,7 +206,7 @@ def main( final_results_df = pd.DataFrame(final_results) Plotter.change_fs(20) - plot_data = final_results_df.drop(["max. accuracy", "avg. accuracy"], axis=0).T + plot_data = final_results_df.drop(["Max Accuracy", "Avg Accuracy"], axis=0).T plot_data = plot_data.reset_index().melt("index", var_name="a", value_name="b") Plotter.plot_data(plot_data, "./comparison_plot.png", title) diff --git a/rdagent/app/kaggle/loop.py b/rdagent/app/kaggle/loop.py index 3839a4eb..ffa6001e 100644 --- a/rdagent/app/kaggle/loop.py +++ b/rdagent/app/kaggle/loop.py @@ -1,5 +1,6 @@ import subprocess from collections import defaultdict +from concurrent.futures import TimeoutError from typing import Any import fire @@ -115,7 +116,7 @@ class KaggleRDLoop(RDLoop): return exp - skip_loop_error = (ModelEmptyError, FactorEmptyError) + skip_loop_error = (ModelEmptyError, FactorEmptyError, TimeoutError) def main(path=None, step_n=None, competition=None): diff --git a/rdagent/components/coder/factor_coder/CoSTEER/evaluators.py b/rdagent/components/coder/factor_coder/CoSTEER/evaluators.py index 2ab26705..9a047915 100644 --- a/rdagent/components/coder/factor_coder/CoSTEER/evaluators.py +++ b/rdagent/components/coder/factor_coder/CoSTEER/evaluators.py @@ -8,6 +8,7 @@ from typing import List, Tuple import pandas as pd from jinja2 import Environment, StrictUndefined +from rdagent.components.coder.factor_coder.config import FACTOR_IMPLEMENT_SETTINGS from rdagent.components.coder.factor_coder.CoSTEER.evolvable_subjects import ( FactorEvolvingItem, ) @@ -92,7 +93,11 @@ class FactorCodeEvaluator(FactorEvaluator): .from_string(evaluate_prompts["evaluator_code_feedback_v1_system"]) .render( scenario=( - self.scen.get_scenario_all_desc(target_task) + self.scen.get_scenario_all_desc( + target_task, + filtered_tag="feature", + simple_background=FACTOR_IMPLEMENT_SETTINGS.simple_background, + ) if self.scen is not None else "No scenario description." ) @@ -190,7 +195,7 @@ class FactorOutputFormatEvaluator(FactorEvaluator): ) buffer = io.StringIO() gen_df.info(buf=buffer) - gen_df_info_str = f"The use is currently working on a feature related task.\nThe output dataframe info is:\n{buffer.getvalue()}" + gen_df_info_str = f"The user is currently working on a feature related task.\nThe output dataframe info is:\n{buffer.getvalue()}" system_prompt = ( Environment(undefined=StrictUndefined) .from_string( @@ -198,7 +203,7 @@ class FactorOutputFormatEvaluator(FactorEvaluator): ) .render( scenario=( - self.scen.get_scenario_all_desc(implementation.target_task) + self.scen.get_scenario_all_desc(implementation.target_task, filtered_tag="feature") if self.scen is not None else "No scenario description." ) @@ -512,7 +517,7 @@ class FactorFinalDecisionEvaluator(Evaluator): .from_string(evaluate_prompts["evaluator_final_decision_v1_system"]) .render( scenario=( - self.scen.get_scenario_all_desc(target_task) + self.scen.get_scenario_all_desc(target_task, filtered_tag="feature") if self.scen is not None else "No scenario description." ) diff --git a/rdagent/components/coder/factor_coder/CoSTEER/evolving_strategy.py b/rdagent/components/coder/factor_coder/CoSTEER/evolving_strategy.py index 9e1e9038..6ef51e8c 100644 --- a/rdagent/components/coder/factor_coder/CoSTEER/evolving_strategy.py +++ b/rdagent/components/coder/factor_coder/CoSTEER/evolving_strategy.py @@ -234,7 +234,7 @@ class FactorEvolvingStrategyWithGraph(MultiProcessEvolvingStrategy): implement_prompts["evolving_strategy_factor_implementation_v1_system"], ) .render( - scenario=self.scen.get_scenario_all_desc(target_task), + scenario=self.scen.get_scenario_all_desc(target_task, filtered_tag="feature"), queried_former_failed_knowledge=queried_former_failed_knowledge_to_render, ) ) diff --git a/rdagent/components/coder/factor_coder/config.py b/rdagent/components/coder/factor_coder/config.py index 4ef2a980..45488cfa 100644 --- a/rdagent/components/coder/factor_coder/config.py +++ b/rdagent/components/coder/factor_coder/config.py @@ -35,6 +35,9 @@ class FactorImplementSettings(BaseSettings): v2_error_summary: bool = False v2_knowledge_sampler: float = 1.0 + simple_background: bool = False + """Whether to use simple background information for code feedback""" + file_based_execution_timeout: int = 120 """Timeout in seconds for each factor implementation execution""" diff --git a/rdagent/components/coder/model_coder/CoSTEER/evaluators.py b/rdagent/components/coder/model_coder/CoSTEER/evaluators.py index 903caa07..136049d9 100644 --- a/rdagent/components/coder/model_coder/CoSTEER/evaluators.py +++ b/rdagent/components/coder/model_coder/CoSTEER/evaluators.py @@ -83,7 +83,7 @@ class ModelCodeEvaluator(Evaluator): .from_string(evaluate_prompts["evaluator_code_feedback"]["system"]) .render( scenario=( - self.scen.get_scenario_all_desc(target_task) + self.scen.get_scenario_all_desc(target_task, filtered_tag=target_task.model_type) if self.scen is not None else "No scenario description." ) @@ -145,7 +145,7 @@ class ModelFinalEvaluator(Evaluator): .from_string(evaluate_prompts["evaluator_final_feedback"]["system"]) .render( scenario=( - self.scen.get_scenario_all_desc(target_task) + self.scen.get_scenario_all_desc(target_task, filtered_tag=target_task.model_type) if self.scen is not None else "No scenario description." ) diff --git a/rdagent/components/coder/model_coder/CoSTEER/evolving_strategy.py b/rdagent/components/coder/model_coder/CoSTEER/evolving_strategy.py index 154dad10..a4d6776e 100644 --- a/rdagent/components/coder/model_coder/CoSTEER/evolving_strategy.py +++ b/rdagent/components/coder/model_coder/CoSTEER/evolving_strategy.py @@ -76,7 +76,7 @@ class ModelCoderEvolvingStrategy(EvolvingStrategy): coder_prompts["evolving_strategy_model_coder"]["system"], ) .render( - scenario=self.scen.get_scenario_all_desc(), + scenario=self.scen.get_scenario_all_desc(filtered_tag=target_task.model_type), queried_former_failed_knowledge=queried_former_failed_knowledge_to_render, current_code=current_code, ) diff --git a/rdagent/components/coder/model_coder/model_execute_template_v2.txt b/rdagent/components/coder/model_coder/model_execute_template_v2.txt index f0af8d0b..78962c2a 100644 --- a/rdagent/components/coder/model_coder/model_execute_template_v2.txt +++ b/rdagent/components/coder/model_coder/model_execute_template_v2.txt @@ -13,9 +13,11 @@ valid_y = pd.Series(np.random.randint(0, 2, 8)) model = fit(train_X, train_y, valid_X, valid_y) execution_model_output = predict(model, valid_X) + if isinstance(execution_model_output, torch.Tensor): execution_model_output = execution_model_output.cpu().detach().numpy() + execution_feedback_str = f"Execution successful, output numpy ndarray shape: {execution_model_output.shape}" pickle.dump(execution_model_output, open("execution_model_output.pkl", "wb")) diff --git a/rdagent/components/proposal/factor_proposal.py b/rdagent/components/proposal/__init__.py similarity index 60% rename from rdagent/components/proposal/factor_proposal.py rename to rdagent/components/proposal/__init__.py index 48148bb7..f2e2462c 100644 --- a/rdagent/components/proposal/factor_proposal.py +++ b/rdagent/components/proposal/__init__.py @@ -4,7 +4,7 @@ from typing import Tuple from jinja2 import Environment, StrictUndefined -from rdagent.components.coder.factor_coder.factor import FactorExperiment +from rdagent.core.experiment import Experiment from rdagent.core.prompts import Prompts from rdagent.core.proposal import ( Hypothesis, @@ -18,10 +18,8 @@ from rdagent.oai.llm_utils import APIBackend prompt_dict = Prompts(file_path=Path(__file__).parent / "prompts.yaml") -FactorHypothesis = Hypothesis +class LLMHypothesisGen(HypothesisGen): - -class FactorHypothesisGen(HypothesisGen): def __init__(self, scen: Scenario): super().__init__(scen) @@ -30,17 +28,17 @@ class FactorHypothesisGen(HypothesisGen): def prepare_context(self, trace: Trace) -> Tuple[dict, bool]: ... @abstractmethod - def convert_response(self, response: str) -> FactorHypothesis: ... + def convert_response(self, response: str) -> Hypothesis: ... - def gen(self, trace: Trace) -> FactorHypothesis: + def gen(self, trace: Trace) -> Hypothesis: context_dict, json_flag = self.prepare_context(trace) system_prompt = ( Environment(undefined=StrictUndefined) .from_string(prompt_dict["hypothesis_gen"]["system_prompt"]) .render( - targets="factors", - scenario=self.scen.get_scenario_all_desc(), + targets=self.targets, + scenario=self.scen.get_scenario_all_desc(filtered_tag="hypothesis_and_experiment"), hypothesis_output_format=context_dict["hypothesis_output_format"], hypothesis_specification=context_dict["hypothesis_specification"], ) @@ -49,7 +47,7 @@ class FactorHypothesisGen(HypothesisGen): Environment(undefined=StrictUndefined) .from_string(prompt_dict["hypothesis_gen"]["user_prompt"]) .render( - targets="factors", + targets=self.targets, hypothesis_and_feedback=context_dict["hypothesis_and_feedback"], RAG=context_dict["RAG"], ) @@ -62,21 +60,39 @@ class FactorHypothesisGen(HypothesisGen): return hypothesis -class FactorHypothesis2Experiment(Hypothesis2Experiment[FactorExperiment]): +class FactorHypothesisGen(LLMHypothesisGen): + def __init__(self, scen: Scenario): + super().__init__(scen) + self.targets = "factors" + + +class ModelHypothesisGen(LLMHypothesisGen): + def __init__(self, scen: Scenario): + super().__init__(scen) + self.targets = "model tuning" + + +class FactorAndModelHypothesisGen(FactorHypothesisGen): + def __init__(self, scen: Scenario): + super().__init__(scen) + self.targets = "feature engineering and model building" + + +class LLMHypothesis2Experiment(Hypothesis2Experiment[Experiment]): @abstractmethod def prepare_context(self, hypothesis: Hypothesis, trace: Trace) -> Tuple[dict, bool]: ... @abstractmethod - def convert_response(self, response: str, trace: Trace) -> FactorExperiment: ... + def convert_response(self, response: str, trace: Trace) -> Experiment: ... - def convert(self, hypothesis: Hypothesis, trace: Trace) -> FactorExperiment: + def convert(self, hypothesis: Hypothesis, trace: Trace) -> Experiment: context, json_flag = self.prepare_context(hypothesis, trace) system_prompt = ( Environment(undefined=StrictUndefined) .from_string(prompt_dict["hypothesis2experiment"]["system_prompt"]) .render( - targets="factors", - scenario=trace.scen.get_scenario_all_desc(), + targets=self.targets, + scenario=trace.scen.get_scenario_all_desc(filtered_tag="hypothesis_and_experiment"), experiment_output_format=context["experiment_output_format"], ) ) @@ -84,7 +100,7 @@ class FactorHypothesis2Experiment(Hypothesis2Experiment[FactorExperiment]): Environment(undefined=StrictUndefined) .from_string(prompt_dict["hypothesis2experiment"]["user_prompt"]) .render( - targets="factors", + targets=self.targets, target_hypothesis=context["target_hypothesis"], hypothesis_and_feedback=context["hypothesis_and_feedback"], target_list=context["target_list"], @@ -95,3 +111,21 @@ class FactorHypothesis2Experiment(Hypothesis2Experiment[FactorExperiment]): resp = APIBackend().build_messages_and_create_chat_completion(user_prompt, system_prompt, json_mode=json_flag) return self.convert_response(resp, trace) + + +class FactorHypothesis2Experiment(LLMHypothesis2Experiment): + def __init__(self): + super().__init__() + self.targets = "factors" + + +class ModelHypothesis2Experiment(LLMHypothesis2Experiment): + def __init__(self): + super().__init__() + self.targets = "model tuning" + + +class FactorAndModelHypothesis2Experiment(LLMHypothesis2Experiment): + def __init__(self): + super().__init__() + self.targets = "feature engineering and model building" diff --git a/rdagent/components/proposal/model_proposal.py b/rdagent/components/proposal/model_proposal.py deleted file mode 100644 index 4dc50025..00000000 --- a/rdagent/components/proposal/model_proposal.py +++ /dev/null @@ -1,99 +0,0 @@ -from abc import abstractmethod -from pathlib import Path -from typing import Tuple - -from jinja2 import Environment, StrictUndefined - -from rdagent.components.coder.model_coder.model import ModelExperiment -from rdagent.core.prompts import Prompts -from rdagent.core.proposal import ( - Hypothesis, - Hypothesis2Experiment, - HypothesisGen, - Scenario, - Trace, -) -from rdagent.oai.llm_utils import APIBackend - -ModelHypothesis = Hypothesis - -prompt_dict = Prompts(file_path=Path(__file__).parent / "prompts.yaml") - - -class ModelHypothesisGen(HypothesisGen): - prompts: Prompts = prompt_dict - - # The following methods are scenario related so they should be implemented in the subclass - @abstractmethod - def prepare_context(self, trace: Trace) -> Tuple[dict, bool]: ... - - @abstractmethod - def convert_response(self, response: str) -> ModelHypothesis: ... - - def gen(self, trace: Trace) -> ModelHypothesis: - context_dict, json_flag = self.prepare_context(trace) - - system_prompt = ( - Environment(undefined=StrictUndefined) - .from_string(ModelHypothesisGen.prompts["hypothesis_gen"]["system_prompt"]) - .render( - targets="model tuning", - scenario=self.scen.get_scenario_all_desc(), - hypothesis_output_format=context_dict["hypothesis_output_format"], - hypothesis_specification=context_dict["hypothesis_specification"], - ) - ) - user_prompt = ( - Environment(undefined=StrictUndefined) - .from_string(ModelHypothesisGen.prompts["hypothesis_gen"]["user_prompt"]) - .render( - targets="model tuning", - RAG=context_dict["RAG"], - ) - ) - - resp = APIBackend().build_messages_and_create_chat_completion(user_prompt, system_prompt, json_mode=json_flag) - - hypothesis = self.convert_response(resp) - - return hypothesis - - -class ModelHypothesis2Experiment(Hypothesis2Experiment[ModelExperiment]): - prompts: Prompts = prompt_dict - - def __init__(self) -> None: - super().__init__() - - @abstractmethod - def prepare_context(self, hypothesis: Hypothesis, trace: Trace) -> Tuple[dict, bool]: ... - - @abstractmethod - def convert_response(self, response: str, trace: Trace) -> ModelExperiment: ... - - def convert(self, hypothesis: Hypothesis, trace: Trace) -> ModelExperiment: - context, json_flag = self.prepare_context(hypothesis, trace) - system_prompt = ( - Environment(undefined=StrictUndefined) - .from_string(ModelHypothesis2Experiment.prompts["hypothesis2experiment"]["system_prompt"]) - .render( - targets="feature engineering and model building", - scenario=trace.scen.get_scenario_all_desc(), - experiment_output_format=context["experiment_output_format"], - ) - ) - user_prompt = ( - Environment(undefined=StrictUndefined) - .from_string(ModelHypothesis2Experiment.prompts["hypothesis2experiment"]["user_prompt"]) - .render( - targets="feature engineering and model building", - target_hypothesis=context["target_hypothesis"], - hypothesis_and_feedback=context["hypothesis_and_feedback"], - target_list=context["target_list"], - RAG=context["RAG"], - ) - ) - - resp = APIBackend().build_messages_and_create_chat_completion(user_prompt, system_prompt, json_mode=json_flag) - - return self.convert_response(resp, trace) diff --git a/rdagent/components/proposal/prompts.yaml b/rdagent/components/proposal/prompts.yaml index 6cf41cc0..f98bde6b 100644 --- a/rdagent/components/proposal/prompts.yaml +++ b/rdagent/components/proposal/prompts.yaml @@ -13,6 +13,11 @@ hypothesis_gen: {{ hypothesis_output_format }} user_prompt: |- + {% if hypothesis_and_feedback|length == 0 %}It is the first round of hypothesis generation. The user has no hypothesis on this scenario yet. + {% else %}It is not the first round, the user has made several hypothesis on this scenario and did several evaluation on them. + The former hypothesis and the corresponding feedbacks are as follows (focus on the last one & the new hypothesis that it provides and reasoning to see if you agree): + {{ hypothesis_and_feedback }} + {% endif %} {% if RAG %} To assist you in generating new {{targets}}, we have provided the following information: {{RAG}}. **Note:** The provided RAG is for reference only. diff --git a/rdagent/components/runner/__init__.py b/rdagent/components/runner/__init__.py index 256b3095..0f62bfce 100644 --- a/rdagent/components/runner/__init__.py +++ b/rdagent/components/runner/__init__.py @@ -1,4 +1,5 @@ import pickle +import shutil from pathlib import Path from typing import Any, Tuple @@ -20,5 +21,12 @@ class CachedRunner(Developer[ASpecificExp]): def assign_cached_result(self, exp: Experiment, cached_res: Experiment) -> Experiment: if exp.based_experiments and exp.based_experiments[-1].result is None: exp.based_experiments[-1].result = cached_res.based_experiments[-1].result + if cached_res.experiment_workspace.workspace_path.exists(): + for csv_file in cached_res.experiment_workspace.workspace_path.glob("*.csv"): + shutil.copy(csv_file, exp.experiment_workspace.workspace_path) + for py_file in (cached_res.experiment_workspace.workspace_path / "feature").glob("*.py"): + shutil.copy(py_file, exp.experiment_workspace.workspace_path / "feature") + for py_file in (cached_res.experiment_workspace.workspace_path / "model").glob("*.py"): + shutil.copy(py_file, exp.experiment_workspace.workspace_path / "model") exp.result = cached_res.result return exp diff --git a/rdagent/core/knowledge_base.py b/rdagent/core/knowledge_base.py index f8385c1b..ceca8ab0 100644 --- a/rdagent/core/knowledge_base.py +++ b/rdagent/core/knowledge_base.py @@ -15,9 +15,9 @@ class KnowledgeBase: with self.path.open("rb") as f: loaded = pickle.load(f) if isinstance(loaded, dict): - self.__dict__.update(loaded) + self.__dict__.update({k: v for k, v in loaded.items() if not k == "path"}) else: - self.__dict__.update(loaded.__dict__) + self.__dict__.update({k: v for k, v in loaded.__dict__.items() if not k == "path"}) def dump(self) -> None: if self.path is not None: diff --git a/rdagent/core/scenario.py b/rdagent/core/scenario.py index 2d970b17..3a384a85 100644 --- a/rdagent/core/scenario.py +++ b/rdagent/core/scenario.py @@ -46,7 +46,12 @@ class Scenario(ABC): """Rich style description to present""" @abstractmethod - def get_scenario_all_desc(self, task: Task | None = None) -> str: + def get_scenario_all_desc( + self, + task: Task | None = None, + filtered_tag: str | None = None, + simple_background: bool | None = None, + ) -> str: """ Combine all descriptions together diff --git a/rdagent/log/ui/app.py b/rdagent/log/ui/app.py index ddc0c837..8fe2d47a 100644 --- a/rdagent/log/ui/app.py +++ b/rdagent/log/ui/app.py @@ -147,6 +147,21 @@ def get_msgs_until(end_func: Callable[[Message], bool] = lambda _: True): sms.name = "alpha158" state.alpha158_metrics = sms + if ( + state.lround == 1 + and len(msg.content.based_experiments) > 0 + and msg.content.based_experiments[-1].result is not None + ): + sms = msg.content.based_experiments[-1].result + if isinstance(state.scenario, DMModelScenario): + sms.index = ["AUROC"] + elif isinstance( + state.scenario, (QlibModelScenario, QlibFactorFromReportScenario, QlibFactorScenario) + ): + sms = sms.loc[QLIB_SELECTED_METRICS] + sms.name = f"Baseline" + state.metric_series.append(sms) + # common metrics if msg.content.result is None: if isinstance(state.scenario, DMModelScenario): diff --git a/rdagent/oai/llm_conf.py b/rdagent/oai/llm_conf.py index 09d9d8b5..500d05b8 100644 --- a/rdagent/oai/llm_conf.py +++ b/rdagent/oai/llm_conf.py @@ -90,5 +90,7 @@ class LLMSettings(BaseSettings): gcr_endpoint_do_sample: bool = False gcr_endpoint_max_token: int = 100 + chat_model_map: str = "{}" + LLM_SETTINGS = LLMSettings() diff --git a/rdagent/oai/llm_utils.py b/rdagent/oai/llm_utils.py index 42b07b84..b27268ab 100644 --- a/rdagent/oai/llm_utils.py +++ b/rdagent/oai/llm_utils.py @@ -1,6 +1,7 @@ from __future__ import annotations import hashlib +import inspect import json import os import random @@ -325,6 +326,7 @@ class APIBackend: ) self.chat_model = LLM_SETTINGS.chat_model if chat_model is None else chat_model + self.chat_model_map = json.loads(LLM_SETTINGS.chat_model_map) self.encoder = tiktoken.encoding_for_model(self.chat_model) self.chat_api_base = LLM_SETTINGS.chat_azure_api_base if chat_api_base is None else chat_api_base self.chat_api_version = ( @@ -627,6 +629,15 @@ class APIBackend: if presence_penalty is None: presence_penalty = LLM_SETTINGS.chat_presence_penalty + # Use index 4 to skip the current function and intermediate calls, + # and get the locals of the caller's frame. + caller_locals = inspect.stack()[4].frame.f_locals + if "self" in caller_locals: + tag = caller_locals["self"].__class__.__name__ + else: + tag = inspect.stack()[4].function + model = self.chat_model_map.get(tag, self.chat_model) + finish_reason = None if self.use_llama2: response = self.generator.chat_completion( @@ -661,7 +672,7 @@ class APIBackend: logger.info(f"{LogColors.CYAN}Response:{resp}{LogColors.END}", tag="llm_messages") else: kwargs = dict( - model=self.chat_model, + model=model, messages=messages, max_tokens=max_tokens, temperature=temperature, @@ -705,6 +716,18 @@ class APIBackend: finish_reason = response.choices[0].finish_reason if LLM_SETTINGS.log_llm_chat_content: logger.info(f"{LogColors.CYAN}Response:{resp}{LogColors.END}", tag="llm_messages") + logger.info( + json.dumps( + { + "tag": tag, + "total_tokens": response.usage.total_tokens, + "prompt_tokens": response.usage.prompt_tokens, + "completion_tokens": response.usage.completion_tokens, + "model": model, + } + ), + tag="llm_messages", + ) if json_mode: json.loads(resp) if self.dump_chat_cache: diff --git a/rdagent/scenarios/data_mining/experiment/model_experiment.py b/rdagent/scenarios/data_mining/experiment/model_experiment.py index f5cef548..1068d2a5 100644 --- a/rdagent/scenarios/data_mining/experiment/model_experiment.py +++ b/rdagent/scenarios/data_mining/experiment/model_experiment.py @@ -63,7 +63,9 @@ The demo showcases the iterative process of hypothesis generation, knowledge con To demonstrate the dynamic evolution of models through the R&D loop, emphasizing how each iteration enhances the model performance and reliability. The performane is measured by the AUROC score (Area Under the Receiver Operating Characteristic), which is a commonly used metric for binary classification. """ - def get_scenario_all_desc(self, task: Task | None = None) -> str: + def get_scenario_all_desc( + self, task: Task | None = None, filtered_tag: str | None = None, simple_background: bool | None = None + ) -> str: return f"""Background of the scenario: {self.background} The interface you should follow to write the runnable code: diff --git a/rdagent/scenarios/data_mining/proposal/model_proposal.py b/rdagent/scenarios/data_mining/proposal/model_proposal.py index a4efc9b7..37f36e2f 100644 --- a/rdagent/scenarios/data_mining/proposal/model_proposal.py +++ b/rdagent/scenarios/data_mining/proposal/model_proposal.py @@ -5,8 +5,8 @@ from typing import List, Tuple from jinja2 import Environment, StrictUndefined from rdagent.components.coder.model_coder.model import ModelExperiment, ModelTask -from rdagent.components.proposal.model_proposal import ( - ModelHypothesis, +from rdagent.components.proposal import ( + Hypothesis, ModelHypothesis2Experiment, ModelHypothesisGen, ) @@ -16,7 +16,7 @@ from rdagent.scenarios.data_mining.experiment.model_experiment import DMModelExp prompt_dict = Prompts(file_path=Path(__file__).parent.parent.parent / "qlib" / "prompts.yaml") -DMModelHypothesis = ModelHypothesis +DMModelHypothesis = Hypothesis class DMModelHypothesisGen(ModelHypothesisGen): @@ -52,7 +52,7 @@ class DMModelHypothesisGen(ModelHypothesisGen): } return context_dict, True - def convert_response(self, response: str) -> ModelHypothesis: + def convert_response(self, response: str) -> Hypothesis: response_dict = json.loads(response) hypothesis = DMModelHypothesis( hypothesis=response_dict["hypothesis"], diff --git a/rdagent/scenarios/general_model/scenario.py b/rdagent/scenarios/general_model/scenario.py index 9fe8c159..5455a64a 100644 --- a/rdagent/scenarios/general_model/scenario.py +++ b/rdagent/scenarios/general_model/scenario.py @@ -41,7 +41,9 @@ class GeneralModelScenario(Scenario): def rich_style_description(self) -> str: return self._rich_style_description - def get_scenario_all_desc(self, task: Task | None = None) -> str: + def get_scenario_all_desc( + self, task: Task | None = None, filtered_tag: str | None = None, simple_background: bool | None = None + ) -> str: return f"""Background of the scenario: {self.background} The interface you should follow to write the runnable code: diff --git a/rdagent/scenarios/kaggle/developer/feedback.py b/rdagent/scenarios/kaggle/developer/feedback.py index 8d4bad54..9fe68957 100644 --- a/rdagent/scenarios/kaggle/developer/feedback.py +++ b/rdagent/scenarios/kaggle/developer/feedback.py @@ -101,7 +101,7 @@ class KGHypothesisExperiment2Feedback(HypothesisExperiment2Feedback): sys_prompt = ( Environment(undefined=StrictUndefined) .from_string(prompt_dict[prompt_key]["system"]) - .render(scenario=self.scen.get_scenario_all_desc()) + .render(scenario=self.scen.get_scenario_all_desc(filtered_tag="feedback")) ) last_task_and_code = None diff --git a/rdagent/scenarios/kaggle/developer/runner.py b/rdagent/scenarios/kaggle/developer/runner.py index 52cf1131..ba6b0144 100644 --- a/rdagent/scenarios/kaggle/developer/runner.py +++ b/rdagent/scenarios/kaggle/developer/runner.py @@ -25,11 +25,8 @@ class KGCachedRunner(CachedRunner[ASpecificExp]): for f in sorted((exp.experiment_workspace.workspace_path / "model").glob("*.py"), key=lambda x: x.name): codes.append(f.read_text()) codes = "\n".join(codes) - if exp.sub_workspace_list is not None: - for i in range(len(exp.sub_workspace_list)): - if exp.sub_workspace_list[i] is not None: - codes += str(exp.sub_workspace_list[i].code_dict.values()) - return md5_hash(codes) + cached_key_from_exp = CachedRunner.get_cache_key(self, exp) + return md5_hash(codes + cached_key_from_exp) @cache_with_pickle(get_cache_key, CachedRunner.assign_cached_result) def init_develop(self, exp: KGFactorExperiment | KGModelExperiment) -> KGFactorExperiment | KGModelExperiment: diff --git a/rdagent/scenarios/kaggle/experiment/covid19-global-forecasting-week-1_template/train.py b/rdagent/scenarios/kaggle/experiment/covid19-global-forecasting-week-1_template/train.py index 6346d061..9898ca39 100644 --- a/rdagent/scenarios/kaggle/experiment/covid19-global-forecasting-week-1_template/train.py +++ b/rdagent/scenarios/kaggle/experiment/covid19-global-forecasting-week-1_template/train.py @@ -36,9 +36,9 @@ X_test_l = [] for f in DIRNAME.glob("feature/feat*.py"): cls = import_module_from_path(f.stem, f).feature_engineering_cls() cls.fit(X_train) - X_train_f = cls.transform(X_train) - X_valid_f = cls.transform(X_valid) - X_test_f = cls.transform(X_test) + X_train_f = cls.transform(X_train.copy()) + X_valid_f = cls.transform(X_valid.copy()) + X_test_f = cls.transform(X_test.copy()) if X_train_f.shape[-1] == X_valid_f.shape[-1] and X_train_f.shape[-1] == X_test_f.shape[-1]: X_train_l.append(X_train_f) diff --git a/rdagent/scenarios/kaggle/experiment/digit-recognizer_template/train.py b/rdagent/scenarios/kaggle/experiment/digit-recognizer_template/train.py index c57cbe4a..51fb2ed6 100644 --- a/rdagent/scenarios/kaggle/experiment/digit-recognizer_template/train.py +++ b/rdagent/scenarios/kaggle/experiment/digit-recognizer_template/train.py @@ -36,9 +36,9 @@ X_test_l = [] for f in DIRNAME.glob("feature/feat*.py"): cls = import_module_from_path(f.stem, f).feature_engineering_cls() cls.fit(X_train) - X_train_f = cls.transform(X_train) - X_valid_f = cls.transform(X_valid) - X_test_f = cls.transform(X_test) + X_train_f = cls.transform(X_train.copy()) + X_valid_f = cls.transform(X_valid.copy()) + X_test_f = cls.transform(X_test.copy()) if X_train_f.shape[-1] == X_valid_f.shape[-1] and X_train_f.shape[-1] == X_test_f.shape[-1]: X_train_l.append(X_train_f) diff --git a/rdagent/scenarios/kaggle/experiment/feedback-prize-english-language-learning_template/train.py b/rdagent/scenarios/kaggle/experiment/feedback-prize-english-language-learning_template/train.py index ddb4eff5..40a3aaf2 100644 --- a/rdagent/scenarios/kaggle/experiment/feedback-prize-english-language-learning_template/train.py +++ b/rdagent/scenarios/kaggle/experiment/feedback-prize-english-language-learning_template/train.py @@ -30,9 +30,9 @@ for f in DIRNAME.glob("feature/feat*.py"): cls = import_module_from_path(f.stem, f).feature_engineering_cls() print(X_train.head()) cls.fit(X_train) - X_train_f = cls.transform(X_train) - X_valid_f = cls.transform(X_valid) - X_test_f = cls.transform(X_test) + X_train_f = cls.transform(X_train.copy()) + X_valid_f = cls.transform(X_valid.copy()) + X_test_f = cls.transform(X_test.copy()) if X_train_f.shape[-1] == X_valid_f.shape[-1] and X_train_f.shape[-1] == X_test_f.shape[-1]: X_train_l.append(X_train_f) diff --git a/rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/train.py b/rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/train.py index 619cb2e2..0a25fd7b 100644 --- a/rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/train.py +++ b/rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/train.py @@ -37,9 +37,9 @@ X_test_l = [] for f in DIRNAME.glob("feature/feat*.py"): cls = import_module_from_path(f.stem, f).feature_engineering_cls() cls.fit(X_train) - X_train_f = cls.transform(X_train) - X_valid_f = cls.transform(X_valid) - X_test_f = cls.transform(X_test) + X_train_f = cls.transform(X_train.copy()) + X_valid_f = cls.transform(X_valid.copy()) + X_test_f = cls.transform(X_test.copy()) if X_train_f.shape[-1] == X_valid_f.shape[-1] and X_train_f.shape[-1] == X_test_f.shape[-1]: X_train_l.append(X_train_f) diff --git a/rdagent/scenarios/kaggle/experiment/meta_tpl_deprecated/train.py b/rdagent/scenarios/kaggle/experiment/meta_tpl_deprecated/train.py index 7d4a5939..f8d3e0ee 100644 --- a/rdagent/scenarios/kaggle/experiment/meta_tpl_deprecated/train.py +++ b/rdagent/scenarios/kaggle/experiment/meta_tpl_deprecated/train.py @@ -45,9 +45,9 @@ X_test_l = [] for f in DIRNAME.glob("feature/feat*.py"): cls = import_module_from_path(f.stem, f).feature_engineering_cls() cls.fit(X_train) - X_train_f = cls.transform(X_train) - X_valid_f = cls.transform(X_valid) - X_test_f = cls.transform(X_test) + X_train_f = cls.transform(X_train.copy()) + X_valid_f = cls.transform(X_valid.copy()) + X_test_f = cls.transform(X_test.copy()) X_train_l.append(X_train_f) X_valid_l.append(X_valid_f) diff --git a/rdagent/scenarios/kaggle/experiment/optiver-realized-volatility-prediction_template/train.py b/rdagent/scenarios/kaggle/experiment/optiver-realized-volatility-prediction_template/train.py index 39a8b154..4528ebd1 100644 --- a/rdagent/scenarios/kaggle/experiment/optiver-realized-volatility-prediction_template/train.py +++ b/rdagent/scenarios/kaggle/experiment/optiver-realized-volatility-prediction_template/train.py @@ -38,9 +38,9 @@ X_test_l = [] for f in DIRNAME.glob("feature/feat*.py"): cls = import_module_from_path(f.stem, f).feature_engineering_cls() cls.fit(X_train) - X_train_f = cls.transform(X_train) - X_valid_f = cls.transform(X_valid) - X_test_f = cls.transform(X_test) + X_train_f = cls.transform(X_train.copy()) + X_valid_f = cls.transform(X_valid.copy()) + X_test_f = cls.transform(X_test.copy()) if X_train_f.shape[-1] == X_valid_f.shape[-1] and X_train_f.shape[-1] == X_test_f.shape[-1]: X_train_l.append(X_train_f) diff --git a/rdagent/scenarios/kaggle/experiment/playground-series-s3e11_template/train.py b/rdagent/scenarios/kaggle/experiment/playground-series-s3e11_template/train.py index 3f2e6281..7e035b66 100644 --- a/rdagent/scenarios/kaggle/experiment/playground-series-s3e11_template/train.py +++ b/rdagent/scenarios/kaggle/experiment/playground-series-s3e11_template/train.py @@ -26,9 +26,9 @@ X_test_l = [] for f in DIRNAME.glob("feature/feat*.py"): cls = import_module_from_path(f.stem, f).feature_engineering_cls() cls.fit(X_train) - X_train_f = cls.transform(X_train) - X_valid_f = cls.transform(X_valid) - X_test_f = cls.transform(X_test) + X_train_f = cls.transform(X_train.copy()) + X_valid_f = cls.transform(X_valid.copy()) + X_test_f = cls.transform(X_test.copy()) if X_train_f.shape[-1] == X_valid_f.shape[-1] and X_train_f.shape[-1] == X_test_f.shape[-1]: X_train_l.append(X_train_f) diff --git a/rdagent/scenarios/kaggle/experiment/playground-series-s3e14_template/train.py b/rdagent/scenarios/kaggle/experiment/playground-series-s3e14_template/train.py index 802f2cc2..345ceb45 100644 --- a/rdagent/scenarios/kaggle/experiment/playground-series-s3e14_template/train.py +++ b/rdagent/scenarios/kaggle/experiment/playground-series-s3e14_template/train.py @@ -26,9 +26,9 @@ X_test_l = [] for f in DIRNAME.glob("feature/feat*.py"): cls = import_module_from_path(f.stem, f).feature_engineering_cls() cls.fit(X_train) - X_train_f = cls.transform(X_train) - X_valid_f = cls.transform(X_valid) - X_test_f = cls.transform(X_test) + X_train_f = cls.transform(X_train.copy()) + X_valid_f = cls.transform(X_valid.copy()) + X_test_f = cls.transform(X_test.copy()) if X_train_f.shape[-1] == X_valid_f.shape[-1] and X_train_f.shape[-1] == X_test_f.shape[-1]: X_train_l.append(X_train_f) diff --git a/rdagent/scenarios/kaggle/experiment/playground-series-s3e16_template/train.py b/rdagent/scenarios/kaggle/experiment/playground-series-s3e16_template/train.py index e04091ee..b6cdbefb 100644 --- a/rdagent/scenarios/kaggle/experiment/playground-series-s3e16_template/train.py +++ b/rdagent/scenarios/kaggle/experiment/playground-series-s3e16_template/train.py @@ -26,9 +26,9 @@ X_test_l = [] for f in DIRNAME.glob("feature/feat*.py"): cls = import_module_from_path(f.stem, f).feature_engineering_cls() cls.fit(X_train) - X_train_f = cls.transform(X_train) - X_valid_f = cls.transform(X_valid) - X_test_f = cls.transform(X_test) + X_train_f = cls.transform(X_train.copy()) + X_valid_f = cls.transform(X_valid.copy()) + X_test_f = cls.transform(X_test.copy()) if X_train_f.shape[-1] == X_valid_f.shape[-1] and X_train_f.shape[-1] == X_test_f.shape[-1]: X_train_l.append(X_train_f) diff --git a/rdagent/scenarios/kaggle/experiment/playground-series-s3e26_template/train.py b/rdagent/scenarios/kaggle/experiment/playground-series-s3e26_template/train.py index 233ab051..11847a4a 100644 --- a/rdagent/scenarios/kaggle/experiment/playground-series-s3e26_template/train.py +++ b/rdagent/scenarios/kaggle/experiment/playground-series-s3e26_template/train.py @@ -40,9 +40,9 @@ X_test_l = [] for f in DIRNAME.glob("feature/feat*.py"): cls = import_module_from_path(f.stem, f).feature_engineering_cls() cls.fit(X_train) - X_train_f = cls.transform(X_train) - X_valid_f = cls.transform(X_valid) - X_test_f = cls.transform(X_test) + X_train_f = cls.transform(X_train.copy()) + X_valid_f = cls.transform(X_valid.copy()) + X_test_f = cls.transform(X_test.copy()) if X_train_f.shape[-1] == X_valid_f.shape[-1] and X_train_f.shape[-1] == X_test_f.shape[-1]: X_train_l.append(X_train_f) diff --git a/rdagent/scenarios/kaggle/experiment/playground-series-s4e5_template/train.py b/rdagent/scenarios/kaggle/experiment/playground-series-s4e5_template/train.py index 2d9a3a95..1a225b64 100644 --- a/rdagent/scenarios/kaggle/experiment/playground-series-s4e5_template/train.py +++ b/rdagent/scenarios/kaggle/experiment/playground-series-s4e5_template/train.py @@ -36,9 +36,9 @@ X_test_l = [] for f in DIRNAME.glob("feature/feat*.py"): cls = import_module_from_path(f.stem, f).feature_engineering_cls() cls.fit(X_train) - X_train_f = cls.transform(X_train) - X_valid_f = cls.transform(X_valid) - X_test_f = cls.transform(X_test) + X_train_f = cls.transform(X_train.copy()) + X_valid_f = cls.transform(X_valid.copy()) + X_test_f = cls.transform(X_test.copy()) if X_train_f.shape[-1] == X_valid_f.shape[-1] and X_train_f.shape[-1] == X_test_f.shape[-1]: X_train_l.append(X_train_f) diff --git a/rdagent/scenarios/kaggle/experiment/playground-series-s4e8_template/train.py b/rdagent/scenarios/kaggle/experiment/playground-series-s4e8_template/train.py index 42d24dc2..cc23c833 100644 --- a/rdagent/scenarios/kaggle/experiment/playground-series-s4e8_template/train.py +++ b/rdagent/scenarios/kaggle/experiment/playground-series-s4e8_template/train.py @@ -38,9 +38,9 @@ X_test_l = [] for f in DIRNAME.glob("feature/feat*.py"): cls = import_module_from_path(f.stem, f).feature_engineering_cls() cls.fit(X_train) - X_train_f = cls.transform(X_train) - X_valid_f = cls.transform(X_valid) - X_test_f = cls.transform(X_test) + X_train_f = cls.transform(X_train.copy()) + X_valid_f = cls.transform(X_valid.copy()) + X_test_f = cls.transform(X_test.copy()) if X_train_f.shape[-1] == X_valid_f.shape[-1] and X_train_f.shape[-1] == X_test_f.shape[-1]: X_train_l.append(X_train_f) diff --git a/rdagent/scenarios/kaggle/experiment/playground-series-s4e9_template/train.py b/rdagent/scenarios/kaggle/experiment/playground-series-s4e9_template/train.py index 81222f38..a6b3cf92 100644 --- a/rdagent/scenarios/kaggle/experiment/playground-series-s4e9_template/train.py +++ b/rdagent/scenarios/kaggle/experiment/playground-series-s4e9_template/train.py @@ -39,9 +39,9 @@ X_test_l = [] for f in DIRNAME.glob("feature/feat*.py"): cls = import_module_from_path(f.stem, f).feature_engineering_cls() cls.fit(X_train) - X_train_f = cls.transform(X_train) - X_valid_f = cls.transform(X_valid) - X_test_f = cls.transform(X_test) + X_train_f = cls.transform(X_train.copy()) + X_valid_f = cls.transform(X_valid.copy()) + X_test_f = cls.transform(X_test.copy()) if X_train_f.shape[-1] == X_valid_f.shape[-1] and X_train_f.shape[-1] == X_test_f.shape[-1]: X_train_l.append(X_train_f) diff --git a/rdagent/scenarios/kaggle/experiment/prompts.yaml b/rdagent/scenarios/kaggle/experiment/prompts.yaml index 528bd68b..92ec9b51 100644 --- a/rdagent/scenarios/kaggle/experiment/prompts.yaml +++ b/rdagent/scenarios/kaggle/experiment/prompts.yaml @@ -64,7 +64,7 @@ kg_background: |- - "Feature engineering": The user will design several new tasks and implement several new features. The new feature might only affect the model using all the feature book. - "Feature processing": The user will design a new task to process the feature book like normalization or one hot encoding to improve the model performance. Any processing with help of a deep model is not included in this task. - Model related: - - "Model feature selection": The user will modify one model to select the most important features from the feature book to improve the model performance. + - "Model feature selection": The user will modify one model to select the part of the features from the feature book to improve the model performance. - "Model tuning": The user will tune the hyperparameters of XGBoost, RandomForest or LightGBM or build or improve the NN model to improve the model performance. For each loop, you need to help user decide which action item to choose and provide the corresponding code to implement the action item. @@ -142,6 +142,7 @@ kg_model_interface: |- Here are some examples of how your Python code should be structured: + {% if tag == "XGBoost" or tag is none %} For XGBoost: ```python import pandas as pd @@ -168,7 +169,8 @@ kg_model_interface: |- return y_pred ``` - + {% endif %} + {% if tag == "RandomForest" or tag is none %} For RandomForest: ```python import pandas as pd @@ -191,7 +193,8 @@ kg_model_interface: |- return y_pred ``` - + {% endif %} + {% if tag == "LightGBM" or tag is none %} For LightGBM: ```python import pandas as pd @@ -213,7 +216,8 @@ kg_model_interface: |- return y_pred ``` - + {% endif %} + {% if tag == "NN" or tag is none %} For Neural Network: ```python import pandas as pd @@ -271,6 +275,7 @@ kg_model_interface: |- return y_pred ``` + {% endif %} kg_feature_simulator: |- The data preprocessing method you provide will be used to prepare data by processing it, concatenating the results with other features, and removing unnecessary features before training the model. @@ -283,6 +288,19 @@ kg_feature_simulator: |- 4. Train a model such as LightGBM, CatBoost, LSTM, or a simple PyTorch model using the processed data. 5. Evaluate the performance of your preprocessing method and provide feedback. +kg_feature_output_format: |- + The output should be a pandas DataFrame with the new features. The columns should be the new features, and the rows should correspond to the number of samples in the input DataFrame. + Sample output dataframe info: + + Index: {Same to the input DataFrame} + Data columns (total N columns): + # Column Dtype + --- ------ ----- + 0 feature_name_0 float64 + 1 feature_name_1 float64 + dtypes: float64(N) + memory usage: {Memory usage of the output DataFrame} + kg_model_output_format: |- For feature related tasks, the output should be a pandas DataFrame with the new features. The columns should be the new features, and the rows should correspond to the number of samples in the input DataFrame. For model related tasks, the output should be an np.ndarray with the appropriate number of predictions. diff --git a/rdagent/scenarios/kaggle/experiment/scenario.py b/rdagent/scenarios/kaggle/experiment/scenario.py index 79e424d7..6c8039ce 100644 --- a/rdagent/scenarios/kaggle/experiment/scenario.py +++ b/rdagent/scenarios/kaggle/experiment/scenario.py @@ -41,7 +41,6 @@ class KGScenario(Scenario): self.competition = competition self.competition_descriptions = crawl_descriptions(competition) self.input_shape = None - self._source_data = self.source_data self.competition_type = None self.competition_description = None @@ -64,11 +63,6 @@ class KGScenario(Scenario): self.vector_base.path = Path(datetime.now(timezone.utc).strftime("%Y-%m-%d-%H-%M-%S") + "_kaggle_kb.pkl") self.vector_base.dump() - self._output_format = self.output_format - self._interface = self.interface - self._simulator = self.simulator - self._background = self.background - self.action_counts = dict.fromkeys(KG_ACTION_LIST, 0) self.reward_estimates = {action: 0.0 for action in KG_ACTION_LIST} # self.reward_estimates["Model feature selection"] = 0.2 @@ -90,7 +84,7 @@ class KGScenario(Scenario): .from_string(prompt_dict["kg_description_template"]["user"]) .render( competition_descriptions=self.competition_descriptions, - raw_data_information=self._source_data, + raw_data_information=self.source_data, evaluation_metric_direction=self.evaluation_metric_direction, ) ) @@ -154,71 +148,82 @@ class KGScenario(Scenario): def source_data(self) -> str: data_folder = Path(KAGGLE_IMPLEMENT_SETTING.local_data_path) / self.competition - if (data_folder / "X_valid.pkl").exists(): - X_valid = pd.read_pickle(data_folder / "X_valid.pkl") - # TODO: Hardcoded for now, need to be fixed - if self.competition == "feedback-prize-english-language-learning": - self.input_shape = X_valid.shape - return "This is a sparse matrix of descriptive text." - buffer = io.StringIO() - X_valid.info(verbose=True, buf=buffer, show_counts=True) - data_info = buffer.getvalue() - self.input_shape = X_valid.shape - return data_info + if not (data_folder / "X_valid.pkl").exists(): + preprocess_experiment = KGFactorExperiment([]) + ( + X_train, + X_valid, + y_train, + y_valid, + X_test, + *others, + ) = preprocess_experiment.experiment_workspace.generate_preprocess_data() - preprocess_experiment = KGFactorExperiment([]) - ( - X_train, - X_valid, - y_train, - y_valid, - X_test, - *others, - ) = preprocess_experiment.experiment_workspace.generate_preprocess_data() + data_folder.mkdir(exist_ok=True, parents=True) + pickle.dump(X_train, open(data_folder / "X_train.pkl", "wb")) + pickle.dump(X_valid, open(data_folder / "X_valid.pkl", "wb")) + pickle.dump(y_train, open(data_folder / "y_train.pkl", "wb")) + pickle.dump(y_valid, open(data_folder / "y_valid.pkl", "wb")) + pickle.dump(X_test, open(data_folder / "X_test.pkl", "wb")) + pickle.dump(others, open(data_folder / "others.pkl", "wb")) - data_folder.mkdir(exist_ok=True, parents=True) - pickle.dump(X_train, open(data_folder / "X_train.pkl", "wb")) - pickle.dump(X_valid, open(data_folder / "X_valid.pkl", "wb")) - pickle.dump(y_train, open(data_folder / "y_train.pkl", "wb")) - pickle.dump(y_valid, open(data_folder / "y_valid.pkl", "wb")) - pickle.dump(X_test, open(data_folder / "X_test.pkl", "wb")) - pickle.dump(others, open(data_folder / "others.pkl", "wb")) - - self.input_shape = X_valid.shape + X_valid = pd.read_pickle(data_folder / "X_valid.pkl") + # TODO: Hardcoded for now, need to be fixed + if self.competition == "feedback-prize-english-language-learning": + return "This is a sparse matrix of descriptive text." buffer = io.StringIO() - X_valid.info(verbose=True, buf=buffer, show_counts=True) + X_valid.info(verbose=True, buf=buffer, show_counts=False) data_info = buffer.getvalue() + self.input_shape = X_valid.shape return data_info - @property - def output_format(self) -> str: - return ( + def output_format(self, tag=None) -> str: + assert tag in [None, "feature", "model"] + feature_output_format = f"""The feature code should output following the format: +{prompt_dict['kg_feature_output_format']}""" + model_output_format = f"""The model code should output following the format:\n""" + ( Environment(undefined=StrictUndefined) .from_string(prompt_dict["kg_model_output_format"]) .render(channel=self.model_output_channel) ) + if tag is None: + return feature_output_format + "\n" + model_output_format + elif tag == "feature": + return feature_output_format + elif tag == "model": + return model_output_format - @property - def interface(self) -> str: - return f"""The feature code should follow the interface: -{prompt_dict['kg_feature_interface']} -The model code should follow the interface: -{prompt_dict['kg_model_interface']} -""" + def interface(self, tag=None) -> str: + assert tag in [None, "feature", "XGBoost", "RandomForest", "LightGBM", "NN"] + feature_interface = f"""The feature code should follow the interface: +{prompt_dict['kg_feature_interface']}""" + if tag == "feature": + return feature_interface - @property - def simulator(self) -> str: - kg_model_simulator = ( + model_interface = "The model code should follow the interface:\n" + ( + Environment(undefined=StrictUndefined).from_string(prompt_dict["kg_model_interface"]).render(tag=tag) + ) + if tag is None: + return feature_interface + "\n" + model_interface + else: + return model_interface + + def simulator(self, tag=None) -> str: + assert tag in [None, "feature", "model"] + kg_feature_simulator = "The feature code will be sent to the simulator:\n" + prompt_dict["kg_feature_simulator"] + + kg_model_simulator = "The model code will be sent to the simulator:\n" + ( Environment(undefined=StrictUndefined) .from_string(prompt_dict["kg_model_simulator"]) .render(submission_specifications=self.submission_specifications) ) - return f"""The feature code should follow the simulator: -{prompt_dict['kg_feature_simulator']} -The model code should follow the simulator: -{kg_model_simulator} -""" + if tag is None: + return kg_feature_simulator + "\n" + kg_model_simulator + elif tag == "feature": + return kg_feature_simulator + elif tag == "model": + return kg_model_simulator @property def rich_style_description(self) -> str: @@ -248,17 +253,42 @@ Current Competition: [{self.competition}](https://www.kaggle.com/competitions/{s To automatically optimize performance metrics within the validation set or Kaggle Leaderboard, ultimately discovering the most efficient features and models through autonomous research and development. """ - def get_scenario_all_desc(self, task: Task | None = None) -> str: - return f"""Background of the scenario: -{self._background} -The source dataset you can use to generate the features: -{self._source_data} -The interface you should follow to write the runnable code: -{self._interface} -The output of your code should be in the format: -{self._output_format} -The simulator user can use to test your model: -{self._simulator} -The expected output & submission format specifications: -{self.submission_specifications} # Added again to emphasize the importance + def get_scenario_all_desc( + self, task: Task | None = None, filtered_tag: str | None = None, simple_background: bool | None = None + ) -> str: + def common_description() -> str: + return f"""\n------Background of the scenario------ +{self.background} + +------The source dataset you can use to generate the features------ +{self.source_data} + +------The expected output & submission format specifications------ +{self.submission_specifications} """ + + def interface(tag: str | None) -> str: + return f""" +------The interface you should follow to write the runnable code------ +{self.interface(tag)} +""" + + def output(tag: str | None) -> str: + return f""" +------The output of your code should be in the format------ +{self.output_format(tag)} +""" + + def simulator(tag: str | None) -> str: + return f""" +------The simulator user can use to test your solution------ +{self.simulator(tag)} +""" + + assert filtered_tag is not None, "filtered_tag should not be None in Kaggle scenario" + if filtered_tag == "hypothesis_and_experiment" or filtered_tag == "feedback": + return common_description() + simulator(None) + elif filtered_tag == "feature": + return common_description() + interface("feature") + output("feature") + simulator("feature") + else: + return common_description() + interface(filtered_tag) + output("model") + simulator("model") diff --git a/rdagent/scenarios/kaggle/experiment/sf-crime_template/train.py b/rdagent/scenarios/kaggle/experiment/sf-crime_template/train.py index 14245c63..8a5e08fd 100644 --- a/rdagent/scenarios/kaggle/experiment/sf-crime_template/train.py +++ b/rdagent/scenarios/kaggle/experiment/sf-crime_template/train.py @@ -40,9 +40,9 @@ X_test_l = [] for f in DIRNAME.glob("feature/feat*.py"): cls = import_module_from_path(f.stem, f).feature_engineering_cls() cls.fit(X_train) - X_train_f = cls.transform(X_train) - X_valid_f = cls.transform(X_valid) - X_test_f = cls.transform(X_test) + X_train_f = cls.transform(X_train.copy()) + X_valid_f = cls.transform(X_valid.copy()) + X_test_f = cls.transform(X_test.copy()) if X_train_f.shape[-1] == X_valid_f.shape[-1] and X_train_f.shape[-1] == X_test_f.shape[-1]: X_train_l.append(X_train_f) diff --git a/rdagent/scenarios/kaggle/experiment/spaceship-titanic_template/train.py b/rdagent/scenarios/kaggle/experiment/spaceship-titanic_template/train.py index c94403b5..3e5f6913 100644 --- a/rdagent/scenarios/kaggle/experiment/spaceship-titanic_template/train.py +++ b/rdagent/scenarios/kaggle/experiment/spaceship-titanic_template/train.py @@ -38,9 +38,9 @@ X_test_l = [] for f in DIRNAME.glob("feature/feat*.py"): cls = import_module_from_path(f.stem, f).feature_engineering_cls() cls.fit(X_train) - X_train_f = cls.transform(X_train) - X_valid_f = cls.transform(X_valid) - X_test_f = cls.transform(X_test) + X_train_f = cls.transform(X_train.copy()) + X_valid_f = cls.transform(X_valid.copy()) + X_test_f = cls.transform(X_test.copy()) if X_train_f.shape[-1] == X_valid_f.shape[-1] and X_train_f.shape[-1] == X_test_f.shape[-1]: X_train_l.append(X_train_f) diff --git a/rdagent/scenarios/kaggle/experiment/statoil-iceberg-classifier-challenge_template/train.py b/rdagent/scenarios/kaggle/experiment/statoil-iceberg-classifier-challenge_template/train.py index 57fcbe88..e1176526 100644 --- a/rdagent/scenarios/kaggle/experiment/statoil-iceberg-classifier-challenge_template/train.py +++ b/rdagent/scenarios/kaggle/experiment/statoil-iceberg-classifier-challenge_template/train.py @@ -38,9 +38,9 @@ X_test_l = [] for f in DIRNAME.glob("feature/feat*.py"): cls = import_module_from_path(f.stem, f).feature_engineering_cls() cls.fit(X_train) - X_train_f = cls.transform(X_train) - X_valid_f = cls.transform(X_valid) - X_test_f = cls.transform(X_test) + X_train_f = cls.transform(X_train.copy()) + X_valid_f = cls.transform(X_valid.copy()) + X_test_f = cls.transform(X_test.copy()) if X_train_f.shape[-1] == X_valid_f.shape[-1] and X_train_f.shape[-1] == X_test_f.shape[-1]: X_train_l.append(X_train_f) diff --git a/rdagent/scenarios/kaggle/prompts.yaml b/rdagent/scenarios/kaggle/prompts.yaml index e6d506c7..2c2fad3e 100644 --- a/rdagent/scenarios/kaggle/prompts.yaml +++ b/rdagent/scenarios/kaggle/prompts.yaml @@ -25,7 +25,7 @@ KG_hypothesis_gen_RAG: |- {% endif %} hypothesis_and_feedback: |- - {% for hypothesis, experiment, feedback in trace.hist %} + {% for hypothesis, experiment, feedback in trace.hist[-10:] %} Hypothesis {{ loop.index }}: {{ hypothesis }} Observation on the result with the hypothesis: {{ feedback.observations }} Feedback on the original hypothesis: {{ feedback.hypothesis_evaluation }} diff --git a/rdagent/scenarios/kaggle/proposal/proposal.py b/rdagent/scenarios/kaggle/proposal/proposal.py index 2b64521c..d59ddaef 100644 --- a/rdagent/scenarios/kaggle/proposal/proposal.py +++ b/rdagent/scenarios/kaggle/proposal/proposal.py @@ -9,10 +9,9 @@ from rdagent.app.kaggle.conf import KAGGLE_IMPLEMENT_SETTING from rdagent.components.coder.factor_coder.factor import FactorTask from rdagent.components.coder.model_coder.model import ModelExperiment, ModelTask from rdagent.components.knowledge_management.vector_base import VectorBase -from rdagent.components.proposal.model_proposal import ( - ModelHypothesis, - ModelHypothesis2Experiment, - ModelHypothesisGen, +from rdagent.components.proposal import ( + FactorAndModelHypothesis2Experiment, + FactorAndModelHypothesisGen, ) from rdagent.core.exception import ModelEmptyError from rdagent.core.prompts import Prompts @@ -67,7 +66,135 @@ Concise Knowledge: {self.concise_knowledge} """ -class KGHypothesisGen(ModelHypothesisGen): +def generate_RAG_content( + scen: KGScenario, + trace: Trace, + hypothesis_and_feedback: str, + target: str = None, + chosen_hypothesis: str = None, + chosen_hypothesis_type: str = None, +) -> str: + if scen.if_using_vector_rag: + if scen.mini_case: + rag_results, _ = scen.vector_base.search_experience(target, hypothesis_and_feedback, topk_k=1) + else: + rag_results, _ = scen.vector_base.search_experience(target, hypothesis_and_feedback, topk_k=5) + return "\n".join([doc.content for doc in rag_results]) + if scen.if_using_graph_rag is False or trace.knowledge_base is None: + return None + same_competition_node = trace.knowledge_base.get_node_by_content(trace.scen.get_competition_full_desc()) + if same_competition_node is not None: + related_hypothesis_nodes = [] + for action in KG_ACTION_LIST: + related_hypothesis_nodes.extend( + trace.knowledge_base.get_nodes_within_steps( + start_node=same_competition_node, + steps=1, + constraint_labels=[action], + )[:1] + ) + else: + related_hypothesis_nodes = [] + experiences = [] + for hypothesis_node in related_hypothesis_nodes: + experience = {"hypothesis": hypothesis_node.content} + experiment_node_list = trace.knowledge_base.get_nodes_within_steps( + start_node=hypothesis_node, steps=1, constraint_labels=["experiments"] + ) + if len(experiment_node_list) > 0: + experience["experiments"] = experiment_node_list[0].content + else: + experience["experiments"] = "No experiment information available." + conclusion_node_list = trace.knowledge_base.get_nodes_within_steps( + start_node=hypothesis_node, steps=1, constraint_labels=["conclusion"] + ) + if len(conclusion_node_list) > 0: + experience["conclusion"] = conclusion_node_list[0].content + else: + experience["conclusion"] = "No conclusion information available." + experiences.append(experience) + + found_nodes = [] + insights = [] + if chosen_hypothesis is not None: + similar_nodes = trace.knowledge_base.semantic_search( + node=chosen_hypothesis, + topk_k=2, + ) + + for similar_node in similar_nodes: + hypothesis_nodes = trace.knowledge_base.get_nodes_within_steps( + start_node=similar_node, + steps=3, + constraint_labels=[chosen_hypothesis_type], + ) + found_nodes.extend(hypothesis_nodes[:5]) + + found_nodes = sorted(list(set(found_nodes)), key=lambda x: len(x.content)) + + for exp_node in found_nodes[:5]: + insight = {"experiments": exp_node.content} + hypothesis_node_list = trace.knowledge_base.get_nodes_within_steps( + start_node=exp_node, steps=2, constraint_labels=KG_ACTION_LIST + ) + if len(hypothesis_node_list) > 0: + insight["hypothesis"] = hypothesis_node_list[0].content + else: + insight["hypothesis"] = "No hypothesis information available." + conclusion_node_list = trace.knowledge_base.get_nodes_within_steps( + start_node=exp_node, steps=2, constraint_labels=["conclusion"] + ) + if len(conclusion_node_list) > 0: + insight["conclusion"] = conclusion_node_list[0].content + else: + insight["conclusion"] = "No conclusion information available." + insights.append(insight) + else: + similar_nodes = trace.knowledge_base.semantic_search( + node=trace.scen.get_competition_full_desc(), + topk_k=2, + ) + + for similar_node in similar_nodes: + for hypothesis_type in KG_ACTION_LIST: + hypothesis_nodes = trace.knowledge_base.get_nodes_within_steps( + start_node=similar_node, + steps=3, + constraint_labels=[hypothesis_type], + ) + found_nodes.extend(hypothesis_nodes[:2]) + + found_nodes = sorted(list(set(found_nodes)), key=lambda x: len(x.content)) + + for hypothesis_node in found_nodes[:5]: + if hypothesis_node in related_hypothesis_nodes: + continue + insight = {"hypothesis": hypothesis_node.content} + experiment_node_list = trace.knowledge_base.get_nodes_within_steps( + start_node=hypothesis_node, steps=2, constraint_labels=["experiments"] + ) + if len(experiment_node_list) > 0: + insight["experiments"] = experiment_node_list[0].content + else: + insight["experiments"] = "No experiment information available." + conclusion_node_list = trace.knowledge_base.get_nodes_within_steps( + start_node=hypothesis_node, steps=2, constraint_labels=["conclusion"] + ) + if len(conclusion_node_list) > 0: + insight["conclusion"] = conclusion_node_list[0].content + else: + insight["conclusion"] = "No conclusion information available." + insights.append(insight) + + RAG_content = ( + Environment(undefined=StrictUndefined) + .from_string(prompt_dict["KG_hypothesis_gen_RAG"]) + .render(insights=insights, experiences=experiences) + ) + return RAG_content + + +class KGHypothesisGen(FactorAndModelHypothesisGen): """ # NOTE: we can share this class across different data mining scenarios # It may better to move the class into components folder like `rdagent/components/proposal/model_proposal.py` @@ -82,91 +209,6 @@ class KGHypothesisGen(ModelHypothesisGen): def __init__(self, scen: Scenario) -> Tuple[dict, bool]: super().__init__(scen) - def generate_RAG_content(self, trace: Trace, hypothesis_and_feedback: str, target: str = None) -> str: - if self.scen.if_using_vector_rag: - if self.scen.mini_case: - rag_results, _ = self.scen.vector_base.search_experience(target, hypothesis_and_feedback, topk_k=1) - else: - rag_results, _ = self.scen.vector_base.search_experience(target, hypothesis_and_feedback, topk_k=5) - return "\n".join([doc.content for doc in rag_results]) - if self.scen.if_using_graph_rag is False or trace.knowledge_base is None: - return None - same_competition_node = trace.knowledge_base.get_node_by_content(trace.scen.get_competition_full_desc()) - if same_competition_node is not None: - related_hypothesis_nodes = [] - for action in KG_ACTION_LIST: - related_hypothesis_nodes.extend( - trace.knowledge_base.get_nodes_within_steps( - start_node=same_competition_node, - steps=1, - constraint_labels=[action], - )[:1] - ) - else: - related_hypothesis_nodes = [] - experiences = [] - for hypothesis_node in related_hypothesis_nodes: - experience = {"hypothesis": hypothesis_node.content} - experiment_node_list = trace.knowledge_base.get_nodes_within_steps( - start_node=hypothesis_node, steps=1, constraint_labels=["experiments"] - ) - if len(experiment_node_list) > 0: - experience["experiments"] = experiment_node_list[0].content - else: - experience["experiments"] = "No experiment information available." - conclusion_node_list = trace.knowledge_base.get_nodes_within_steps( - start_node=hypothesis_node, steps=1, constraint_labels=["conclusion"] - ) - if len(conclusion_node_list) > 0: - experience["conclusion"] = conclusion_node_list[0].content - else: - experience["conclusion"] = "No conclusion information available." - experiences.append(experience) - - similar_nodes = trace.knowledge_base.semantic_search( - node=trace.scen.get_competition_full_desc(), - topk_k=2, - ) - - found_hypothesis_nodes = [] - for similar_node in similar_nodes: - hypothesis_nodes = trace.knowledge_base.get_nodes_within_steps( - start_node=similar_node, - steps=3, - constraint_labels=[target], - ) - found_hypothesis_nodes.extend(hypothesis_nodes[:2]) - - found_hypothesis_nodes = sorted(list(set(found_hypothesis_nodes)), key=lambda x: len(x.content)) - - insights = [] - for hypothesis_node in found_hypothesis_nodes[:5]: - if hypothesis_node in related_hypothesis_nodes: - continue - insight = {"hypothesis": hypothesis_node.content} - experiment_node_list = trace.knowledge_base.get_nodes_within_steps( - start_node=hypothesis_node, steps=1, constraint_labels=["experiments"] - ) - if len(experiment_node_list) > 0: - insight["experiments"] = experiment_node_list[0].content - else: - insight["experiments"] = "No experiment information available." - conclusion_node_list = trace.knowledge_base.get_nodes_within_steps( - start_node=hypothesis_node, steps=1, constraint_labels=["conclusion"] - ) - if len(conclusion_node_list) > 0: - insight["conclusion"] = conclusion_node_list[0].content - else: - insight["conclusion"] = "No conclusion information available." - insights.append(insight) - - RAG_content = ( - Environment(undefined=StrictUndefined) - .from_string(prompt_dict["KG_hypothesis_gen_RAG"]) - .render(insights=insights, experiences=experiences) - ) - return RAG_content - def update_reward_estimates(self, trace: Trace) -> None: if len(trace.hist) > 0: last_entry = trace.hist[-1] @@ -233,7 +275,8 @@ class KGHypothesisGen(ModelHypothesisGen): context_dict = { "hypothesis_and_feedback": hypothesis_and_feedback, - "RAG": self.generate_RAG_content( + "RAG": generate_RAG_content( + scen=self.scen, trace=trace, hypothesis_and_feedback=hypothesis_and_feedback, target=action if self.scen.if_action_choosing_based_on_UCB else None, @@ -250,7 +293,7 @@ class KGHypothesisGen(ModelHypothesisGen): } return context_dict, True - def convert_response(self, response: str) -> ModelHypothesis: + def convert_response(self, response: str) -> Hypothesis: response_dict = json.loads(response) hypothesis = KGHypothesis( @@ -266,9 +309,9 @@ class KGHypothesisGen(ModelHypothesisGen): return hypothesis -class KGHypothesis2Experiment(ModelHypothesis2Experiment): +class KGHypothesis2Experiment(FactorAndModelHypothesis2Experiment): def prepare_context(self, hypothesis: Hypothesis, trace: Trace) -> Tuple[dict, bool]: - scenario = trace.scen.get_scenario_all_desc() + scenario = trace.scen.get_scenario_all_desc(filtered_tag="hypothesis_and_experiment") assert isinstance(hypothesis, KGHypothesis) experiment_output_format = ( prompt_dict["feature_experiment_output_format"] @@ -300,7 +343,13 @@ class KGHypothesis2Experiment(ModelHypothesis2Experiment): "hypothesis_and_feedback": hypothesis_and_feedback, "experiment_output_format": experiment_output_format, "target_list": model_list, - "RAG": ..., + "RAG": generate_RAG_content( + trace.scen, + trace, + hypothesis_and_feedback, + chosen_hypothesis=hypothesis.hypothesis, + chosen_hypothesis_type=hypothesis.action, + ), }, True def convert_feature_experiment(self, response: str, trace: Trace) -> KGFactorExperiment: diff --git a/rdagent/scenarios/qlib/experiment/factor_experiment.py b/rdagent/scenarios/qlib/experiment/factor_experiment.py index b45f22e6..c9031213 100644 --- a/rdagent/scenarios/qlib/experiment/factor_experiment.py +++ b/rdagent/scenarios/qlib/experiment/factor_experiment.py @@ -60,8 +60,13 @@ class QlibFactorScenario(Scenario): def experiment_setting(self) -> str: return self._experiment_setting - def get_scenario_all_desc(self, task: Task | None = None) -> str: + def get_scenario_all_desc( + self, task: Task | None = None, filtered_tag: str | None = None, simple_background: bool | None = None + ) -> str: """A static scenario describer""" + if simple_background: + return f"""Background of the scenario: +{self.background}""" return f"""Background of the scenario: {self.background} The source data you can use: diff --git a/rdagent/scenarios/qlib/experiment/model_experiment.py b/rdagent/scenarios/qlib/experiment/model_experiment.py index cd9543df..ea971315 100644 --- a/rdagent/scenarios/qlib/experiment/model_experiment.py +++ b/rdagent/scenarios/qlib/experiment/model_experiment.py @@ -58,7 +58,9 @@ class QlibModelScenario(Scenario): def experiment_setting(self) -> str: return self._experiment_setting - def get_scenario_all_desc(self, task: Task | None = None) -> str: + def get_scenario_all_desc( + self, task: Task | None = None, filtered_tag: str | None = None, simple_background: bool | None = None + ) -> str: return f"""Background of the scenario: {self.background} The interface you should follow to write the runnable code: diff --git a/rdagent/scenarios/qlib/experiment/utils.py b/rdagent/scenarios/qlib/experiment/utils.py index fe2ed15b..690d7bd5 100644 --- a/rdagent/scenarios/qlib/experiment/utils.py +++ b/rdagent/scenarios/qlib/experiment/utils.py @@ -51,7 +51,7 @@ def generate_data_folder_from_qlib(): ) -def get_file_desc(p: Path) -> str: +def get_file_desc(p: Path, variable_list=[]) -> str: """ Get the description of a file based on its type. @@ -83,22 +83,29 @@ def get_file_desc(p: Path) -> str: pd.set_option("display.max_rows", None) # or 1000 pd.set_option("display.max_colwidth", None) # or 199 - buffer = io.StringIO() - df.info(verbose=True, buf=buffer, show_counts=False) - - df_info = buffer.getvalue() if isinstance(df.index, pd.MultiIndex): - df_info += f"\nMultiIndex names:, {df.index.names})" + df_info = f"MultiIndex names:, {df.index.names})\n" + else: + df_info = f"Index name: {df.index.name}\n" + columns = df.dtypes.to_dict() + filtered_columns = [f"{i, j}" for i, j in columns.items() if i in variable_list] + if filtered_columns: + df_info += "Related Data columns: \n" + df_info += ",".join(filtered_columns) + else: + df_info += "Data columns: \n" + df_info += ",".join(columns) + df_info += "\n" if "REPORT_PERIOD" in df.columns: one_instrument = df.index.get_level_values("instrument")[0] df_on_one_instrument = df.loc[pd.IndexSlice[:, one_instrument], ["REPORT_PERIOD"]] df_info += f""" A snapshot of one instrument, from which you can tell the distribution of the data: -{df_on_one_instrument.head(10)} +{df_on_one_instrument.head(5)} """ return JJ_TPL.render( file_name=p.name, - type_desc="h5, generated by `df.info(verbose=True, show_counts=False)` and appendix info", + type_desc="h5 info", content=df_info, ) elif p.name.endswith(".md"): @@ -115,7 +122,7 @@ A snapshot of one instrument, from which you can tell the distribution of the da ) -def get_data_folder_intro(fname_reg: str = ".*", flags=0) -> str: +def get_data_folder_intro(fname_reg: str = ".*", flags=0, variable_mapping=None) -> str: """ Directly get the info of the data folder. It is for preparing prompting message. @@ -144,5 +151,8 @@ def get_data_folder_intro(fname_reg: str = ".*", flags=0) -> str: content_l = [] for p in Path(FACTOR_IMPLEMENT_SETTINGS.data_folder_debug).iterdir(): if re.match(fname_reg, p.name, flags) is not None: - content_l.append(get_file_desc(p)) + if variable_mapping: + content_l.append(get_file_desc(p, variable_mapping.get(p.stem, []))) + else: + content_l.append(get_file_desc(p)) return "\n----------------- file splitter -------------\n".join(content_l) diff --git a/rdagent/scenarios/qlib/prompts.yaml b/rdagent/scenarios/qlib/prompts.yaml index 95f9a531..dc84f8de 100644 --- a/rdagent/scenarios/qlib/prompts.yaml +++ b/rdagent/scenarios/qlib/prompts.yaml @@ -1,5 +1,5 @@ hypothesis_and_feedback: |- - {% for hypothesis, experiment, feedback in trace.hist %} + {% for hypothesis, experiment, feedback in trace.hist[-10:] %} Hypothesis {{ loop.index }}: {{ hypothesis }} Corresponding Code (that leads to the difference in performance): {{experiment.sub_workspace_list[0].code_dict.get("model.py")}} Observation on the result with the hypothesis: {{ feedback.observations }} diff --git a/rdagent/scenarios/qlib/proposal/factor_proposal.py b/rdagent/scenarios/qlib/proposal/factor_proposal.py index 3856a179..204e9bcb 100644 --- a/rdagent/scenarios/qlib/proposal/factor_proposal.py +++ b/rdagent/scenarios/qlib/proposal/factor_proposal.py @@ -5,18 +5,14 @@ from typing import List, Tuple from jinja2 import Environment, StrictUndefined from rdagent.components.coder.factor_coder.factor import FactorExperiment, FactorTask -from rdagent.components.proposal.factor_proposal import ( - FactorHypothesis, - FactorHypothesis2Experiment, - FactorHypothesisGen, -) +from rdagent.components.proposal import FactorHypothesis2Experiment, FactorHypothesisGen from rdagent.core.prompts import Prompts from rdagent.core.proposal import Hypothesis, Scenario, Trace from rdagent.scenarios.qlib.experiment.factor_experiment import QlibFactorExperiment prompt_dict = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml") -QlibFactorHypothesis = FactorHypothesis +QlibFactorHypothesis = Hypothesis class QlibFactorHypothesisGen(FactorHypothesisGen): @@ -41,7 +37,7 @@ class QlibFactorHypothesisGen(FactorHypothesisGen): } return context_dict, True - def convert_response(self, response: str) -> FactorHypothesis: + def convert_response(self, response: str) -> Hypothesis: response_dict = json.loads(response) hypothesis = QlibFactorHypothesis( hypothesis=response_dict["hypothesis"], diff --git a/rdagent/scenarios/qlib/proposal/model_proposal.py b/rdagent/scenarios/qlib/proposal/model_proposal.py index 614b25ba..51d92b03 100644 --- a/rdagent/scenarios/qlib/proposal/model_proposal.py +++ b/rdagent/scenarios/qlib/proposal/model_proposal.py @@ -5,18 +5,14 @@ from typing import List, Tuple from jinja2 import Environment, StrictUndefined from rdagent.components.coder.model_coder.model import ModelExperiment, ModelTask -from rdagent.components.proposal.model_proposal import ( - ModelHypothesis, - ModelHypothesis2Experiment, - ModelHypothesisGen, -) +from rdagent.components.proposal import ModelHypothesis2Experiment, ModelHypothesisGen from rdagent.core.prompts import Prompts from rdagent.core.proposal import Hypothesis, Scenario, Trace from rdagent.scenarios.qlib.experiment.model_experiment import QlibModelExperiment prompt_dict = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml") -QlibModelHypothesis = ModelHypothesis +QlibModelHypothesis = Hypothesis class QlibModelHypothesisGen(ModelHypothesisGen): @@ -41,7 +37,7 @@ class QlibModelHypothesisGen(ModelHypothesisGen): } return context_dict, True - def convert_response(self, response: str) -> ModelHypothesis: + def convert_response(self, response: str) -> Hypothesis: response_dict = json.loads(response) hypothesis = QlibModelHypothesis( hypothesis=response_dict["hypothesis"], diff --git a/rdagent/utils/env.py b/rdagent/utils/env.py index d3104994..20e15915 100644 --- a/rdagent/utils/env.py +++ b/rdagent/utils/env.py @@ -15,6 +15,7 @@ import sys import uuid import zipfile from abc import abstractmethod +from concurrent.futures import ThreadPoolExecutor, TimeoutError from pathlib import Path from typing import Dict, Generic, Optional, TypeVar @@ -138,6 +139,8 @@ class DockerConf(BaseSettings): enable_gpu: bool = True # because we will automatically disable GPU if not available. So we enable it by default. mem_limit: str | None = "48g" # Add memory limit attribute + running_timeout_period: int = 3600 # 1 hour + class QlibDockerConf(DockerConf): class Config: @@ -186,6 +189,8 @@ class KGDockerConf(DockerConf): # Path("git_ignore_folder/data").resolve(): "/root/.data/" # } + running_timeout_period: int = 600 + # physionet.org/files/mimic-eicu-fiddle-feature/1.0.0/FIDDLE_mimic3 class DockerEnv(Env[DockerConf]): @@ -269,7 +274,7 @@ class DockerEnv(Env[DockerConf]): return {} return gpu_kwargs - def run( + def __run( self, entry: str | None = None, local_path: str | None = None, @@ -334,6 +339,20 @@ class DockerEnv(Env[DockerConf]): except docker.errors.APIError as e: raise RuntimeError(f"Error while running the container: {e}") + def run( + self, + entry: str | None = None, + local_path: str | None = None, + env: dict | None = None, + running_extra_volume: dict | None = None, + ): + with ThreadPoolExecutor() as executor: + future = executor.submit(self.__run, entry, local_path, env, running_extra_volume) + try: + return future.result(timeout=self.conf.running_timeout_period) + except TimeoutError: + raise TimeoutError(f"Timeout while running the container: {self.conf.running_timeout_period} seconds") + def dump_python_code_run_and_get_results( self, code: str,