gavindiaz/NexQuant
1
0
Fork 0
mirror of https://github.com/NicolasBohn/NexQuant.git synced 2026-07-27 15:37:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(kaggle): several update in kaggle scenarios (#476)

Browse Source 
* udpate plot

* log and reduce token

* trace tag

* add simple_background parameter to get_scenario_all_desc

* update trace

* update first version code

* chat model map

* add annotation for stack index

* add annotation

* reformatted by black

* several update on kaggle scenarios

* update some new change

* fix CI

* fix CI

* fix a bug

* fix bugs in graph RAG

---------

Co-authored-by: Tim <illking@foxmail.com>
This commit is contained in:
Xu Yang
2024-11-06 13:14:35 +08:00
committed by GitHub
parent 5986f8f4db
commit c095e4992f
49 changed files with 535 additions and 400 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rdagent/app/benchmark/factor/analysis.py
+22 -17
View File
@@ -82,12 +82,12 @@ class BenchmarkAnalyzer:
for i in x:
order_v.append(
{
"avg. Run successful rate": 0,
"avg. Format successful rate": 1,
"avg. Correlation (value only)": 2,
"max. Correlation": 3,
"max. accuracy": 4,
"avg. accuracy": 5,
"Avg Run SR": 0,
"Avg Format SR": 1,
"Avg Correlation": 2,
"Max Correlation": 3,
"Max Accuracy": 4,
"Avg Accuracy": 5,
}.get(i, i),
)
return order_v
@@ -140,12 +140,12 @@ class BenchmarkAnalyzer:
result_all = pd.concat(
{
"avg. Correlation (value only)": corr_res.iloc[:, 0],
"avg. Format successful rate": format_succ_rate_f.iloc[:, 0],
"avg. Run successful rate": succ_rate_f.iloc[:, 0],
"max. Correlation": corr_max_res.iloc[:, 0],
"max. accuracy": value_max_res.iloc[:, 0],
"avg. accuracy": value_avg_res.iloc[:, 0],
"Avg Correlation": corr_res.iloc[:, 0],
"Avg Format SR": format_succ_rate_f.iloc[:, 0],
"Avg Run SR": succ_rate_f.iloc[:, 0],
"Max Correlation": corr_max_res.iloc[:, 0],
"Max Accuracy": value_max_res.iloc[:, 0],
"Avg Accuracy": value_avg_res.iloc[:, 0],
},
axis=1,
)
@@ -179,11 +179,16 @@ class Plotter:
@staticmethod
def plot_data(data, file_name, title):
plt.figure(figsize=(10, 6))
sns.barplot(x="index", y="b", hue="a", data=data)
plt.xlabel("Method")
plt.figure(figsize=(10, 10))
plt.ylabel("Value")
plt.title(title)
colors = ["#3274A1", "#E1812C", "#3A923A", "#C03D3E"]
plt.bar(data["a"], data["b"], color=colors, capsize=5)
for idx, row in data.iterrows():
plt.text(idx, row["b"] + 0.01, f"{row['b']:.2f}", ha="center", va="bottom")
plt.suptitle(title, y=0.98)
plt.xticks(rotation=45)
plt.ylim(0, 1)
plt.tight_layout()
plt.savefig(file_name)
@@ -201,7 +206,7 @@ def main(
final_results_df = pd.DataFrame(final_results)
Plotter.change_fs(20)
plot_data = final_results_df.drop(["max. accuracy", "avg. accuracy"], axis=0).T
plot_data = final_results_df.drop(["Max Accuracy", "Avg Accuracy"], axis=0).T
plot_data = plot_data.reset_index().melt("index", var_name="a", value_name="b")
Plotter.plot_data(plot_data, "./comparison_plot.png", title)
rdagent/app/kaggle/loop.py
+2 -1
View File
@@ -1,5 +1,6 @@
import subprocess
from collections import defaultdict
from concurrent.futures import TimeoutError
from typing import Any
import fire
@@ -115,7 +116,7 @@ class KaggleRDLoop(RDLoop):
return exp
skip_loop_error = (ModelEmptyError, FactorEmptyError)
skip_loop_error = (ModelEmptyError, FactorEmptyError, TimeoutError)
def main(path=None, step_n=None, competition=None):
rdagent/components/coder/factor_coder/CoSTEER/evaluators.py
+9 -4
View File
@@ -8,6 +8,7 @@ from typing import List, Tuple
import pandas as pd
from jinja2 import Environment, StrictUndefined
from rdagent.components.coder.factor_coder.config import FACTOR_IMPLEMENT_SETTINGS
from rdagent.components.coder.factor_coder.CoSTEER.evolvable_subjects import (
FactorEvolvingItem,
)
@@ -92,7 +93,11 @@ class FactorCodeEvaluator(FactorEvaluator):
.from_string(evaluate_prompts["evaluator_code_feedback_v1_system"])
.render(
scenario=(
self.scen.get_scenario_all_desc(target_task)
self.scen.get_scenario_all_desc(
target_task,
filtered_tag="feature",
simple_background=FACTOR_IMPLEMENT_SETTINGS.simple_background,
)
if self.scen is not None
else "No scenario description."
)
@@ -190,7 +195,7 @@ class FactorOutputFormatEvaluator(FactorEvaluator):
)
buffer = io.StringIO()
gen_df.info(buf=buffer)
gen_df_info_str = f"The use is currently working on a feature related task.\nThe output dataframe info is:\n{buffer.getvalue()}"
gen_df_info_str = f"The user is currently working on a feature related task.\nThe output dataframe info is:\n{buffer.getvalue()}"
system_prompt = (
Environment(undefined=StrictUndefined)
.from_string(
@@ -198,7 +203,7 @@ class FactorOutputFormatEvaluator(FactorEvaluator):
)
.render(
scenario=(
self.scen.get_scenario_all_desc(implementation.target_task)
self.scen.get_scenario_all_desc(implementation.target_task, filtered_tag="feature")
if self.scen is not None
else "No scenario description."
)
@@ -512,7 +517,7 @@ class FactorFinalDecisionEvaluator(Evaluator):
.from_string(evaluate_prompts["evaluator_final_decision_v1_system"])
.render(
scenario=(
self.scen.get_scenario_all_desc(target_task)
self.scen.get_scenario_all_desc(target_task, filtered_tag="feature")
if self.scen is not None
else "No scenario description."
)
rdagent/components/coder/factor_coder/CoSTEER/evolving_strategy.py
+1 -1
View File
@@ -234,7 +234,7 @@ class FactorEvolvingStrategyWithGraph(MultiProcessEvolvingStrategy):
implement_prompts["evolving_strategy_factor_implementation_v1_system"],
)
.render(
scenario=self.scen.get_scenario_all_desc(target_task),
scenario=self.scen.get_scenario_all_desc(target_task, filtered_tag="feature"),
queried_former_failed_knowledge=queried_former_failed_knowledge_to_render,
)
)
rdagent/components/coder/factor_coder/config.py
+3
View File
@@ -35,6 +35,9 @@ class FactorImplementSettings(BaseSettings):
v2_error_summary: bool = False
v2_knowledge_sampler: float = 1.0
simple_background: bool = False
"""Whether to use simple background information for code feedback"""
file_based_execution_timeout: int = 120
"""Timeout in seconds for each factor implementation execution"""
rdagent/components/coder/model_coder/CoSTEER/evaluators.py
+2 -2
View File
@@ -83,7 +83,7 @@ class ModelCodeEvaluator(Evaluator):
.from_string(evaluate_prompts["evaluator_code_feedback"]["system"])
.render(
scenario=(
self.scen.get_scenario_all_desc(target_task)
self.scen.get_scenario_all_desc(target_task, filtered_tag=target_task.model_type)
if self.scen is not None
else "No scenario description."
)
@@ -145,7 +145,7 @@ class ModelFinalEvaluator(Evaluator):
.from_string(evaluate_prompts["evaluator_final_feedback"]["system"])
.render(
scenario=(
self.scen.get_scenario_all_desc(target_task)
self.scen.get_scenario_all_desc(target_task, filtered_tag=target_task.model_type)
if self.scen is not None
else "No scenario description."
)
rdagent/components/coder/model_coder/CoSTEER/evolving_strategy.py
+1 -1
View File
@@ -76,7 +76,7 @@ class ModelCoderEvolvingStrategy(EvolvingStrategy):
coder_prompts["evolving_strategy_model_coder"]["system"],
)
.render(
scenario=self.scen.get_scenario_all_desc(),
scenario=self.scen.get_scenario_all_desc(filtered_tag=target_task.model_type),
queried_former_failed_knowledge=queried_former_failed_knowledge_to_render,
current_code=current_code,
)
rdagent/components/coder/model_coder/model_execute_template_v2.txt
+2
View File
@@ -13,9 +13,11 @@ valid_y = pd.Series(np.random.randint(0, 2, 8))
model = fit(train_X, train_y, valid_X, valid_y)
execution_model_output = predict(model, valid_X)
if isinstance(execution_model_output, torch.Tensor):
execution_model_output = execution_model_output.cpu().detach().numpy()
execution_feedback_str = f"Execution successful, output numpy ndarray shape: {execution_model_output.shape}"
pickle.dump(execution_model_output, open("execution_model_output.pkl", "wb"))
rdagent/components/proposal/factor_proposal.py → rdagent/components/proposal/__init__.py
+49 -15
View File
@@ -4,7 +4,7 @@ from typing import Tuple
from jinja2 import Environment, StrictUndefined
from rdagent.components.coder.factor_coder.factor import FactorExperiment
from rdagent.core.experiment import Experiment
from rdagent.core.prompts import Prompts
from rdagent.core.proposal import (
Hypothesis,
@@ -18,10 +18,8 @@ from rdagent.oai.llm_utils import APIBackend
prompt_dict = Prompts(file_path=Path(__file__).parent / "prompts.yaml")
FactorHypothesis = Hypothesis
class LLMHypothesisGen(HypothesisGen):
class FactorHypothesisGen(HypothesisGen):
def __init__(self, scen: Scenario):
super().__init__(scen)
@@ -30,17 +28,17 @@ class FactorHypothesisGen(HypothesisGen):
def prepare_context(self, trace: Trace) -> Tuple[dict, bool]: ...
@abstractmethod
def convert_response(self, response: str) -> FactorHypothesis: ...
def convert_response(self, response: str) -> Hypothesis: ...
def gen(self, trace: Trace) -> FactorHypothesis:
def gen(self, trace: Trace) -> Hypothesis:
context_dict, json_flag = self.prepare_context(trace)
system_prompt = (
Environment(undefined=StrictUndefined)
.from_string(prompt_dict["hypothesis_gen"]["system_prompt"])
.render(
targets="factors",
scenario=self.scen.get_scenario_all_desc(),
targets=self.targets,
scenario=self.scen.get_scenario_all_desc(filtered_tag="hypothesis_and_experiment"),
hypothesis_output_format=context_dict["hypothesis_output_format"],
hypothesis_specification=context_dict["hypothesis_specification"],
)
@@ -49,7 +47,7 @@ class FactorHypothesisGen(HypothesisGen):
Environment(undefined=StrictUndefined)
.from_string(prompt_dict["hypothesis_gen"]["user_prompt"])
.render(
targets="factors",
targets=self.targets,
hypothesis_and_feedback=context_dict["hypothesis_and_feedback"],
RAG=context_dict["RAG"],
)
@@ -62,21 +60,39 @@ class FactorHypothesisGen(HypothesisGen):
return hypothesis
class FactorHypothesis2Experiment(Hypothesis2Experiment[FactorExperiment]):
class FactorHypothesisGen(LLMHypothesisGen):
def __init__(self, scen: Scenario):
super().__init__(scen)
self.targets = "factors"
class ModelHypothesisGen(LLMHypothesisGen):
def __init__(self, scen: Scenario):
super().__init__(scen)
self.targets = "model tuning"
class FactorAndModelHypothesisGen(FactorHypothesisGen):
def __init__(self, scen: Scenario):
super().__init__(scen)
self.targets = "feature engineering and model building"
class LLMHypothesis2Experiment(Hypothesis2Experiment[Experiment]):
@abstractmethod
def prepare_context(self, hypothesis: Hypothesis, trace: Trace) -> Tuple[dict, bool]: ...
@abstractmethod
def convert_response(self, response: str, trace: Trace) -> FactorExperiment: ...
def convert_response(self, response: str, trace: Trace) -> Experiment: ...
def convert(self, hypothesis: Hypothesis, trace: Trace) -> FactorExperiment:
def convert(self, hypothesis: Hypothesis, trace: Trace) -> Experiment:
context, json_flag = self.prepare_context(hypothesis, trace)
system_prompt = (
Environment(undefined=StrictUndefined)
.from_string(prompt_dict["hypothesis2experiment"]["system_prompt"])
.render(
targets="factors",
scenario=trace.scen.get_scenario_all_desc(),
targets=self.targets,
scenario=trace.scen.get_scenario_all_desc(filtered_tag="hypothesis_and_experiment"),
experiment_output_format=context["experiment_output_format"],
)
)
@@ -84,7 +100,7 @@ class FactorHypothesis2Experiment(Hypothesis2Experiment[FactorExperiment]):
Environment(undefined=StrictUndefined)
.from_string(prompt_dict["hypothesis2experiment"]["user_prompt"])
.render(
targets="factors",
targets=self.targets,
target_hypothesis=context["target_hypothesis"],
hypothesis_and_feedback=context["hypothesis_and_feedback"],
target_list=context["target_list"],
@@ -95,3 +111,21 @@ class FactorHypothesis2Experiment(Hypothesis2Experiment[FactorExperiment]):
resp = APIBackend().build_messages_and_create_chat_completion(user_prompt, system_prompt, json_mode=json_flag)
return self.convert_response(resp, trace)
class FactorHypothesis2Experiment(LLMHypothesis2Experiment):
def __init__(self):
super().__init__()
self.targets = "factors"
class ModelHypothesis2Experiment(LLMHypothesis2Experiment):
def __init__(self):
super().__init__()
self.targets = "model tuning"
class FactorAndModelHypothesis2Experiment(LLMHypothesis2Experiment):
def __init__(self):
super().__init__()
self.targets = "feature engineering and model building"
rdagent/components/proposal/model_proposal.py
-99
View File
@@ -1,99 +0,0 @@
from abc import abstractmethod
from pathlib import Path
from typing import Tuple
from jinja2 import Environment, StrictUndefined
from rdagent.components.coder.model_coder.model import ModelExperiment
from rdagent.core.prompts import Prompts
from rdagent.core.proposal import (
Hypothesis,
Hypothesis2Experiment,
HypothesisGen,
Scenario,
Trace,
)
from rdagent.oai.llm_utils import APIBackend
ModelHypothesis = Hypothesis
prompt_dict = Prompts(file_path=Path(__file__).parent / "prompts.yaml")
class ModelHypothesisGen(HypothesisGen):
prompts: Prompts = prompt_dict
# The following methods are scenario related so they should be implemented in the subclass
@abstractmethod
def prepare_context(self, trace: Trace) -> Tuple[dict, bool]: ...
@abstractmethod
def convert_response(self, response: str) -> ModelHypothesis: ...
def gen(self, trace: Trace) -> ModelHypothesis:
context_dict, json_flag = self.prepare_context(trace)
system_prompt = (
Environment(undefined=StrictUndefined)
.from_string(ModelHypothesisGen.prompts["hypothesis_gen"]["system_prompt"])
.render(
targets="model tuning",
scenario=self.scen.get_scenario_all_desc(),
hypothesis_output_format=context_dict["hypothesis_output_format"],
hypothesis_specification=context_dict["hypothesis_specification"],
)
)
user_prompt = (
Environment(undefined=StrictUndefined)
.from_string(ModelHypothesisGen.prompts["hypothesis_gen"]["user_prompt"])
.render(
targets="model tuning",
RAG=context_dict["RAG"],
)
)
resp = APIBackend().build_messages_and_create_chat_completion(user_prompt, system_prompt, json_mode=json_flag)
hypothesis = self.convert_response(resp)
return hypothesis
class ModelHypothesis2Experiment(Hypothesis2Experiment[ModelExperiment]):
prompts: Prompts = prompt_dict
def __init__(self) -> None:
super().__init__()
@abstractmethod
def prepare_context(self, hypothesis: Hypothesis, trace: Trace) -> Tuple[dict, bool]: ...
@abstractmethod
def convert_response(self, response: str, trace: Trace) -> ModelExperiment: ...
def convert(self, hypothesis: Hypothesis, trace: Trace) -> ModelExperiment:
context, json_flag = self.prepare_context(hypothesis, trace)
system_prompt = (
Environment(undefined=StrictUndefined)
.from_string(ModelHypothesis2Experiment.prompts["hypothesis2experiment"]["system_prompt"])
.render(
targets="feature engineering and model building",
scenario=trace.scen.get_scenario_all_desc(),
experiment_output_format=context["experiment_output_format"],
)
)
user_prompt = (
Environment(undefined=StrictUndefined)
.from_string(ModelHypothesis2Experiment.prompts["hypothesis2experiment"]["user_prompt"])
.render(
targets="feature engineering and model building",
target_hypothesis=context["target_hypothesis"],
hypothesis_and_feedback=context["hypothesis_and_feedback"],
target_list=context["target_list"],
RAG=context["RAG"],
)
)
resp = APIBackend().build_messages_and_create_chat_completion(user_prompt, system_prompt, json_mode=json_flag)
return self.convert_response(resp, trace)
rdagent/components/proposal/prompts.yaml
+5
View File
@@ -13,6 +13,11 @@ hypothesis_gen:
{{ hypothesis_output_format }}
user_prompt: |-
{% if hypothesis_and_feedback|length == 0 %}It is the first round of hypothesis generation. The user has no hypothesis on this scenario yet.
{% else %}It is not the first round, the user has made several hypothesis on this scenario and did several evaluation on them.
The former hypothesis and the corresponding feedbacks are as follows (focus on the last one & the new hypothesis that it provides and reasoning to see if you agree):
{{ hypothesis_and_feedback }}
{% endif %}
{% if RAG %}
To assist you in generating new {{targets}}, we have provided the following information: {{RAG}}.
**Note:** The provided RAG is for reference only.
rdagent/components/runner/__init__.py
+8
View File
@@ -1,4 +1,5 @@
import pickle
import shutil
from pathlib import Path
from typing import Any, Tuple
@@ -20,5 +21,12 @@ class CachedRunner(Developer[ASpecificExp]):
def assign_cached_result(self, exp: Experiment, cached_res: Experiment) -> Experiment:
if exp.based_experiments and exp.based_experiments[-1].result is None:
exp.based_experiments[-1].result = cached_res.based_experiments[-1].result
if cached_res.experiment_workspace.workspace_path.exists():
for csv_file in cached_res.experiment_workspace.workspace_path.glob("*.csv"):
shutil.copy(csv_file, exp.experiment_workspace.workspace_path)
for py_file in (cached_res.experiment_workspace.workspace_path / "feature").glob("*.py"):
shutil.copy(py_file, exp.experiment_workspace.workspace_path / "feature")
for py_file in (cached_res.experiment_workspace.workspace_path / "model").glob("*.py"):
shutil.copy(py_file, exp.experiment_workspace.workspace_path / "model")
exp.result = cached_res.result
return exp
rdagent/core/knowledge_base.py
+2 -2
View File
@@ -15,9 +15,9 @@ class KnowledgeBase:
with self.path.open("rb") as f:
loaded = pickle.load(f)
if isinstance(loaded, dict):
self.__dict__.update(loaded)
self.__dict__.update({k: v for k, v in loaded.items() if not k == "path"})
else:
self.__dict__.update(loaded.__dict__)
self.__dict__.update({k: v for k, v in loaded.__dict__.items() if not k == "path"})
def dump(self) -> None:
if self.path is not None:
rdagent/core/scenario.py
+6 -1
View File
@@ -46,7 +46,12 @@ class Scenario(ABC):
"""Rich style description to present"""
@abstractmethod
def get_scenario_all_desc(self, task: Task | None = None) -> str:
def get_scenario_all_desc(
self,
task: Task | None = None,
filtered_tag: str | None = None,
simple_background: bool | None = None,
) -> str:
"""
Combine all descriptions together
rdagent/log/ui/app.py
+15
View File
@@ -147,6 +147,21 @@ def get_msgs_until(end_func: Callable[[Message], bool] = lambda _: True):
sms.name = "alpha158"
state.alpha158_metrics = sms
if (
state.lround == 1
and len(msg.content.based_experiments) > 0
and msg.content.based_experiments[-1].result is not None
):
sms = msg.content.based_experiments[-1].result
if isinstance(state.scenario, DMModelScenario):
sms.index = ["AUROC"]
elif isinstance(
state.scenario, (QlibModelScenario, QlibFactorFromReportScenario, QlibFactorScenario)
):
sms = sms.loc[QLIB_SELECTED_METRICS]
sms.name = f"Baseline"
state.metric_series.append(sms)
# common metrics
if msg.content.result is None:
if isinstance(state.scenario, DMModelScenario):
rdagent/oai/llm_conf.py
+2
View File
@@ -90,5 +90,7 @@ class LLMSettings(BaseSettings):
gcr_endpoint_do_sample: bool = False
gcr_endpoint_max_token: int = 100
chat_model_map: str = "{}"
LLM_SETTINGS = LLMSettings()
rdagent/oai/llm_utils.py
+24 -1
View File
@@ -1,6 +1,7 @@
from __future__ import annotations
import hashlib
import inspect
import json
import os
import random
@@ -325,6 +326,7 @@ class APIBackend:
)
self.chat_model = LLM_SETTINGS.chat_model if chat_model is None else chat_model
self.chat_model_map = json.loads(LLM_SETTINGS.chat_model_map)
self.encoder = tiktoken.encoding_for_model(self.chat_model)
self.chat_api_base = LLM_SETTINGS.chat_azure_api_base if chat_api_base is None else chat_api_base
self.chat_api_version = (
@@ -627,6 +629,15 @@ class APIBackend:
if presence_penalty is None:
presence_penalty = LLM_SETTINGS.chat_presence_penalty
# Use index 4 to skip the current function and intermediate calls,
# and get the locals of the caller's frame.
caller_locals = inspect.stack()[4].frame.f_locals
if "self" in caller_locals:
tag = caller_locals["self"].__class__.__name__
else:
tag = inspect.stack()[4].function
model = self.chat_model_map.get(tag, self.chat_model)
finish_reason = None
if self.use_llama2:
response = self.generator.chat_completion(
@@ -661,7 +672,7 @@ class APIBackend:
logger.info(f"{LogColors.CYAN}Response:{resp}{LogColors.END}", tag="llm_messages")
else:
kwargs = dict(
model=self.chat_model,
model=model,
messages=messages,
max_tokens=max_tokens,
temperature=temperature,
@@ -705,6 +716,18 @@ class APIBackend:
finish_reason = response.choices[0].finish_reason
if LLM_SETTINGS.log_llm_chat_content:
logger.info(f"{LogColors.CYAN}Response:{resp}{LogColors.END}", tag="llm_messages")
logger.info(
json.dumps(
{
"tag": tag,
"total_tokens": response.usage.total_tokens,
"prompt_tokens": response.usage.prompt_tokens,
"completion_tokens": response.usage.completion_tokens,
"model": model,
}
),
tag="llm_messages",
)
if json_mode:
json.loads(resp)
if self.dump_chat_cache:
rdagent/scenarios/data_mining/experiment/model_experiment.py
+3 -1
View File
@@ -63,7 +63,9 @@ The demo showcases the iterative process of hypothesis generation, knowledge con
To demonstrate the dynamic evolution of models through the R&D loop, emphasizing how each iteration enhances the model performance and reliability. The performane is measured by the AUROC score (Area Under the Receiver Operating Characteristic), which is a commonly used metric for binary classification. """
def get_scenario_all_desc(self, task: Task | None = None) -> str:
def get_scenario_all_desc(
self, task: Task | None = None, filtered_tag: str | None = None, simple_background: bool | None = None
) -> str:
return f"""Background of the scenario:
{self.background}
The interface you should follow to write the runnable code:
rdagent/scenarios/data_mining/proposal/model_proposal.py
+4 -4
View File
@@ -5,8 +5,8 @@ from typing import List, Tuple
from jinja2 import Environment, StrictUndefined
from rdagent.components.coder.model_coder.model import ModelExperiment, ModelTask
from rdagent.components.proposal.model_proposal import (
ModelHypothesis,
from rdagent.components.proposal import (
Hypothesis,
ModelHypothesis2Experiment,
ModelHypothesisGen,
)
@@ -16,7 +16,7 @@ from rdagent.scenarios.data_mining.experiment.model_experiment import DMModelExp
prompt_dict = Prompts(file_path=Path(__file__).parent.parent.parent / "qlib" / "prompts.yaml")
DMModelHypothesis = ModelHypothesis
DMModelHypothesis = Hypothesis
class DMModelHypothesisGen(ModelHypothesisGen):
@@ -52,7 +52,7 @@ class DMModelHypothesisGen(ModelHypothesisGen):
}
return context_dict, True
def convert_response(self, response: str) -> ModelHypothesis:
def convert_response(self, response: str) -> Hypothesis:
response_dict = json.loads(response)
hypothesis = DMModelHypothesis(
hypothesis=response_dict["hypothesis"],
rdagent/scenarios/general_model/scenario.py
+3 -1
View File
@@ -41,7 +41,9 @@ class GeneralModelScenario(Scenario):
def rich_style_description(self) -> str:
return self._rich_style_description
def get_scenario_all_desc(self, task: Task | None = None) -> str:
def get_scenario_all_desc(
self, task: Task | None = None, filtered_tag: str | None = None, simple_background: bool | None = None
) -> str:
return f"""Background of the scenario:
{self.background}
The interface you should follow to write the runnable code:
rdagent/scenarios/kaggle/developer/feedback.py
+1 -1
View File
@@ -101,7 +101,7 @@ class KGHypothesisExperiment2Feedback(HypothesisExperiment2Feedback):
sys_prompt = (
Environment(undefined=StrictUndefined)
.from_string(prompt_dict[prompt_key]["system"])
.render(scenario=self.scen.get_scenario_all_desc())
.render(scenario=self.scen.get_scenario_all_desc(filtered_tag="feedback"))
)
last_task_and_code = None
rdagent/scenarios/kaggle/developer/runner.py
+2 -5
View File
@@ -25,11 +25,8 @@ class KGCachedRunner(CachedRunner[ASpecificExp]):
for f in sorted((exp.experiment_workspace.workspace_path / "model").glob("*.py"), key=lambda x: x.name):
codes.append(f.read_text())
codes = "\n".join(codes)
if exp.sub_workspace_list is not None:
for i in range(len(exp.sub_workspace_list)):
if exp.sub_workspace_list[i] is not None:
codes += str(exp.sub_workspace_list[i].code_dict.values())
return md5_hash(codes)
cached_key_from_exp = CachedRunner.get_cache_key(self, exp)
return md5_hash(codes + cached_key_from_exp)
@cache_with_pickle(get_cache_key, CachedRunner.assign_cached_result)
def init_develop(self, exp: KGFactorExperiment | KGModelExperiment) -> KGFactorExperiment | KGModelExperiment:
rdagent/scenarios/kaggle/experiment/covid19-global-forecasting-week-1_template/train.py
+3 -3
View File
@@ -36,9 +36,9 @@ X_test_l = []
for f in DIRNAME.glob("feature/feat*.py"):
cls = import_module_from_path(f.stem, f).feature_engineering_cls()
cls.fit(X_train)
X_train_f = cls.transform(X_train)
X_valid_f = cls.transform(X_valid)
X_test_f = cls.transform(X_test)
X_train_f = cls.transform(X_train.copy())
X_valid_f = cls.transform(X_valid.copy())
X_test_f = cls.transform(X_test.copy())
if X_train_f.shape[-1] == X_valid_f.shape[-1] and X_train_f.shape[-1] == X_test_f.shape[-1]:
X_train_l.append(X_train_f)
rdagent/scenarios/kaggle/experiment/digit-recognizer_template/train.py
+3 -3
View File
@@ -36,9 +36,9 @@ X_test_l = []
for f in DIRNAME.glob("feature/feat*.py"):
cls = import_module_from_path(f.stem, f).feature_engineering_cls()
cls.fit(X_train)
X_train_f = cls.transform(X_train)
X_valid_f = cls.transform(X_valid)
X_test_f = cls.transform(X_test)
X_train_f = cls.transform(X_train.copy())
X_valid_f = cls.transform(X_valid.copy())
X_test_f = cls.transform(X_test.copy())
if X_train_f.shape[-1] == X_valid_f.shape[-1] and X_train_f.shape[-1] == X_test_f.shape[-1]:
X_train_l.append(X_train_f)
rdagent/scenarios/kaggle/experiment/feedback-prize-english-language-learning_template/train.py
+3 -3
View File
@@ -30,9 +30,9 @@ for f in DIRNAME.glob("feature/feat*.py"):
cls = import_module_from_path(f.stem, f).feature_engineering_cls()
print(X_train.head())
cls.fit(X_train)
X_train_f = cls.transform(X_train)
X_valid_f = cls.transform(X_valid)
X_test_f = cls.transform(X_test)
X_train_f = cls.transform(X_train.copy())
X_valid_f = cls.transform(X_valid.copy())
X_test_f = cls.transform(X_test.copy())
if X_train_f.shape[-1] == X_valid_f.shape[-1] and X_train_f.shape[-1] == X_test_f.shape[-1]:
X_train_l.append(X_train_f)
rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/train.py
+3 -3
View File
@@ -37,9 +37,9 @@ X_test_l = []
for f in DIRNAME.glob("feature/feat*.py"):
cls = import_module_from_path(f.stem, f).feature_engineering_cls()
cls.fit(X_train)
X_train_f = cls.transform(X_train)
X_valid_f = cls.transform(X_valid)
X_test_f = cls.transform(X_test)
X_train_f = cls.transform(X_train.copy())
X_valid_f = cls.transform(X_valid.copy())
X_test_f = cls.transform(X_test.copy())
if X_train_f.shape[-1] == X_valid_f.shape[-1] and X_train_f.shape[-1] == X_test_f.shape[-1]:
X_train_l.append(X_train_f)
rdagent/scenarios/kaggle/experiment/meta_tpl_deprecated/train.py
+3 -3
View File
@@ -45,9 +45,9 @@ X_test_l = []
for f in DIRNAME.glob("feature/feat*.py"):
cls = import_module_from_path(f.stem, f).feature_engineering_cls()
cls.fit(X_train)
X_train_f = cls.transform(X_train)
X_valid_f = cls.transform(X_valid)
X_test_f = cls.transform(X_test)
X_train_f = cls.transform(X_train.copy())
X_valid_f = cls.transform(X_valid.copy())
X_test_f = cls.transform(X_test.copy())
X_train_l.append(X_train_f)
X_valid_l.append(X_valid_f)
rdagent/scenarios/kaggle/experiment/optiver-realized-volatility-prediction_template/train.py
+3 -3
View File
@@ -38,9 +38,9 @@ X_test_l = []
for f in DIRNAME.glob("feature/feat*.py"):
cls = import_module_from_path(f.stem, f).feature_engineering_cls()
cls.fit(X_train)
X_train_f = cls.transform(X_train)
X_valid_f = cls.transform(X_valid)
X_test_f = cls.transform(X_test)
X_train_f = cls.transform(X_train.copy())
X_valid_f = cls.transform(X_valid.copy())
X_test_f = cls.transform(X_test.copy())
if X_train_f.shape[-1] == X_valid_f.shape[-1] and X_train_f.shape[-1] == X_test_f.shape[-1]:
X_train_l.append(X_train_f)
rdagent/scenarios/kaggle/experiment/playground-series-s3e11_template/train.py
+3 -3
View File
@@ -26,9 +26,9 @@ X_test_l = []
for f in DIRNAME.glob("feature/feat*.py"):
cls = import_module_from_path(f.stem, f).feature_engineering_cls()
cls.fit(X_train)
X_train_f = cls.transform(X_train)
X_valid_f = cls.transform(X_valid)
X_test_f = cls.transform(X_test)
X_train_f = cls.transform(X_train.copy())
X_valid_f = cls.transform(X_valid.copy())
X_test_f = cls.transform(X_test.copy())
if X_train_f.shape[-1] == X_valid_f.shape[-1] and X_train_f.shape[-1] == X_test_f.shape[-1]:
X_train_l.append(X_train_f)
rdagent/scenarios/kaggle/experiment/playground-series-s3e14_template/train.py
+3 -3
View File
@@ -26,9 +26,9 @@ X_test_l = []
for f in DIRNAME.glob("feature/feat*.py"):
cls = import_module_from_path(f.stem, f).feature_engineering_cls()
cls.fit(X_train)
X_train_f = cls.transform(X_train)
X_valid_f = cls.transform(X_valid)
X_test_f = cls.transform(X_test)
X_train_f = cls.transform(X_train.copy())
X_valid_f = cls.transform(X_valid.copy())
X_test_f = cls.transform(X_test.copy())
if X_train_f.shape[-1] == X_valid_f.shape[-1] and X_train_f.shape[-1] == X_test_f.shape[-1]:
X_train_l.append(X_train_f)
rdagent/scenarios/kaggle/experiment/playground-series-s3e16_template/train.py
+3 -3
View File
@@ -26,9 +26,9 @@ X_test_l = []
for f in DIRNAME.glob("feature/feat*.py"):
cls = import_module_from_path(f.stem, f).feature_engineering_cls()
cls.fit(X_train)
X_train_f = cls.transform(X_train)
X_valid_f = cls.transform(X_valid)
X_test_f = cls.transform(X_test)
X_train_f = cls.transform(X_train.copy())
X_valid_f = cls.transform(X_valid.copy())
X_test_f = cls.transform(X_test.copy())
if X_train_f.shape[-1] == X_valid_f.shape[-1] and X_train_f.shape[-1] == X_test_f.shape[-1]:
X_train_l.append(X_train_f)
rdagent/scenarios/kaggle/experiment/playground-series-s3e26_template/train.py
+3 -3
View File
@@ -40,9 +40,9 @@ X_test_l = []
for f in DIRNAME.glob("feature/feat*.py"):
cls = import_module_from_path(f.stem, f).feature_engineering_cls()
cls.fit(X_train)
X_train_f = cls.transform(X_train)
X_valid_f = cls.transform(X_valid)
X_test_f = cls.transform(X_test)
X_train_f = cls.transform(X_train.copy())
X_valid_f = cls.transform(X_valid.copy())
X_test_f = cls.transform(X_test.copy())
if X_train_f.shape[-1] == X_valid_f.shape[-1] and X_train_f.shape[-1] == X_test_f.shape[-1]:
X_train_l.append(X_train_f)
rdagent/scenarios/kaggle/experiment/playground-series-s4e5_template/train.py
+3 -3
View File
@@ -36,9 +36,9 @@ X_test_l = []
for f in DIRNAME.glob("feature/feat*.py"):
cls = import_module_from_path(f.stem, f).feature_engineering_cls()
cls.fit(X_train)
X_train_f = cls.transform(X_train)
X_valid_f = cls.transform(X_valid)
X_test_f = cls.transform(X_test)
X_train_f = cls.transform(X_train.copy())
X_valid_f = cls.transform(X_valid.copy())
X_test_f = cls.transform(X_test.copy())
if X_train_f.shape[-1] == X_valid_f.shape[-1] and X_train_f.shape[-1] == X_test_f.shape[-1]:
X_train_l.append(X_train_f)
rdagent/scenarios/kaggle/experiment/playground-series-s4e8_template/train.py
+3 -3
View File
@@ -38,9 +38,9 @@ X_test_l = []
for f in DIRNAME.glob("feature/feat*.py"):
cls = import_module_from_path(f.stem, f).feature_engineering_cls()
cls.fit(X_train)
X_train_f = cls.transform(X_train)
X_valid_f = cls.transform(X_valid)
X_test_f = cls.transform(X_test)
X_train_f = cls.transform(X_train.copy())
X_valid_f = cls.transform(X_valid.copy())
X_test_f = cls.transform(X_test.copy())
if X_train_f.shape[-1] == X_valid_f.shape[-1] and X_train_f.shape[-1] == X_test_f.shape[-1]:
X_train_l.append(X_train_f)
rdagent/scenarios/kaggle/experiment/playground-series-s4e9_template/train.py
+3 -3
View File
@@ -39,9 +39,9 @@ X_test_l = []
for f in DIRNAME.glob("feature/feat*.py"):
cls = import_module_from_path(f.stem, f).feature_engineering_cls()
cls.fit(X_train)
X_train_f = cls.transform(X_train)
X_valid_f = cls.transform(X_valid)
X_test_f = cls.transform(X_test)
X_train_f = cls.transform(X_train.copy())
X_valid_f = cls.transform(X_valid.copy())
X_test_f = cls.transform(X_test.copy())
if X_train_f.shape[-1] == X_valid_f.shape[-1] and X_train_f.shape[-1] == X_test_f.shape[-1]:
X_train_l.append(X_train_f)
rdagent/scenarios/kaggle/experiment/prompts.yaml
+22 -4
View File
@@ -64,7 +64,7 @@ kg_background: |-
- "Feature engineering": The user will design several new tasks and implement several new features. The new feature might only affect the model using all the feature book.
- "Feature processing": The user will design a new task to process the feature book like normalization or one hot encoding to improve the model performance. Any processing with help of a deep model is not included in this task.
- Model related:
- "Model feature selection": The user will modify one model to select the most important features from the feature book to improve the model performance.
- "Model feature selection": The user will modify one model to select the part of the features from the feature book to improve the model performance.
- "Model tuning": The user will tune the hyperparameters of XGBoost, RandomForest or LightGBM or build or improve the NN model to improve the model performance.
For each loop, you need to help user decide which action item to choose and provide the corresponding code to implement the action item.
@@ -142,6 +142,7 @@ kg_model_interface: |-
Here are some examples of how your Python code should be structured:
{% if tag == "XGBoost" or tag is none %}
For XGBoost:
```python
import pandas as pd
@@ -168,7 +169,8 @@ kg_model_interface: |-
return y_pred
```
{% endif %}
{% if tag == "RandomForest" or tag is none %}
For RandomForest:
```python
import pandas as pd
@@ -191,7 +193,8 @@ kg_model_interface: |-
return y_pred
```
{% endif %}
{% if tag == "LightGBM" or tag is none %}
For LightGBM:
```python
import pandas as pd
@@ -213,7 +216,8 @@ kg_model_interface: |-
return y_pred
```
{% endif %}
{% if tag == "NN" or tag is none %}
For Neural Network:
```python
import pandas as pd
@@ -271,6 +275,7 @@ kg_model_interface: |-
return y_pred
```
{% endif %}
kg_feature_simulator: |-
The data preprocessing method you provide will be used to prepare data by processing it, concatenating the results with other features, and removing unnecessary features before training the model.
@@ -283,6 +288,19 @@ kg_feature_simulator: |-
4. Train a model such as LightGBM, CatBoost, LSTM, or a simple PyTorch model using the processed data.
5. Evaluate the performance of your preprocessing method and provide feedback.
kg_feature_output_format: |-
The output should be a pandas DataFrame with the new features. The columns should be the new features, and the rows should correspond to the number of samples in the input DataFrame.
Sample output dataframe info:
<class 'pandas.core.frame.DataFrame'>
Index: {Same to the input DataFrame}
Data columns (total N columns):
# Column Dtype
--- ------ -----
0 feature_name_0 float64
1 feature_name_1 float64
dtypes: float64(N)
memory usage: {Memory usage of the output DataFrame}
kg_model_output_format: |-
For feature related tasks, the output should be a pandas DataFrame with the new features. The columns should be the new features, and the rows should correspond to the number of samples in the input DataFrame.
For model related tasks, the output should be an np.ndarray with the appropriate number of predictions.
rdagent/scenarios/kaggle/experiment/scenario.py
+98 -68
View File
@@ -41,7 +41,6 @@ class KGScenario(Scenario):
self.competition = competition
self.competition_descriptions = crawl_descriptions(competition)
self.input_shape = None
self._source_data = self.source_data
self.competition_type = None
self.competition_description = None
@@ -64,11 +63,6 @@ class KGScenario(Scenario):
self.vector_base.path = Path(datetime.now(timezone.utc).strftime("%Y-%m-%d-%H-%M-%S") + "_kaggle_kb.pkl")
self.vector_base.dump()
self._output_format = self.output_format
self._interface = self.interface
self._simulator = self.simulator
self._background = self.background
self.action_counts = dict.fromkeys(KG_ACTION_LIST, 0)
self.reward_estimates = {action: 0.0 for action in KG_ACTION_LIST}
# self.reward_estimates["Model feature selection"] = 0.2
@@ -90,7 +84,7 @@ class KGScenario(Scenario):
.from_string(prompt_dict["kg_description_template"]["user"])
.render(
competition_descriptions=self.competition_descriptions,
raw_data_information=self._source_data,
raw_data_information=self.source_data,
evaluation_metric_direction=self.evaluation_metric_direction,
)
)
@@ -154,71 +148,82 @@ class KGScenario(Scenario):
def source_data(self) -> str:
data_folder = Path(KAGGLE_IMPLEMENT_SETTING.local_data_path) / self.competition
if (data_folder / "X_valid.pkl").exists():
X_valid = pd.read_pickle(data_folder / "X_valid.pkl")
# TODO: Hardcoded for now, need to be fixed
if self.competition == "feedback-prize-english-language-learning":
self.input_shape = X_valid.shape
return "This is a sparse matrix of descriptive text."
buffer = io.StringIO()
X_valid.info(verbose=True, buf=buffer, show_counts=True)
data_info = buffer.getvalue()
self.input_shape = X_valid.shape
return data_info
if not (data_folder / "X_valid.pkl").exists():
preprocess_experiment = KGFactorExperiment([])
(
X_train,
X_valid,
y_train,
y_valid,
X_test,
*others,
) = preprocess_experiment.experiment_workspace.generate_preprocess_data()
preprocess_experiment = KGFactorExperiment([])
(
X_train,
X_valid,
y_train,
y_valid,
X_test,
*others,
) = preprocess_experiment.experiment_workspace.generate_preprocess_data()
data_folder.mkdir(exist_ok=True, parents=True)
pickle.dump(X_train, open(data_folder / "X_train.pkl", "wb"))
pickle.dump(X_valid, open(data_folder / "X_valid.pkl", "wb"))
pickle.dump(y_train, open(data_folder / "y_train.pkl", "wb"))
pickle.dump(y_valid, open(data_folder / "y_valid.pkl", "wb"))
pickle.dump(X_test, open(data_folder / "X_test.pkl", "wb"))
pickle.dump(others, open(data_folder / "others.pkl", "wb"))
data_folder.mkdir(exist_ok=True, parents=True)
pickle.dump(X_train, open(data_folder / "X_train.pkl", "wb"))
pickle.dump(X_valid, open(data_folder / "X_valid.pkl", "wb"))
pickle.dump(y_train, open(data_folder / "y_train.pkl", "wb"))
pickle.dump(y_valid, open(data_folder / "y_valid.pkl", "wb"))
pickle.dump(X_test, open(data_folder / "X_test.pkl", "wb"))
pickle.dump(others, open(data_folder / "others.pkl", "wb"))
self.input_shape = X_valid.shape
X_valid = pd.read_pickle(data_folder / "X_valid.pkl")
# TODO: Hardcoded for now, need to be fixed
if self.competition == "feedback-prize-english-language-learning":
return "This is a sparse matrix of descriptive text."
buffer = io.StringIO()
X_valid.info(verbose=True, buf=buffer, show_counts=True)
X_valid.info(verbose=True, buf=buffer, show_counts=False)
data_info = buffer.getvalue()
self.input_shape = X_valid.shape
return data_info
@property
def output_format(self) -> str:
return (
def output_format(self, tag=None) -> str:
assert tag in [None, "feature", "model"]
feature_output_format = f"""The feature code should output following the format:
{prompt_dict['kg_feature_output_format']}"""
model_output_format = f"""The model code should output following the format:\n""" + (
Environment(undefined=StrictUndefined)
.from_string(prompt_dict["kg_model_output_format"])
.render(channel=self.model_output_channel)
)
if tag is None:
return feature_output_format + "\n" + model_output_format
elif tag == "feature":
return feature_output_format
elif tag == "model":
return model_output_format
@property
def interface(self) -> str:
return f"""The feature code should follow the interface:
{prompt_dict['kg_feature_interface']}
The model code should follow the interface:
{prompt_dict['kg_model_interface']}
"""
def interface(self, tag=None) -> str:
assert tag in [None, "feature", "XGBoost", "RandomForest", "LightGBM", "NN"]
feature_interface = f"""The feature code should follow the interface:
{prompt_dict['kg_feature_interface']}"""
if tag == "feature":
return feature_interface
@property
def simulator(self) -> str:
kg_model_simulator = (
model_interface = "The model code should follow the interface:\n" + (
Environment(undefined=StrictUndefined).from_string(prompt_dict["kg_model_interface"]).render(tag=tag)
)
if tag is None:
return feature_interface + "\n" + model_interface
else:
return model_interface
def simulator(self, tag=None) -> str:
assert tag in [None, "feature", "model"]
kg_feature_simulator = "The feature code will be sent to the simulator:\n" + prompt_dict["kg_feature_simulator"]
kg_model_simulator = "The model code will be sent to the simulator:\n" + (
Environment(undefined=StrictUndefined)
.from_string(prompt_dict["kg_model_simulator"])
.render(submission_specifications=self.submission_specifications)
)
return f"""The feature code should follow the simulator:
{prompt_dict['kg_feature_simulator']}
The model code should follow the simulator:
{kg_model_simulator}
"""
if tag is None:
return kg_feature_simulator + "\n" + kg_model_simulator
elif tag == "feature":
return kg_feature_simulator
elif tag == "model":
return kg_model_simulator
@property
def rich_style_description(self) -> str:
@@ -248,17 +253,42 @@ Current Competition: [{self.competition}](https://www.kaggle.com/competitions/{s
To automatically optimize performance metrics within the validation set or Kaggle Leaderboard, ultimately discovering the most efficient features and models through autonomous research and development.
"""
def get_scenario_all_desc(self, task: Task | None = None) -> str:
return f"""Background of the scenario:
{self._background}
The source dataset you can use to generate the features:
{self._source_data}
The interface you should follow to write the runnable code:
{self._interface}
The output of your code should be in the format:
{self._output_format}
The simulator user can use to test your model:
{self._simulator}
The expected output & submission format specifications:
{self.submission_specifications} # Added again to emphasize the importance
def get_scenario_all_desc(
self, task: Task | None = None, filtered_tag: str | None = None, simple_background: bool | None = None
) -> str:
def common_description() -> str:
return f"""\n------Background of the scenario------
{self.background}
------The source dataset you can use to generate the features------
{self.source_data}
------The expected output & submission format specifications------
{self.submission_specifications}
"""
def interface(tag: str | None) -> str:
return f"""
------The interface you should follow to write the runnable code------
{self.interface(tag)}
"""
def output(tag: str | None) -> str:
return f"""
------The output of your code should be in the format------
{self.output_format(tag)}
"""
def simulator(tag: str | None) -> str:
return f"""
------The simulator user can use to test your solution------
{self.simulator(tag)}
"""
assert filtered_tag is not None, "filtered_tag should not be None in Kaggle scenario"
if filtered_tag == "hypothesis_and_experiment" or filtered_tag == "feedback":
return common_description() + simulator(None)
elif filtered_tag == "feature":
return common_description() + interface("feature") + output("feature") + simulator("feature")
else:
return common_description() + interface(filtered_tag) + output("model") + simulator("model")
rdagent/scenarios/kaggle/experiment/sf-crime_template/train.py
+3 -3
View File
@@ -40,9 +40,9 @@ X_test_l = []
for f in DIRNAME.glob("feature/feat*.py"):
cls = import_module_from_path(f.stem, f).feature_engineering_cls()
cls.fit(X_train)
X_train_f = cls.transform(X_train)
X_valid_f = cls.transform(X_valid)
X_test_f = cls.transform(X_test)
X_train_f = cls.transform(X_train.copy())
X_valid_f = cls.transform(X_valid.copy())
X_test_f = cls.transform(X_test.copy())
if X_train_f.shape[-1] == X_valid_f.shape[-1] and X_train_f.shape[-1] == X_test_f.shape[-1]:
X_train_l.append(X_train_f)
rdagent/scenarios/kaggle/experiment/spaceship-titanic_template/train.py
+3 -3
View File
@@ -38,9 +38,9 @@ X_test_l = []
for f in DIRNAME.glob("feature/feat*.py"):
cls = import_module_from_path(f.stem, f).feature_engineering_cls()
cls.fit(X_train)
X_train_f = cls.transform(X_train)
X_valid_f = cls.transform(X_valid)
X_test_f = cls.transform(X_test)
X_train_f = cls.transform(X_train.copy())
X_valid_f = cls.transform(X_valid.copy())
X_test_f = cls.transform(X_test.copy())
if X_train_f.shape[-1] == X_valid_f.shape[-1] and X_train_f.shape[-1] == X_test_f.shape[-1]:
X_train_l.append(X_train_f)
rdagent/scenarios/kaggle/experiment/statoil-iceberg-classifier-challenge_template/train.py
+3 -3
View File
@@ -38,9 +38,9 @@ X_test_l = []
for f in DIRNAME.glob("feature/feat*.py"):
cls = import_module_from_path(f.stem, f).feature_engineering_cls()
cls.fit(X_train)
X_train_f = cls.transform(X_train)
X_valid_f = cls.transform(X_valid)
X_test_f = cls.transform(X_test)
X_train_f = cls.transform(X_train.copy())
X_valid_f = cls.transform(X_valid.copy())
X_test_f = cls.transform(X_test.copy())
if X_train_f.shape[-1] == X_valid_f.shape[-1] and X_train_f.shape[-1] == X_test_f.shape[-1]:
X_train_l.append(X_train_f)
rdagent/scenarios/kaggle/prompts.yaml
+1 -1
View File
@@ -25,7 +25,7 @@ KG_hypothesis_gen_RAG: |-
{% endif %}
hypothesis_and_feedback: |-
{% for hypothesis, experiment, feedback in trace.hist %}
{% for hypothesis, experiment, feedback in trace.hist[-10:] %}
Hypothesis {{ loop.index }}: {{ hypothesis }}
Observation on the result with the hypothesis: {{ feedback.observations }}
Feedback on the original hypothesis: {{ feedback.hypothesis_evaluation }}
rdagent/scenarios/kaggle/proposal/proposal.py
+144 -95
View File
@@ -9,10 +9,9 @@ from rdagent.app.kaggle.conf import KAGGLE_IMPLEMENT_SETTING
from rdagent.components.coder.factor_coder.factor import FactorTask
from rdagent.components.coder.model_coder.model import ModelExperiment, ModelTask
from rdagent.components.knowledge_management.vector_base import VectorBase
from rdagent.components.proposal.model_proposal import (
ModelHypothesis,
ModelHypothesis2Experiment,
ModelHypothesisGen,
from rdagent.components.proposal import (
FactorAndModelHypothesis2Experiment,
FactorAndModelHypothesisGen,
)
from rdagent.core.exception import ModelEmptyError
from rdagent.core.prompts import Prompts
@@ -67,7 +66,135 @@ Concise Knowledge: {self.concise_knowledge}
"""
class KGHypothesisGen(ModelHypothesisGen):
def generate_RAG_content(
scen: KGScenario,
trace: Trace,
hypothesis_and_feedback: str,
target: str = None,
chosen_hypothesis: str = None,
chosen_hypothesis_type: str = None,
) -> str:
if scen.if_using_vector_rag:
if scen.mini_case:
rag_results, _ = scen.vector_base.search_experience(target, hypothesis_and_feedback, topk_k=1)
else:
rag_results, _ = scen.vector_base.search_experience(target, hypothesis_and_feedback, topk_k=5)
return "\n".join([doc.content for doc in rag_results])
if scen.if_using_graph_rag is False or trace.knowledge_base is None:
return None
same_competition_node = trace.knowledge_base.get_node_by_content(trace.scen.get_competition_full_desc())
if same_competition_node is not None:
related_hypothesis_nodes = []
for action in KG_ACTION_LIST:
related_hypothesis_nodes.extend(
trace.knowledge_base.get_nodes_within_steps(
start_node=same_competition_node,
steps=1,
constraint_labels=[action],
)[:1]
)
else:
related_hypothesis_nodes = []
experiences = []
for hypothesis_node in related_hypothesis_nodes:
experience = {"hypothesis": hypothesis_node.content}
experiment_node_list = trace.knowledge_base.get_nodes_within_steps(
start_node=hypothesis_node, steps=1, constraint_labels=["experiments"]
)
if len(experiment_node_list) > 0:
experience["experiments"] = experiment_node_list[0].content
else:
experience["experiments"] = "No experiment information available."
conclusion_node_list = trace.knowledge_base.get_nodes_within_steps(
start_node=hypothesis_node, steps=1, constraint_labels=["conclusion"]
)
if len(conclusion_node_list) > 0:
experience["conclusion"] = conclusion_node_list[0].content
else:
experience["conclusion"] = "No conclusion information available."
experiences.append(experience)
found_nodes = []
insights = []
if chosen_hypothesis is not None:
similar_nodes = trace.knowledge_base.semantic_search(
node=chosen_hypothesis,
topk_k=2,
)
for similar_node in similar_nodes:
hypothesis_nodes = trace.knowledge_base.get_nodes_within_steps(
start_node=similar_node,
steps=3,
constraint_labels=[chosen_hypothesis_type],
)
found_nodes.extend(hypothesis_nodes[:5])
found_nodes = sorted(list(set(found_nodes)), key=lambda x: len(x.content))
for exp_node in found_nodes[:5]:
insight = {"experiments": exp_node.content}
hypothesis_node_list = trace.knowledge_base.get_nodes_within_steps(
start_node=exp_node, steps=2, constraint_labels=KG_ACTION_LIST
)
if len(hypothesis_node_list) > 0:
insight["hypothesis"] = hypothesis_node_list[0].content
else:
insight["hypothesis"] = "No hypothesis information available."
conclusion_node_list = trace.knowledge_base.get_nodes_within_steps(
start_node=exp_node, steps=2, constraint_labels=["conclusion"]
)
if len(conclusion_node_list) > 0:
insight["conclusion"] = conclusion_node_list[0].content
else:
insight["conclusion"] = "No conclusion information available."
insights.append(insight)
else:
similar_nodes = trace.knowledge_base.semantic_search(
node=trace.scen.get_competition_full_desc(),
topk_k=2,
)
for similar_node in similar_nodes:
for hypothesis_type in KG_ACTION_LIST:
hypothesis_nodes = trace.knowledge_base.get_nodes_within_steps(
start_node=similar_node,
steps=3,
constraint_labels=[hypothesis_type],
)
found_nodes.extend(hypothesis_nodes[:2])
found_nodes = sorted(list(set(found_nodes)), key=lambda x: len(x.content))
for hypothesis_node in found_nodes[:5]:
if hypothesis_node in related_hypothesis_nodes:
continue
insight = {"hypothesis": hypothesis_node.content}
experiment_node_list = trace.knowledge_base.get_nodes_within_steps(
start_node=hypothesis_node, steps=2, constraint_labels=["experiments"]
)
if len(experiment_node_list) > 0:
insight["experiments"] = experiment_node_list[0].content
else:
insight["experiments"] = "No experiment information available."
conclusion_node_list = trace.knowledge_base.get_nodes_within_steps(
start_node=hypothesis_node, steps=2, constraint_labels=["conclusion"]
)
if len(conclusion_node_list) > 0:
insight["conclusion"] = conclusion_node_list[0].content
else:
insight["conclusion"] = "No conclusion information available."
insights.append(insight)
RAG_content = (
Environment(undefined=StrictUndefined)
.from_string(prompt_dict["KG_hypothesis_gen_RAG"])
.render(insights=insights, experiences=experiences)
)
return RAG_content
class KGHypothesisGen(FactorAndModelHypothesisGen):
"""
# NOTE: we can share this class across different data mining scenarios
# It may better to move the class into components folder like `rdagent/components/proposal/model_proposal.py`
@@ -82,91 +209,6 @@ class KGHypothesisGen(ModelHypothesisGen):
def __init__(self, scen: Scenario) -> Tuple[dict, bool]:
super().__init__(scen)
def generate_RAG_content(self, trace: Trace, hypothesis_and_feedback: str, target: str = None) -> str:
if self.scen.if_using_vector_rag:
if self.scen.mini_case:
rag_results, _ = self.scen.vector_base.search_experience(target, hypothesis_and_feedback, topk_k=1)
else:
rag_results, _ = self.scen.vector_base.search_experience(target, hypothesis_and_feedback, topk_k=5)
return "\n".join([doc.content for doc in rag_results])
if self.scen.if_using_graph_rag is False or trace.knowledge_base is None:
return None
same_competition_node = trace.knowledge_base.get_node_by_content(trace.scen.get_competition_full_desc())
if same_competition_node is not None:
related_hypothesis_nodes = []
for action in KG_ACTION_LIST:
related_hypothesis_nodes.extend(
trace.knowledge_base.get_nodes_within_steps(
start_node=same_competition_node,
steps=1,
constraint_labels=[action],
)[:1]
)
else:
related_hypothesis_nodes = []
experiences = []
for hypothesis_node in related_hypothesis_nodes:
experience = {"hypothesis": hypothesis_node.content}
experiment_node_list = trace.knowledge_base.get_nodes_within_steps(
start_node=hypothesis_node, steps=1, constraint_labels=["experiments"]
)
if len(experiment_node_list) > 0:
experience["experiments"] = experiment_node_list[0].content
else:
experience["experiments"] = "No experiment information available."
conclusion_node_list = trace.knowledge_base.get_nodes_within_steps(
start_node=hypothesis_node, steps=1, constraint_labels=["conclusion"]
)
if len(conclusion_node_list) > 0:
experience["conclusion"] = conclusion_node_list[0].content
else:
experience["conclusion"] = "No conclusion information available."
experiences.append(experience)
similar_nodes = trace.knowledge_base.semantic_search(
node=trace.scen.get_competition_full_desc(),
topk_k=2,
)
found_hypothesis_nodes = []
for similar_node in similar_nodes:
hypothesis_nodes = trace.knowledge_base.get_nodes_within_steps(
start_node=similar_node,
steps=3,
constraint_labels=[target],
)
found_hypothesis_nodes.extend(hypothesis_nodes[:2])
found_hypothesis_nodes = sorted(list(set(found_hypothesis_nodes)), key=lambda x: len(x.content))
insights = []
for hypothesis_node in found_hypothesis_nodes[:5]:
if hypothesis_node in related_hypothesis_nodes:
continue
insight = {"hypothesis": hypothesis_node.content}
experiment_node_list = trace.knowledge_base.get_nodes_within_steps(
start_node=hypothesis_node, steps=1, constraint_labels=["experiments"]
)
if len(experiment_node_list) > 0:
insight["experiments"] = experiment_node_list[0].content
else:
insight["experiments"] = "No experiment information available."
conclusion_node_list = trace.knowledge_base.get_nodes_within_steps(
start_node=hypothesis_node, steps=1, constraint_labels=["conclusion"]
)
if len(conclusion_node_list) > 0:
insight["conclusion"] = conclusion_node_list[0].content
else:
insight["conclusion"] = "No conclusion information available."
insights.append(insight)
RAG_content = (
Environment(undefined=StrictUndefined)
.from_string(prompt_dict["KG_hypothesis_gen_RAG"])
.render(insights=insights, experiences=experiences)
)
return RAG_content
def update_reward_estimates(self, trace: Trace) -> None:
if len(trace.hist) > 0:
last_entry = trace.hist[-1]
@@ -233,7 +275,8 @@ class KGHypothesisGen(ModelHypothesisGen):
context_dict = {
"hypothesis_and_feedback": hypothesis_and_feedback,
"RAG": self.generate_RAG_content(
"RAG": generate_RAG_content(
scen=self.scen,
trace=trace,
hypothesis_and_feedback=hypothesis_and_feedback,
target=action if self.scen.if_action_choosing_based_on_UCB else None,
@@ -250,7 +293,7 @@ class KGHypothesisGen(ModelHypothesisGen):
}
return context_dict, True
def convert_response(self, response: str) -> ModelHypothesis:
def convert_response(self, response: str) -> Hypothesis:
response_dict = json.loads(response)
hypothesis = KGHypothesis(
@@ -266,9 +309,9 @@ class KGHypothesisGen(ModelHypothesisGen):
return hypothesis
class KGHypothesis2Experiment(ModelHypothesis2Experiment):
class KGHypothesis2Experiment(FactorAndModelHypothesis2Experiment):
def prepare_context(self, hypothesis: Hypothesis, trace: Trace) -> Tuple[dict, bool]:
scenario = trace.scen.get_scenario_all_desc()
scenario = trace.scen.get_scenario_all_desc(filtered_tag="hypothesis_and_experiment")
assert isinstance(hypothesis, KGHypothesis)
experiment_output_format = (
prompt_dict["feature_experiment_output_format"]
@@ -300,7 +343,13 @@ class KGHypothesis2Experiment(ModelHypothesis2Experiment):
"hypothesis_and_feedback": hypothesis_and_feedback,
"experiment_output_format": experiment_output_format,
"target_list": model_list,
"RAG": ...,
"RAG": generate_RAG_content(
trace.scen,
trace,
hypothesis_and_feedback,
chosen_hypothesis=hypothesis.hypothesis,
chosen_hypothesis_type=hypothesis.action,
),
}, True
def convert_feature_experiment(self, response: str, trace: Trace) -> KGFactorExperiment:
rdagent/scenarios/qlib/experiment/factor_experiment.py
+6 -1
View File
@@ -60,8 +60,13 @@ class QlibFactorScenario(Scenario):
def experiment_setting(self) -> str:
return self._experiment_setting
def get_scenario_all_desc(self, task: Task | None = None) -> str:
def get_scenario_all_desc(
self, task: Task | None = None, filtered_tag: str | None = None, simple_background: bool | None = None
) -> str:
"""A static scenario describer"""
if simple_background:
return f"""Background of the scenario:
{self.background}"""
return f"""Background of the scenario:
{self.background}
The source data you can use:
rdagent/scenarios/qlib/experiment/model_experiment.py
+3 -1
View File
@@ -58,7 +58,9 @@ class QlibModelScenario(Scenario):
def experiment_setting(self) -> str:
return self._experiment_setting
def get_scenario_all_desc(self, task: Task | None = None) -> str:
def get_scenario_all_desc(
self, task: Task | None = None, filtered_tag: str | None = None, simple_background: bool | None = None
) -> str:
return f"""Background of the scenario:
{self.background}
The interface you should follow to write the runnable code:
rdagent/scenarios/qlib/experiment/utils.py
+20 -10
View File
@@ -51,7 +51,7 @@ def generate_data_folder_from_qlib():
)
def get_file_desc(p: Path) -> str:
def get_file_desc(p: Path, variable_list=[]) -> str:
"""
Get the description of a file based on its type.
@@ -83,22 +83,29 @@ def get_file_desc(p: Path) -> str:
pd.set_option("display.max_rows", None) # or 1000
pd.set_option("display.max_colwidth", None) # or 199
buffer = io.StringIO()
df.info(verbose=True, buf=buffer, show_counts=False)
df_info = buffer.getvalue()
if isinstance(df.index, pd.MultiIndex):
df_info += f"\nMultiIndex names:, {df.index.names})"
df_info = f"MultiIndex names:, {df.index.names})\n"
else:
df_info = f"Index name: {df.index.name}\n"
columns = df.dtypes.to_dict()
filtered_columns = [f"{i, j}" for i, j in columns.items() if i in variable_list]
if filtered_columns:
df_info += "Related Data columns: \n"
df_info += ",".join(filtered_columns)
else:
df_info += "Data columns: \n"
df_info += ",".join(columns)
df_info += "\n"
if "REPORT_PERIOD" in df.columns:
one_instrument = df.index.get_level_values("instrument")[0]
df_on_one_instrument = df.loc[pd.IndexSlice[:, one_instrument], ["REPORT_PERIOD"]]
df_info += f"""
A snapshot of one instrument, from which you can tell the distribution of the data:
{df_on_one_instrument.head(10)}
{df_on_one_instrument.head(5)}
"""
return JJ_TPL.render(
file_name=p.name,
type_desc="h5, generated by `df.info(verbose=True, show_counts=False)` and appendix info",
type_desc="h5 info",
content=df_info,
)
elif p.name.endswith(".md"):
@@ -115,7 +122,7 @@ A snapshot of one instrument, from which you can tell the distribution of the da
)
def get_data_folder_intro(fname_reg: str = ".*", flags=0) -> str:
def get_data_folder_intro(fname_reg: str = ".*", flags=0, variable_mapping=None) -> str:
"""
Directly get the info of the data folder.
It is for preparing prompting message.
@@ -144,5 +151,8 @@ def get_data_folder_intro(fname_reg: str = ".*", flags=0) -> str:
content_l = []
for p in Path(FACTOR_IMPLEMENT_SETTINGS.data_folder_debug).iterdir():
if re.match(fname_reg, p.name, flags) is not None:
content_l.append(get_file_desc(p))
if variable_mapping:
content_l.append(get_file_desc(p, variable_mapping.get(p.stem, [])))
else:
content_l.append(get_file_desc(p))
return "\n----------------- file splitter -------------\n".join(content_l)
rdagent/scenarios/qlib/prompts.yaml
+1 -1
View File
@@ -1,5 +1,5 @@
hypothesis_and_feedback: |-
{% for hypothesis, experiment, feedback in trace.hist %}
{% for hypothesis, experiment, feedback in trace.hist[-10:] %}
Hypothesis {{ loop.index }}: {{ hypothesis }}
Corresponding Code (that leads to the difference in performance): {{experiment.sub_workspace_list[0].code_dict.get("model.py")}}
Observation on the result with the hypothesis: {{ feedback.observations }}
rdagent/scenarios/qlib/proposal/factor_proposal.py
+3 -7
View File
@@ -5,18 +5,14 @@ from typing import List, Tuple
from jinja2 import Environment, StrictUndefined
from rdagent.components.coder.factor_coder.factor import FactorExperiment, FactorTask
from rdagent.components.proposal.factor_proposal import (
FactorHypothesis,
FactorHypothesis2Experiment,
FactorHypothesisGen,
)
from rdagent.components.proposal import FactorHypothesis2Experiment, FactorHypothesisGen
from rdagent.core.prompts import Prompts
from rdagent.core.proposal import Hypothesis, Scenario, Trace
from rdagent.scenarios.qlib.experiment.factor_experiment import QlibFactorExperiment
prompt_dict = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")
QlibFactorHypothesis = FactorHypothesis
QlibFactorHypothesis = Hypothesis
class QlibFactorHypothesisGen(FactorHypothesisGen):
@@ -41,7 +37,7 @@ class QlibFactorHypothesisGen(FactorHypothesisGen):
}
return context_dict, True
def convert_response(self, response: str) -> FactorHypothesis:
def convert_response(self, response: str) -> Hypothesis:
response_dict = json.loads(response)
hypothesis = QlibFactorHypothesis(
hypothesis=response_dict["hypothesis"],
rdagent/scenarios/qlib/proposal/model_proposal.py
+3 -7
View File
@@ -5,18 +5,14 @@ from typing import List, Tuple
from jinja2 import Environment, StrictUndefined
from rdagent.components.coder.model_coder.model import ModelExperiment, ModelTask
from rdagent.components.proposal.model_proposal import (
ModelHypothesis,
ModelHypothesis2Experiment,
ModelHypothesisGen,
)
from rdagent.components.proposal import ModelHypothesis2Experiment, ModelHypothesisGen
from rdagent.core.prompts import Prompts
from rdagent.core.proposal import Hypothesis, Scenario, Trace
from rdagent.scenarios.qlib.experiment.model_experiment import QlibModelExperiment
prompt_dict = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")
QlibModelHypothesis = ModelHypothesis
QlibModelHypothesis = Hypothesis
class QlibModelHypothesisGen(ModelHypothesisGen):
@@ -41,7 +37,7 @@ class QlibModelHypothesisGen(ModelHypothesisGen):
}
return context_dict, True
def convert_response(self, response: str) -> ModelHypothesis:
def convert_response(self, response: str) -> Hypothesis:
response_dict = json.loads(response)
hypothesis = QlibModelHypothesis(
hypothesis=response_dict["hypothesis"],
rdagent/utils/env.py
+20 -1
View File
@@ -15,6 +15,7 @@ import sys
import uuid
import zipfile
from abc import abstractmethod
from concurrent.futures import ThreadPoolExecutor, TimeoutError
from pathlib import Path
from typing import Dict, Generic, Optional, TypeVar
@@ -138,6 +139,8 @@ class DockerConf(BaseSettings):
enable_gpu: bool = True # because we will automatically disable GPU if not available. So we enable it by default.
mem_limit: str | None = "48g" # Add memory limit attribute
running_timeout_period: int = 3600 # 1 hour
class QlibDockerConf(DockerConf):
class Config:
@@ -186,6 +189,8 @@ class KGDockerConf(DockerConf):
# Path("git_ignore_folder/data").resolve(): "/root/.data/"
# }
running_timeout_period: int = 600
# physionet.org/files/mimic-eicu-fiddle-feature/1.0.0/FIDDLE_mimic3
class DockerEnv(Env[DockerConf]):
@@ -269,7 +274,7 @@ class DockerEnv(Env[DockerConf]):
return {}
return gpu_kwargs
def run(
def __run(
self,
entry: str | None = None,
local_path: str | None = None,
@@ -334,6 +339,20 @@ class DockerEnv(Env[DockerConf]):
except docker.errors.APIError as e:
raise RuntimeError(f"Error while running the container: {e}")
def run(
self,
entry: str | None = None,
local_path: str | None = None,
env: dict | None = None,
running_extra_volume: dict | None = None,
):
with ThreadPoolExecutor() as executor:
future = executor.submit(self.__run, entry, local_path, env, running_extra_volume)
try:
return future.result(timeout=self.conf.running_timeout_period)
except TimeoutError:
raise TimeoutError(f"Timeout while running the container: {self.conf.running_timeout_period} seconds")
def dump_python_code_run_and_get_results(
self,
code: str,