mirror of
https://github.com/NicolasBohn/NexQuant.git
synced 2026-07-28 16:07:46 +00:00
feat: add pipeline coder (#742)
* init commit * limit problem numbers * ensemble lower case * add runtime and spec to coder * submission check notice * sub EDA in sample execution * avoid lightgbm * add time limit to scenario * rephrase the submission check * give positive feedback when facing warning in check * ENABLE FEEDBACK * fix feedback bug --------- Co-authored-by: Xu Yang <peteryang@vip.qq.com>
This commit is contained in:
Roland Minrui
@@ -0,0 +1,163 @@
|
||||
"""
|
||||
|
||||
Loop should not large change exclude
|
||||
- Action Choice[current data loader & spec]
|
||||
- other should share
|
||||
- Propose[choice] => Task[Choice] => CoSTEER =>
|
||||
-
|
||||
|
||||
Extra feature:
|
||||
- cache
|
||||
|
||||
|
||||
File structure
|
||||
- ___init__.py: the entrance/agent of coder
|
||||
- evaluator.py
|
||||
- conf.py
|
||||
- exp.py: everything under the experiment, e.g.
|
||||
- Task
|
||||
- Experiment
|
||||
- Workspace
|
||||
- test.py
|
||||
- Each coder could be tested.
|
||||
"""
|
||||
|
||||
import json
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Dict
|
||||
|
||||
from rdagent.app.data_science.conf import DS_RD_SETTING
|
||||
from rdagent.components.coder.CoSTEER import CoSTEER
|
||||
from rdagent.components.coder.CoSTEER.evaluators import (
|
||||
CoSTEERMultiEvaluator,
|
||||
CoSTEERSingleFeedback,
|
||||
)
|
||||
from rdagent.components.coder.CoSTEER.evolving_strategy import (
|
||||
MultiProcessEvolvingStrategy,
|
||||
)
|
||||
from rdagent.components.coder.CoSTEER.knowledge_management import (
|
||||
CoSTEERQueriedKnowledge,
|
||||
)
|
||||
from rdagent.components.coder.data_science.conf import (
|
||||
DSCoderCoSTEERSettings,
|
||||
get_ds_env,
|
||||
)
|
||||
from rdagent.components.coder.data_science.pipeline.eval import PipelineCoSTEEREvaluator
|
||||
from rdagent.components.coder.data_science.raw_data_loader.eval import (
|
||||
DataLoaderCoSTEEREvaluator,
|
||||
)
|
||||
from rdagent.components.coder.data_science.raw_data_loader.exp import DataLoaderTask
|
||||
from rdagent.core.exception import CoderError
|
||||
from rdagent.core.experiment import FBWorkspace
|
||||
from rdagent.core.scenario import Scenario
|
||||
from rdagent.oai.llm_utils import APIBackend
|
||||
from rdagent.utils.agent.ret import PythonAgentOut
|
||||
from rdagent.utils.agent.tpl import T
|
||||
|
||||
DIRNAME = Path(__file__).absolute().resolve().parent
|
||||
|
||||
|
||||
class PipelineMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
|
||||
def implement_one_task(
|
||||
self,
|
||||
target_task: DataLoaderTask,
|
||||
queried_knowledge: CoSTEERQueriedKnowledge | None = None,
|
||||
workspace: FBWorkspace | None = None,
|
||||
prev_task_feedback: CoSTEERSingleFeedback | None = None,
|
||||
) -> dict[str, str]:
|
||||
competition_info = self.scen.get_scenario_all_desc()
|
||||
runtime_environment = self.scen.get_runtime_environment()
|
||||
data_folder_info = self.scen.processed_data_folder_description
|
||||
pipeline_task_info = target_task.get_task_information()
|
||||
|
||||
queried_similar_successful_knowledge = (
|
||||
queried_knowledge.task_to_similar_task_successful_knowledge[pipeline_task_info]
|
||||
if queried_knowledge is not None
|
||||
else []
|
||||
)
|
||||
queried_former_failed_knowledge = (
|
||||
queried_knowledge.task_to_former_failed_traces[pipeline_task_info] if queried_knowledge is not None else []
|
||||
)
|
||||
queried_former_failed_knowledge = (
|
||||
[
|
||||
knowledge
|
||||
for knowledge in queried_former_failed_knowledge[0]
|
||||
if knowledge.implementation.file_dict.get("main.py") != workspace.file_dict.get("main.py")
|
||||
],
|
||||
queried_former_failed_knowledge[1],
|
||||
)
|
||||
|
||||
system_prompt = T(".prompts:pipeline_coder.system").r(
|
||||
task_desc=pipeline_task_info,
|
||||
queried_similar_successful_knowledge=queried_similar_successful_knowledge,
|
||||
queried_former_failed_knowledge=queried_former_failed_knowledge[0],
|
||||
out_spec=PythonAgentOut.get_spec(),
|
||||
runtime_environment=runtime_environment,
|
||||
spec=T("scenarios.data_science.share:component_spec.Pipeline").r(),
|
||||
)
|
||||
user_prompt = T(".prompts:pipeline_coder.user").r(
|
||||
competition_info=competition_info,
|
||||
folder_spec=data_folder_info,
|
||||
latest_code=workspace.file_dict.get("main.py"),
|
||||
latest_code_feedback=prev_task_feedback,
|
||||
)
|
||||
|
||||
for _ in range(5):
|
||||
pipeline_code = PythonAgentOut.extract_output(
|
||||
APIBackend().build_messages_and_create_chat_completion(
|
||||
user_prompt=user_prompt,
|
||||
system_prompt=system_prompt,
|
||||
)
|
||||
)
|
||||
if pipeline_code != workspace.file_dict.get("main.py"):
|
||||
break
|
||||
else:
|
||||
user_prompt = user_prompt + "\nPlease avoid generating same code to former code!"
|
||||
else:
|
||||
raise CoderError("Failed to generate a new pipeline code.")
|
||||
|
||||
return {
|
||||
"main.py": pipeline_code,
|
||||
}
|
||||
|
||||
def assign_code_list_to_evo(self, code_list: list[dict[str, str]], evo):
|
||||
"""
|
||||
Assign the code list to the evolving item.
|
||||
|
||||
The code list is aligned with the evolving item's sub-tasks.
|
||||
If a task is not implemented, put a None in the list.
|
||||
"""
|
||||
for index in range(len(evo.sub_tasks)):
|
||||
if code_list[index] is None:
|
||||
continue
|
||||
if evo.sub_workspace_list[index] is None:
|
||||
# evo.sub_workspace_list[index] = FBWorkspace(target_task=evo.sub_tasks[index])
|
||||
evo.sub_workspace_list[index] = evo.experiment_workspace
|
||||
evo.sub_workspace_list[index].inject_files(**code_list[index])
|
||||
return evo
|
||||
|
||||
|
||||
class PipelineCoSTEER(CoSTEER):
|
||||
def __init__(
|
||||
self,
|
||||
scen: Scenario,
|
||||
*args,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
settings = DSCoderCoSTEERSettings()
|
||||
eva = CoSTEERMultiEvaluator(
|
||||
PipelineCoSTEEREvaluator(scen=scen), scen=scen
|
||||
) # Please specify whether you agree running your eva in parallel or not
|
||||
es = PipelineMultiProcessEvolvingStrategy(scen=scen, settings=settings)
|
||||
|
||||
super().__init__(
|
||||
*args,
|
||||
settings=settings,
|
||||
eva=eva,
|
||||
es=es,
|
||||
evolving_version=2,
|
||||
scen=scen,
|
||||
max_loop=DS_RD_SETTING.coder_max_loop,
|
||||
**kwargs,
|
||||
)
|
||||
@@ -0,0 +1,119 @@
|
||||
# tess successfully running.
|
||||
# (GPT) if it aligns with the spec & rationality of the spec.
|
||||
import json
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
import pandas as pd
|
||||
|
||||
from rdagent.app.data_science.conf import DS_RD_SETTING
|
||||
from rdagent.components.coder.CoSTEER import CoSTEERMultiFeedback
|
||||
from rdagent.components.coder.CoSTEER.evaluators import (
|
||||
CoSTEEREvaluator,
|
||||
CoSTEERSingleFeedback,
|
||||
)
|
||||
from rdagent.components.coder.CoSTEER.knowledge_management import (
|
||||
CoSTEERQueriedKnowledgeV2,
|
||||
)
|
||||
from rdagent.components.coder.data_science.conf import get_ds_env
|
||||
from rdagent.core.experiment import FBWorkspace, Task
|
||||
from rdagent.utils.agent.tpl import T
|
||||
from rdagent.utils.agent.workflow import build_cls_from_json_with_retry
|
||||
|
||||
DIRNAME = Path(__file__).absolute().resolve().parent
|
||||
|
||||
PipelineSingleFeedback = CoSTEERSingleFeedback
|
||||
PipelineMultiFeedback = CoSTEERMultiFeedback
|
||||
|
||||
|
||||
class PipelineCoSTEEREvaluator(CoSTEEREvaluator):
|
||||
|
||||
def evaluate(
|
||||
self,
|
||||
target_task: Task,
|
||||
implementation: FBWorkspace,
|
||||
gt_implementation: FBWorkspace,
|
||||
queried_knowledge: CoSTEERQueriedKnowledgeV2 = None,
|
||||
**kwargs,
|
||||
) -> PipelineSingleFeedback:
|
||||
|
||||
target_task_information = target_task.get_task_information()
|
||||
if (
|
||||
queried_knowledge is not None
|
||||
and target_task_information in queried_knowledge.success_task_to_knowledge_dict
|
||||
):
|
||||
return queried_knowledge.success_task_to_knowledge_dict[target_task_information].feedback
|
||||
elif queried_knowledge is not None and target_task_information in queried_knowledge.failed_task_info_set:
|
||||
return PipelineSingleFeedback(
|
||||
execution="This task has failed too many times, skip implementation.",
|
||||
return_checking="This task has failed too many times, skip implementation.",
|
||||
code="This task has failed too many times, skip implementation.",
|
||||
final_decision=False,
|
||||
)
|
||||
|
||||
env = get_ds_env()
|
||||
env.conf.extra_volumes = {f"{DS_RD_SETTING.local_data_path}/sample/{self.scen.competition}": "/kaggle/input"}
|
||||
|
||||
# Clean the scores.csv & submission.csv.
|
||||
implementation.execute(env=env, entry=f"rm submission.csv scores.csv")
|
||||
stdout = implementation.execute(env=env, entry=f"python main.py")
|
||||
stdout = re.sub(r"=== Start of EDA part ===(.*)=== End of EDA part ===", "", stdout)
|
||||
|
||||
score_fp = implementation.workspace_path / "scores.csv"
|
||||
score_ret_code = 0
|
||||
score_check_text = ""
|
||||
if not score_fp.exists():
|
||||
score_check_text = "[Error] Metrics file (scores.csv) is not generated!"
|
||||
score_ret_code = 1
|
||||
else:
|
||||
try:
|
||||
score_df = pd.read_csv(score_fp, index_col=0)
|
||||
model_set_in_scores = set(score_df.index)
|
||||
|
||||
# Check model names (index)
|
||||
if "ensemble" not in model_set_in_scores:
|
||||
score_check_text += (
|
||||
f"\n[Error] The score dataframe doesn't contain the ensemble model.\nscore_df is:\n{score_df}"
|
||||
)
|
||||
score_ret_code = 1
|
||||
|
||||
# Check metric name (columns)
|
||||
if score_df.columns.tolist() != [self.scen.metric_name]:
|
||||
score_check_text += f"\n[Error] The scores dataframe does not contain the correct column names.\nCorrect columns is: ['{self.scen.metric_name}']\nBut got: {score_df.columns.tolist()}"
|
||||
score_ret_code = 1
|
||||
|
||||
except Exception as e:
|
||||
score_check_text += f"\n[Error] in checking the scores.csv file: {e}\nscores.csv's content:\n-----\n{score_fp.read_text()}\n-----"
|
||||
score_ret_code = 1
|
||||
|
||||
# Check submission file
|
||||
base_check_code = (DIRNAME / "eval_tests" / "submission_format_test.txt").read_text()
|
||||
implementation.inject_files(**{"test/submission_format_test.py": base_check_code})
|
||||
# stdout += "----Submission Check 1-----\n"
|
||||
submission_check_out, submission_ret_code = implementation.execute_ret_code(
|
||||
env=env, entry="python test/submission_format_test.py"
|
||||
)
|
||||
stdout += "\n" + submission_check_out
|
||||
|
||||
system_prompt = T(".prompts:pipeline_eval.system").r(
|
||||
scenario=self.scen.get_scenario_all_desc(),
|
||||
task_desc=target_task.get_task_information(),
|
||||
spec=T("scenarios.data_science.share:component_spec.Pipeline").r(),
|
||||
)
|
||||
user_prompt = T(".prompts:pipeline_eval.user").r(
|
||||
stdout=stdout.strip(),
|
||||
code=implementation.file_dict["main.py"],
|
||||
)
|
||||
wfb = build_cls_from_json_with_retry(
|
||||
PipelineSingleFeedback,
|
||||
system_prompt=system_prompt,
|
||||
user_prompt=user_prompt,
|
||||
init_kwargs_update_func=PipelineSingleFeedback.val_and_update_init_dict,
|
||||
)
|
||||
if score_ret_code != 0:
|
||||
wfb.final_decision = False
|
||||
wfb.return_checking += "\n" + score_check_text
|
||||
if submission_ret_code != 0:
|
||||
wfb.final_decision = False
|
||||
wfb.return_checking += "\nSubmission file check failed."
|
||||
return wfb
|
||||
@@ -0,0 +1,77 @@
|
||||
from pathlib import Path
|
||||
import pandas as pd
|
||||
import hashlib
|
||||
|
||||
def calculate_md5(file_path):
|
||||
with open(file_path, "rb") as f:
|
||||
file_hash = hashlib.md5(f.read()).hexdigest()
|
||||
return file_hash
|
||||
|
||||
file_md5 = calculate_md5("scores.csv")
|
||||
|
||||
"""
|
||||
find . | grep -i sample | grep -i submission | grep -v sample_submission.csv | grep -v zip_files | grep -v 'sample/'
|
||||
./denoising-dirty-documents/sampleSubmission.csv
|
||||
./the-icml-2013-whale-challenge-right-whale-redux/sampleSubmission.csv
|
||||
./text-normalization-challenge-russian-language/ru_sample_submission_2.csv.zip
|
||||
./text-normalization-challenge-russian-language/ru_sample_submission_2.csv
|
||||
./random-acts-of-pizza/sampleSubmission.csv
|
||||
./text-normalization-challenge-english-language/en_sample_submission_2.csv.zip
|
||||
./text-normalization-challenge-english-language/en_sample_submission_2.csv
|
||||
./detecting-insults-in-social-commentary/sample_submission_null.csv
|
||||
"""
|
||||
|
||||
# Find sample submission file dynamically
|
||||
input_dir = Path("/kaggle/input")
|
||||
# Look for common variations of sample submission filenames
|
||||
sample_submission_files = list(input_dir.glob("*sample_submission*.csv")) + \
|
||||
list(input_dir.glob("*sampleSubmission*.csv"))
|
||||
|
||||
assert sample_submission_files, "Error: No sample submission file found in /kaggle/input/"
|
||||
|
||||
# Use first matching file
|
||||
sample_submission_name = sample_submission_files[0].name
|
||||
SAMPLE_SUBMISSION_PATH = str(sample_submission_files[0])
|
||||
print(f"Using sample submission file: {sample_submission_name}")
|
||||
|
||||
# Check if the sample submission file exists
|
||||
assert Path(SAMPLE_SUBMISSION_PATH).exists(), f"Error: {sample_submission_name} not found at {SAMPLE_SUBMISSION_PATH}"
|
||||
|
||||
# Check if our submission file exists
|
||||
assert Path('submission.csv').exists(), "Error: submission.csv not found"
|
||||
|
||||
sample_submission = pd.read_csv(SAMPLE_SUBMISSION_PATH)
|
||||
our_submission = pd.read_csv('submission.csv')
|
||||
|
||||
success = True
|
||||
# Print the columns of the sample submission file
|
||||
print(f"Columns in {sample_submission_name}:", sample_submission.columns)
|
||||
print("Columns in our_submission.csv:", our_submission.columns)
|
||||
|
||||
for col in sample_submission.columns:
|
||||
if col not in our_submission.columns:
|
||||
success = False
|
||||
print(f'Column {col} not found in submission.csv')
|
||||
|
||||
if success:
|
||||
print(f'submission.csv\'s columns aligns with {sample_submission_name} .')
|
||||
|
||||
|
||||
# Print the first 5 rows of the two submission files, with columns separated by commas.
|
||||
def print_first_rows(file_path, file_name, num_rows=5):
|
||||
print(f"\nFirst {num_rows} rows of {file_name}:")
|
||||
try:
|
||||
with open(file_path, 'r') as file:
|
||||
for i, line in enumerate(file):
|
||||
if i < num_rows:
|
||||
print(line.strip())
|
||||
else:
|
||||
break
|
||||
except FileNotFoundError:
|
||||
print(f"Error: {file_name} not found.")
|
||||
|
||||
print_first_rows(SAMPLE_SUBMISSION_PATH, sample_submission_name)
|
||||
print_first_rows('submission.csv', 'submission.csv')
|
||||
|
||||
assert calculate_md5("scores.csv") == file_md5, "scores.csv should not be rewritten"
|
||||
print(f"\nPlease Checked the content of the submission file(submission.csv should has the same format with {sample_submission_name} but might not the same index with {sample_submission_name}). ")
|
||||
@@ -0,0 +1,6 @@
|
||||
from rdagent.components.coder.CoSTEER.task import CoSTEERTask
|
||||
|
||||
|
||||
# Because we use isinstance to distinguish between different types of tasks, we need to use sub classes to represent different types of tasks
|
||||
class PipelineTask(CoSTEERTask):
|
||||
pass
|
||||
@@ -0,0 +1,134 @@
|
||||
pipeline_coder:
|
||||
system: |-
|
||||
You are a world-class data scientist and machine learning engineer with deep expertise in statistics, mathematics, and computer science.
|
||||
Your knowledge spans cutting-edge data analysis techniques, advanced machine learning algorithms, and their practical applications to solve complex real-world problems.
|
||||
|
||||
## Task Description
|
||||
{{ task_desc }}
|
||||
|
||||
## The runtime environment your code will running on
|
||||
{{ runtime_environment }}
|
||||
|
||||
## Specification your code should follow
|
||||
{{ spec }}
|
||||
|
||||
{% if queried_similar_successful_knowledge|length != 0 or queried_former_failed_knowledge|length != 0 %}
|
||||
## Relevant Information for This Task
|
||||
{% endif %}
|
||||
|
||||
{% if queried_similar_successful_knowledge|length != 0 %}
|
||||
--------- Successful Implementations for Similar Models ---------
|
||||
====={% for similar_successful_knowledge in queried_similar_successful_knowledge %} Model {{ loop.index }}:=====
|
||||
{{ similar_successful_knowledge.target_task.get_task_information() }}
|
||||
=====Code:=====
|
||||
{{ similar_successful_knowledge.implementation.all_codes }}
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
|
||||
{% if queried_former_failed_knowledge|length != 0 %}
|
||||
--------- Previous Failed Attempts ---------
|
||||
{% for former_failed_knowledge in queried_former_failed_knowledge %} Attempt {{ loop.index }}:
|
||||
=====Code:=====
|
||||
{{ former_failed_knowledge.implementation.all_codes }}
|
||||
=====Feedback:=====
|
||||
{{ former_failed_knowledge.feedback }}
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
|
||||
|
||||
## Guidelines
|
||||
1. Ensure that the dataset is loaded strictly from `/kaggle/input/`, following the exact folder structure described in the **Data Folder Description**, and do not attempt to load data from the current directory (`./`).
|
||||
2. You should avoid using logging module to output information in your generated code, and instead use the print() function.
|
||||
|
||||
## Exploratory Data Analysis (EDA) part(Required):
|
||||
- Before returning the data, you should always add an EDA part describing the data to help the following steps understand the data better.
|
||||
- The EDA part should include but not limited in the following information in plain text:
|
||||
- The shape of the data.
|
||||
- The first 5 rows of the data.
|
||||
- The data types of each column.
|
||||
- The number of missing values in each column.
|
||||
- The number of unique values in each column.
|
||||
- The distribution of the target variable.
|
||||
- Any other information that you think is important for the following steps.
|
||||
- The EDA part should be drafted in plain text sending to standard output with command print or other similar functions with no more than ten thousand characters in the following schema:
|
||||
=== Start of EDA part ===
|
||||
{ You EDA output content }
|
||||
=== End of EDA part ===
|
||||
User will use the following code to match: re.search(r"(.*?)=== Start of EDA part ===(.*)=== End of EDA part ===", stdout, re.DOTALL).groups()[1]
|
||||
- An evaluation agent will help to check whether the EDA part is added correctly.
|
||||
- During the EDA part, you should try to avoid any irrelevant information sending to the standard output.
|
||||
|
||||
## Output Format
|
||||
{% if out_spec %}
|
||||
{{ out_spec }}
|
||||
{% else %}
|
||||
Please response the code in the following json format. Here is an example structure for the JSON output:
|
||||
{
|
||||
"code": "The Python code as a string."
|
||||
}
|
||||
{% endif %}
|
||||
|
||||
user: |-
|
||||
--------- Competition Information ---------
|
||||
{{ competition_info }}
|
||||
|
||||
--------- Data Folder Description (All path are relative to the data folder) ---------
|
||||
{{ folder_spec }}
|
||||
|
||||
{% if latest_code %}
|
||||
--------- Former code ---------
|
||||
{{ latest_code }}
|
||||
{% if latest_code_feedback is not none %}
|
||||
--------- Feedback to former code ---------
|
||||
{{ latest_code_feedback }}
|
||||
The former code contains errors. You should correct the code based on the provided information, ensuring you do not repeat the same mistakes.
|
||||
{% else %}
|
||||
The former code is correct. You should try to improve the code based on the provided task while not changing the irrelevant parts.
|
||||
{% endif %}
|
||||
{% endif %}
|
||||
|
||||
You should strictly follow the code specifications provided by the specification to implement the function.
|
||||
|
||||
|
||||
pipeline_eval:
|
||||
system: |-
|
||||
You are a data scientist responsible for evaluating code generation.
|
||||
|
||||
## Task Description
|
||||
The user is trying to build a code in the following scenario:
|
||||
{{ scenario }}
|
||||
|
||||
The main code generation task is as follows:
|
||||
{{ task_desc }}
|
||||
|
||||
The details on how to structure the code are given in the specification:
|
||||
{{ spec }}
|
||||
|
||||
## Evaluation Scope
|
||||
Your focus is to check whether the workflow code:
|
||||
1. Executes successfully, correctly generating a final submission.
|
||||
2. Generates predictions in the correct format, ensuring they align with the submission structure!
|
||||
|
||||
## Evaluation Criteria
|
||||
You will be given the execution output (`stdout`) to determine correctness.
|
||||
|
||||
[Note]
|
||||
1. Model performance is NOT a concern in this evaluation—only correct execution and formatting matter.
|
||||
2. You only check the format of the submission since we only feed you part of the data, so the submission might has different index to the sample submission data.
|
||||
|
||||
Please respond with your feedback in the following JSON format and order
|
||||
```json
|
||||
{
|
||||
"execution": "Describe whether the code executed successfully, correctly integrating all components and generating the final submission. Include any errors or issues encountered, and append all error messages and full traceback details without summarizing or omitting any information.",
|
||||
"return_checking": "Verify the generated files, particularly the submission file. Ensure that its format matches the sample submission, checking the index, column names, and CSV content.",
|
||||
"code": "Provide feedback on code quality, readability, and adherence to the given specifications.",
|
||||
"final_decision": <true/false>
|
||||
}
|
||||
```
|
||||
|
||||
user: |-
|
||||
--------- code generated by user ---------
|
||||
{{ code }}
|
||||
|
||||
--------- code running stdout ---------
|
||||
{{ stdout }}
|
||||
Reference in New Issue
Block a user