mirror of
https://github.com/NicolasBohn/NexQuant.git
synced 2026-07-27 23:47:46 +00:00
feat: out spec change for o1-preview (#666)
* add not_json batcheditout * ensemble out_spec change * feature out_spec change * model out_spec change * workflow out_spec change * runner debugger out_spec change * filter_progress_bar return format fix * data_loader and spec out_spec change * show finish_reason in llm log * json_mode fix * remove hardcode * fix CI * fix grammer * complete PythonBatchEditOut logic --------- Co-authored-by: yuanteli <1957922024@qq.com>
This commit is contained in:
XianBW
@@ -32,6 +32,7 @@ from rdagent.core.exception import CoderError
|
||||
from rdagent.core.experiment import FBWorkspace
|
||||
from rdagent.core.scenario import Scenario
|
||||
from rdagent.oai.llm_utils import APIBackend
|
||||
from rdagent.utils.agent.ret import PythonAgentOut
|
||||
from rdagent.utils.agent.tpl import T
|
||||
|
||||
|
||||
@@ -76,6 +77,7 @@ class EnsembleMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
|
||||
queried_former_failed_knowledge[0] if queried_former_failed_knowledge else None
|
||||
),
|
||||
all_code=workspace.all_codes,
|
||||
out_spec=PythonAgentOut.get_spec(),
|
||||
)
|
||||
user_prompt = T(".prompts:ensemble_coder.user").r(
|
||||
ensemble_spec=workspace.file_dict["spec/ensemble.md"],
|
||||
@@ -84,14 +86,12 @@ class EnsembleMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
|
||||
)
|
||||
|
||||
for _ in range(5):
|
||||
ensemble_code = json.loads(
|
||||
ensemble_code = PythonAgentOut.extract_output(
|
||||
APIBackend().build_messages_and_create_chat_completion(
|
||||
user_prompt=user_prompt,
|
||||
system_prompt=system_prompt,
|
||||
json_mode=True,
|
||||
json_target_type=Dict[str, str],
|
||||
)
|
||||
)["code"]
|
||||
)
|
||||
if ensemble_code != workspace.file_dict.get("ensemble.py"):
|
||||
break
|
||||
else:
|
||||
|
||||
@@ -41,10 +41,14 @@ ensemble_coder:
|
||||
2. You should avoid using logging module to output information in your generated code, and instead use the print() function.
|
||||
|
||||
## Output Format
|
||||
{% if out_spec %}
|
||||
{{ out_spec }}
|
||||
{% else %}
|
||||
Please response the code in the following json format. Here is an example structure for the JSON output:
|
||||
{
|
||||
"code": "The Python code as a string."
|
||||
}
|
||||
{% endif %}
|
||||
|
||||
user: |-
|
||||
--------- Ensemble Specification ---------
|
||||
|
||||
@@ -19,6 +19,7 @@ from rdagent.core.exception import CoderError
|
||||
from rdagent.core.experiment import FBWorkspace
|
||||
from rdagent.core.scenario import Scenario
|
||||
from rdagent.oai.llm_utils import APIBackend
|
||||
from rdagent.utils.agent.ret import PythonAgentOut
|
||||
from rdagent.utils.agent.tpl import T
|
||||
|
||||
|
||||
@@ -61,6 +62,7 @@ class FeatureMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
|
||||
data_loader_code=workspace.file_dict.get("load_data.py"),
|
||||
queried_similar_successful_knowledge=queried_similar_successful_knowledge,
|
||||
queried_former_failed_knowledge=queried_former_failed_knowledge[0],
|
||||
out_spec=PythonAgentOut.get_spec(),
|
||||
)
|
||||
user_prompt = T(".prompts:feature_coder.user").r(
|
||||
feature_spec=workspace.file_dict["spec/feature.md"],
|
||||
@@ -69,14 +71,12 @@ class FeatureMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
|
||||
)
|
||||
|
||||
for _ in range(5):
|
||||
feature_code = json.loads(
|
||||
feature_code = PythonAgentOut.extract_output(
|
||||
APIBackend().build_messages_and_create_chat_completion(
|
||||
user_prompt=user_prompt,
|
||||
system_prompt=system_prompt,
|
||||
json_mode=True,
|
||||
json_target_type=Dict[str, str],
|
||||
)
|
||||
)["code"]
|
||||
)
|
||||
if feature_code != workspace.file_dict.get("feature.py"):
|
||||
break
|
||||
else:
|
||||
|
||||
@@ -45,10 +45,14 @@ feature_coder:
|
||||
- You should avoid using logging module to output information in your generated code, and instead use the print() function.
|
||||
|
||||
## Output Format
|
||||
{% if out_spec %}
|
||||
{{ out_spec }}
|
||||
{% else %}
|
||||
Please response the code in the following json format. Here is an example structure for the JSON output:
|
||||
{
|
||||
"code": "The Python code as a string."
|
||||
}
|
||||
{% endif %}
|
||||
|
||||
user: |-
|
||||
--------- Feature Processing Specification ---------
|
||||
|
||||
@@ -20,7 +20,7 @@ from rdagent.core.exception import CoderError
|
||||
from rdagent.core.experiment import FBWorkspace
|
||||
from rdagent.core.scenario import Scenario
|
||||
from rdagent.oai.llm_utils import APIBackend
|
||||
from rdagent.utils.agent.ret import BatchEditOut
|
||||
from rdagent.utils.agent.ret import PythonBatchEditOut
|
||||
from rdagent.utils.agent.tpl import T
|
||||
|
||||
|
||||
@@ -63,7 +63,7 @@ class ModelMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
|
||||
feature_code=workspace.file_dict["feature.py"],
|
||||
queried_similar_successful_knowledge=queried_similar_successful_knowledge,
|
||||
queried_former_failed_knowledge=queried_former_failed_knowledge[0],
|
||||
out_spec=BatchEditOut.get_spec(),
|
||||
out_spec=PythonBatchEditOut.get_spec(),
|
||||
)
|
||||
# user_prompt = T(".prompts:model_coder.user").r(
|
||||
# model_spec=workspace.file_dict["spec/model.md"],
|
||||
@@ -80,12 +80,10 @@ class ModelMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
|
||||
)
|
||||
|
||||
for _ in range(5):
|
||||
batch_edit = BatchEditOut.extract_output(
|
||||
batch_edit = PythonBatchEditOut.extract_output(
|
||||
APIBackend().build_messages_and_create_chat_completion(
|
||||
user_prompt=user_prompt,
|
||||
system_prompt=system_prompt,
|
||||
json_mode=BatchEditOut.json_mode,
|
||||
json_target_type=Dict[str, str],
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@@ -50,6 +50,7 @@ from rdagent.core.exception import CoderError
|
||||
from rdagent.core.experiment import FBWorkspace
|
||||
from rdagent.core.scenario import Scenario
|
||||
from rdagent.oai.llm_utils import APIBackend
|
||||
from rdagent.utils.agent.ret import PythonAgentOut
|
||||
from rdagent.utils.agent.tpl import T
|
||||
|
||||
|
||||
@@ -110,31 +111,11 @@ class DataLoaderMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
|
||||
|
||||
spec_session = APIBackend().build_chat_session(session_system_prompt=system_prompt)
|
||||
|
||||
data_loader_spec = json.loads(
|
||||
spec_session.build_chat_completion(
|
||||
user_prompt=data_loader_prompt, json_mode=True, json_target_type=Dict[str, str]
|
||||
)
|
||||
)["spec"]
|
||||
feature_spec = json.loads(
|
||||
spec_session.build_chat_completion(
|
||||
user_prompt=feature_prompt, json_mode=True, json_target_type=Dict[str, str]
|
||||
)
|
||||
)["spec"]
|
||||
model_spec = json.loads(
|
||||
spec_session.build_chat_completion(
|
||||
user_prompt=model_prompt, json_mode=True, json_target_type=Dict[str, str]
|
||||
)
|
||||
)["spec"]
|
||||
ensemble_spec = json.loads(
|
||||
spec_session.build_chat_completion(
|
||||
user_prompt=ensemble_prompt, json_mode=True, json_target_type=Dict[str, str]
|
||||
)
|
||||
)["spec"]
|
||||
workflow_spec = json.loads(
|
||||
spec_session.build_chat_completion(
|
||||
user_prompt=workflow_prompt, json_mode=True, json_target_type=Dict[str, str]
|
||||
)
|
||||
)["spec"]
|
||||
data_loader_spec = spec_session.build_chat_completion(user_prompt=data_loader_prompt)
|
||||
feature_spec = spec_session.build_chat_completion(user_prompt=feature_prompt)
|
||||
model_spec = spec_session.build_chat_completion(user_prompt=model_prompt)
|
||||
ensemble_spec = spec_session.build_chat_completion(user_prompt=ensemble_prompt)
|
||||
workflow_spec = spec_session.build_chat_completion(user_prompt=workflow_prompt)
|
||||
else:
|
||||
data_loader_spec = workspace.file_dict["spec/data_loader.md"]
|
||||
feature_spec = workspace.file_dict["spec/feature.md"]
|
||||
@@ -147,6 +128,7 @@ class DataLoaderMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
|
||||
task_desc=data_loader_task_info,
|
||||
queried_similar_successful_knowledge=queried_similar_successful_knowledge,
|
||||
queried_former_failed_knowledge=queried_former_failed_knowledge[0],
|
||||
out_spec=PythonAgentOut.get_spec(),
|
||||
)
|
||||
user_prompt = T(".prompts:data_loader_coder.user").r(
|
||||
competition_info=competition_info,
|
||||
@@ -157,14 +139,12 @@ class DataLoaderMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
|
||||
)
|
||||
|
||||
for _ in range(5):
|
||||
data_loader_code = json.loads(
|
||||
data_loader_code = PythonAgentOut.extract_output(
|
||||
APIBackend().build_messages_and_create_chat_completion(
|
||||
user_prompt=user_prompt,
|
||||
system_prompt=system_prompt,
|
||||
json_mode=True,
|
||||
json_target_type=Dict[str, str],
|
||||
)
|
||||
)["code"]
|
||||
)
|
||||
if data_loader_code != workspace.file_dict.get("load_data.py"):
|
||||
break
|
||||
else:
|
||||
|
||||
@@ -82,10 +82,8 @@ spec:
|
||||
You should follow the provided specifications to improve this task.
|
||||
{% endif %}
|
||||
|
||||
Please respond with a JSON structure as follows:
|
||||
{
|
||||
"spec": "The function definition in code format, tailored to the Competition Information, with detailed explanations provided in the docstring."
|
||||
}
|
||||
## Output Format
|
||||
You should return the specification in markdown format directly, while the **function definition** within it should be in code format, tailored to the Competition Information, with detailed explanations provided in the docstring.
|
||||
|
||||
feature: |-
|
||||
Feature engineering specification text should adhere to the following requirements:
|
||||
@@ -137,10 +135,8 @@ spec:
|
||||
You should follow the provided specifications to improve this task.
|
||||
{% endif %}
|
||||
|
||||
Please respond with a JSON structure as follows:
|
||||
{
|
||||
"spec": "The function definition in code format, tailored to the Competition Information, with detailed explanations provided in the docstring."
|
||||
}
|
||||
## Output Format
|
||||
You should return the specification in markdown format directly, while the **function definition** within it should be in code format, tailored to the Competition Information, with detailed explanations provided in the docstring.
|
||||
|
||||
model: |-
|
||||
Model building specification text should adhere to the following requirements:
|
||||
@@ -184,10 +180,8 @@ spec:
|
||||
You should follow the provided specifications to improve this task.
|
||||
{% endif %}
|
||||
|
||||
Please respond in the following JSON format:
|
||||
{
|
||||
"spec": "The function definition in code format, tailored to the Competition Information, with detailed explanations provided in the docstring."
|
||||
}
|
||||
## Output Format
|
||||
You should return the specification in markdown format directly, while the **function definition** within it should be in code format, tailored to the Competition Information, with detailed explanations provided in the docstring.
|
||||
|
||||
ensemble: |-
|
||||
Ensemble specification text adhere to the following requirements:
|
||||
@@ -244,72 +238,69 @@ spec:
|
||||
You should follow the provided specifications to improve this task.
|
||||
{% endif %}
|
||||
|
||||
Please respond in the following JSON format:
|
||||
{
|
||||
"spec": "The function definition in code format, tailored to the Competition Information, with detailed explanations provided in the docstring."
|
||||
}
|
||||
## Output Format
|
||||
You should return the specification in markdown format directly, while the **function definition** within it should be in code format, tailored to the Competition Information, with detailed explanations provided in the docstring.
|
||||
|
||||
workflow: |-
|
||||
Your task is to implement the main workflow script (`main.py`) for a Kaggle-style machine learning competition project.
|
||||
Follow the provided project structure and specifications to ensure consistency and maintainability:
|
||||
1. Workflow Integration:
|
||||
- Integrate the following components into the workflow:
|
||||
- Data loading (`load_data.py`).
|
||||
- Feature engineering (`feature.py`).
|
||||
- Model workflow for training and testing (`model_*.py`).
|
||||
- Ensemble workflow that combines results from the model workflow to obtain the final prediction (`ensemble.py`).
|
||||
- Treat each component as a modular and callable Python function.
|
||||
- The workflow script should be flexible enough to handle either a single model or multiple models, with filenames (model_*.py) that are not determined at the outset.
|
||||
For multiple model selection, utilize Python code to identify eligible models based on filenames, for example:
|
||||
```python
|
||||
available_models = [f for f in os.listdir('.') if f.startswith('model_') and 'test' not in f]
|
||||
```
|
||||
2. Feature Engineering
|
||||
- The feature engineering should be called only once. For example:
|
||||
`X_transformed, y_transformed, X_test_transformed = feat_eng(X, y, X_test)`
|
||||
- It should be called before dataset splitting.
|
||||
1. Workflow Integration:
|
||||
- Integrate the following components into the workflow:
|
||||
- Data loading (`load_data.py`).
|
||||
- Feature engineering (`feature.py`).
|
||||
- Model workflow for training and testing (`model_*.py`).
|
||||
- Ensemble workflow that combines results from the model workflow to obtain the final prediction (`ensemble.py`).
|
||||
- Treat each component as a modular and callable Python function.
|
||||
- The workflow script should be flexible enough to handle either a single model or multiple models, with filenames (model_*.py) that are not determined at the outset.
|
||||
For multiple model selection, utilize Python code to identify eligible models based on filenames, for example:
|
||||
```python
|
||||
available_models = [f for f in os.listdir('.') if f.startswith('model_') and 'test' not in f]
|
||||
```
|
||||
2. Feature Engineering
|
||||
- The feature engineering should be called only once. For example:
|
||||
`X_transformed, y_transformed, X_test_transformed = feat_eng(X, y, X_test)`
|
||||
- It should be called before dataset splitting.
|
||||
|
||||
3. Dataset Splitting
|
||||
- The dataset returned by `load_data` is not pre-split. After calling `feat_eng`, split the data into training and test sets.
|
||||
- If feasible, apply cross-validation on the training set (`X_transformed`, `y_transformed`) to ensure a reliable assessment of model performance.
|
||||
- Keep the test set (`X_test_transformed`) unchanged, as it is only used for generating the final predictions.
|
||||
3. Dataset Splitting
|
||||
- The dataset returned by `load_data` is not pre-split. After calling `feat_eng`, split the data into training and test sets.
|
||||
- If feasible, apply cross-validation on the training set (`X_transformed`, `y_transformed`) to ensure a reliable assessment of model performance.
|
||||
- Keep the test set (`X_test_transformed`) unchanged, as it is only used for generating the final predictions.
|
||||
|
||||
4. Submission File:
|
||||
- Save the final predictions as `submission.csv`, ensuring the format matches the competition requirements (refer to `sample_submission` in the Folder Description for the correct structure).
|
||||
- Present the required submission format explicitly and ensure the output adheres to it.
|
||||
4. Submission File:
|
||||
- Save the final predictions as `submission.csv`, ensuring the format matches the competition requirements (refer to `sample_submission` in the Folder Description for the correct structure).
|
||||
- Present the required submission format explicitly and ensure the output adheres to it.
|
||||
|
||||
5. Code Standards:
|
||||
- Do not use progress bars (e.g., tqdm) in the code.
|
||||
5. Code Standards:
|
||||
- Do not use progress bars (e.g., tqdm) in the code.
|
||||
|
||||
6. Ensemble Strategy:
|
||||
Consolidate all model outputs into a dictionary, where each key is the model's filename (excluding the .py extension) and its corresponding value is the model's output.
|
||||
Sample code:
|
||||
{% raw %}
|
||||
{% for model_name in model_names %}
|
||||
model_module = __import__(model_name.replace('.py', ''))
|
||||
val_pred, test_pred, _ = model_module.model_workflow(
|
||||
X=train_X,
|
||||
y=train_y,
|
||||
val_X=val_X,
|
||||
val_y=val_y,
|
||||
test_X=X_test_transformed
|
||||
)
|
||||
val_preds_dict[model_module.__name__] = val_pred
|
||||
test_preds_dict[model_module.__name__] = test_pred
|
||||
{% endfor %}
|
||||
final_pred = ensemble_workflow(test_preds_dict, val_preds_dict, val_y)
|
||||
{% endraw %}
|
||||
6. Ensemble Strategy:
|
||||
Consolidate all model outputs into a dictionary, where each key is the model's filename (excluding the .py extension) and its corresponding value is the model's output.
|
||||
Sample code:
|
||||
{% raw %}
|
||||
{% for model_name in model_names %}
|
||||
model_module = __import__(model_name.replace('.py', ''))
|
||||
val_pred, test_pred, _ = model_module.model_workflow(
|
||||
X=train_X,
|
||||
y=train_y,
|
||||
val_X=val_X,
|
||||
val_y=val_y,
|
||||
test_X=X_test_transformed
|
||||
)
|
||||
val_preds_dict[model_module.__name__] = val_pred
|
||||
test_preds_dict[model_module.__name__] = test_pred
|
||||
{% endfor %}
|
||||
final_pred = ensemble_workflow(test_preds_dict, val_preds_dict, val_y)
|
||||
{% endraw %}
|
||||
|
||||
{% if latest_spec %}
|
||||
7. Former Specification:
|
||||
{{ latest_spec }}
|
||||
You should follow the provided specifications to improve this task.
|
||||
{% endif %}
|
||||
{% if latest_spec %}
|
||||
7. Former Specification:
|
||||
{{ latest_spec }}
|
||||
You should follow the provided specifications to improve this task.
|
||||
{% endif %}
|
||||
|
||||
Please response the specification in the following json format. Here is an example structure for the JSON output:
|
||||
{
|
||||
"spec": "The corresponding specification string as described above. You should create the rules based on the competition information instead of copying the requirements."
|
||||
}
|
||||
## Output Format
|
||||
You should return the specification in markdown format directly.
|
||||
You should create the rules based on the competition information instead of copying the requirements.
|
||||
|
||||
data_loader_coder:
|
||||
system: |-
|
||||
@@ -365,10 +356,14 @@ data_loader_coder:
|
||||
- During the EDA part, you should try to avoid any irrelevant information sending to the standard output.
|
||||
|
||||
## Output Format
|
||||
{% if out_spec %}
|
||||
{{ out_spec }}
|
||||
{% else %}
|
||||
Please response the code in the following json format. Here is an example structure for the JSON output:
|
||||
{
|
||||
"code": "The Python code as a string."
|
||||
}
|
||||
{% endif %}
|
||||
|
||||
user: |-
|
||||
--------- Competition Information ---------
|
||||
|
||||
@@ -21,6 +21,7 @@ from rdagent.core.exception import CoderError
|
||||
from rdagent.core.experiment import FBWorkspace
|
||||
from rdagent.core.scenario import Scenario
|
||||
from rdagent.oai.llm_utils import APIBackend
|
||||
from rdagent.utils.agent.ret import PythonAgentOut
|
||||
from rdagent.utils.agent.tpl import T
|
||||
|
||||
|
||||
@@ -60,6 +61,7 @@ class WorkflowMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
|
||||
competition_info=self.scen.get_scenario_all_desc(),
|
||||
queried_similar_successful_knowledge=queried_similar_successful_knowledge,
|
||||
queried_former_failed_knowledge=queried_former_failed_knowledge[0],
|
||||
out_spec=PythonAgentOut.get_spec(),
|
||||
)
|
||||
user_prompt = T(".prompts:workflow_coder.user").r(
|
||||
load_data_code=workspace.file_dict["load_data.py"],
|
||||
@@ -72,14 +74,12 @@ class WorkflowMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
|
||||
)
|
||||
|
||||
for _ in range(5):
|
||||
workflow_code = json.loads(
|
||||
workflow_code = PythonAgentOut.extract_output(
|
||||
APIBackend().build_messages_and_create_chat_completion(
|
||||
user_prompt=user_prompt,
|
||||
system_prompt=system_prompt,
|
||||
json_mode=True,
|
||||
json_target_type=Dict[str, str],
|
||||
)
|
||||
)["code"]
|
||||
)
|
||||
if workflow_code != workspace.file_dict.get("main.py"):
|
||||
break
|
||||
else:
|
||||
|
||||
@@ -42,10 +42,14 @@ workflow_coder:
|
||||
5. You should avoid using logging module to output information in your generated code, and instead use the print() function.
|
||||
|
||||
## Output Format
|
||||
{% if out_spec %}
|
||||
{{ out_spec }}
|
||||
{% else %}
|
||||
Please response the code in the following json format. Here is an example structure for the JSON output:
|
||||
{
|
||||
"code": "The Python code as a string."
|
||||
}
|
||||
{% endif %}
|
||||
|
||||
user: |-
|
||||
--------- Workflow Specification ---------
|
||||
|
||||
@@ -403,7 +403,7 @@ class APIBackend(ABC):
|
||||
try:
|
||||
json.loads(all_response)
|
||||
except:
|
||||
match = re.search(r"```json(.*?)```", all_response, re.DOTALL)
|
||||
match = re.search(r"```json(.*)```", all_response, re.DOTALL)
|
||||
all_response = match.groups()[0] if match else all_response
|
||||
json.loads(all_response)
|
||||
if json_target_type is not None:
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
from typing import Any
|
||||
|
||||
from litellm import completion, embedding, token_counter
|
||||
from litellm import completion, embedding, supports_response_schema, token_counter
|
||||
|
||||
from rdagent.log import LogColors
|
||||
from rdagent.log import rdagent_logger as logger
|
||||
@@ -69,7 +69,7 @@ class LiteLLMAPIBackend(APIBackend):
|
||||
"""
|
||||
Call the chat completion function
|
||||
"""
|
||||
if json_mode:
|
||||
if json_mode and supports_response_schema(model=LITELLM_SETTINGS.chat_model):
|
||||
kwargs["response_format"] = {"type": "json_object"}
|
||||
|
||||
# Call LiteLLM completion
|
||||
@@ -104,6 +104,11 @@ class LiteLLMAPIBackend(APIBackend):
|
||||
else:
|
||||
content = str(response.choices[0].message.content)
|
||||
finish_reason = response.choices[0].finish_reason
|
||||
logger.info(f"{LogColors.BLUE}assistant:{LogColors.END} {content}", tag="llm_messages")
|
||||
finish_reason_str = (
|
||||
f"({LogColors.RED}Finish reason: {finish_reason}{LogColors.END})"
|
||||
if finish_reason and finish_reason != "stop"
|
||||
else ""
|
||||
)
|
||||
logger.info(f"{LogColors.BLUE}assistant:{LogColors.END} {finish_reason_str}\n{content}", tag="llm_messages")
|
||||
|
||||
return content, finish_reason
|
||||
|
||||
@@ -23,7 +23,7 @@ from rdagent.core.scenario import Scenario
|
||||
from rdagent.log import rdagent_logger as logger
|
||||
from rdagent.oai.llm_utils import APIBackend, md5_hash
|
||||
from rdagent.scenarios.data_science.dev.runner.eval import DSCoSTEERCoSTEEREvaluator
|
||||
from rdagent.utils.agent.ret import BatchEditOut
|
||||
from rdagent.utils.agent.ret import PythonBatchEditOut
|
||||
from rdagent.utils.agent.tpl import T
|
||||
from rdagent.utils.env import DockerEnv, MLEBDockerConf
|
||||
|
||||
@@ -44,19 +44,17 @@ class DSRunnerMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
|
||||
# 1. code
|
||||
system_prompt = T(".prompts:DSCoSTEER_debugger.system").r(
|
||||
task_desc=task_information_str,
|
||||
out_spec=BatchEditOut.get_spec(with_del=False),
|
||||
out_spec=PythonBatchEditOut.get_spec(with_del=False),
|
||||
)
|
||||
user_prompt = T(".prompts:DSCoSTEER_debugger.user").r(
|
||||
code=workspace.all_codes,
|
||||
feedback=prev_task_feedback,
|
||||
)
|
||||
|
||||
batch_edit = BatchEditOut.extract_output(
|
||||
batch_edit = PythonBatchEditOut.extract_output(
|
||||
APIBackend().build_messages_and_create_chat_completion(
|
||||
user_prompt=user_prompt,
|
||||
system_prompt=system_prompt,
|
||||
json_mode=BatchEditOut.json_mode,
|
||||
json_target_type=Dict[str, str],
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@@ -48,3 +48,22 @@ class BatchEditOut(AgentOut):
|
||||
@classmethod
|
||||
def extract_output(cls, resp: str):
|
||||
return json.loads(resp)
|
||||
|
||||
|
||||
class PythonBatchEditOut(AgentOut):
|
||||
|
||||
@classmethod
|
||||
def get_spec(cls, with_del=True):
|
||||
return T(".tpl:PythonBatchEditOut").r(with_del=with_del)
|
||||
|
||||
@classmethod
|
||||
def extract_output(cls, resp: str):
|
||||
code_blocks = {}
|
||||
pattern = re.compile(r"```(.*?)\n(.*?)\n```", re.DOTALL)
|
||||
matches = pattern.findall(resp)
|
||||
|
||||
for match in matches:
|
||||
file_name, code = match
|
||||
code_blocks[file_name.strip()] = code.strip()
|
||||
|
||||
return code_blocks
|
||||
|
||||
@@ -5,9 +5,9 @@ PythonAgentOut: |-
|
||||
```
|
||||
|
||||
BatchEditOut: |-
|
||||
You should return a edition that applies to multiple files in a workspace in JSON.
|
||||
You should return an edition that applies to multiple files in a workspace in JSON.
|
||||
Except for the model file, other files should not be renamed.
|
||||
Files that do not need to be modified do not need to be included in the returned dict.
|
||||
Files that do not need modifications should not be included in the returned text.
|
||||
|
||||
For example:
|
||||
Inject the code into the folder. Your file name should always contain the suffix. Your file name keys should be unique to avoid delete or replace conflicts.
|
||||
@@ -19,3 +19,31 @@ BatchEditOut: |-
|
||||
<file name2> (optional): "<code>" // indicate writing <code> into <file name2> (create a new file or update an existing file)
|
||||
{% endif %}
|
||||
}
|
||||
|
||||
PythonBatchEditOut: |-
|
||||
You should return an edition that applies to multiple files in a workspace.
|
||||
Except for the model file, other files should not be renamed.
|
||||
Files that do not need modifications should not be included in the returned text.
|
||||
|
||||
Response format should be like:
|
||||
```<file name 1>
|
||||
<code>
|
||||
```
|
||||
```<file name 2>
|
||||
<code>
|
||||
```
|
||||
{% if with_del %}
|
||||
```<file name 3>
|
||||
__DEL__
|
||||
```
|
||||
{% endif %}
|
||||
...
|
||||
|
||||
NOTE:
|
||||
- The file name should always contain the suffix.
|
||||
- The file name should be unique to prevent conflicts during removal or replacement.
|
||||
- To indicate writing code into a file, provide the corresponding <code> within the code block (creating a new file or updating an existing one).
|
||||
{% if with_del %}
|
||||
- To explicitly remove a file, provide only `__DEL__` within the code block for that file.
|
||||
- To replace a file with a new one, first provide ` __DEL__` for the original file, then include a separate entry with new file name and the new code.
|
||||
{% endif %}
|
||||
@@ -5,7 +5,6 @@ filter_progress_bar:
|
||||
Note: About the training log message, if the log message contains useful information like loss or accuracy and it is reported in each epoch, it should not be removed. If the log message is not useful, for example, reporting nan in each iteration or just reporting the iteration number, please remove them.
|
||||
|
||||
Respond in the following JSON format and order:
|
||||
```json
|
||||
{
|
||||
"needs_sub": <true/false>,
|
||||
"regex_patterns": ["regex pattern 1", "regex pattern 2", ...]
|
||||
|
||||
Reference in New Issue
Block a user