mirror of
https://github.com/NicolasBohn/NexQuant.git
synced 2026-08-02 09:57:44 +00:00
fix: add spec for hyperparameters in task design and coder (#995)
* init commit * remove the 5-fold spec from prompts * refine the hyperparameter specification * do not sample data * a small spelling issue * refine prompt to avoid submission cheating * do not sample data * simplify code * refine the coder evaluator prompt * refine wording * remove runtime from proposal * refine wording * refine prompt * add gpu info in runtime_info.py * modify the spec * add router and add refinement exp gen * fix prompt bug * use rule-based logic for router * complete the prompt * fix circular import bug * fix bug * make refine_decision optional * update pipeline prompts: (1) add scenary: in an iterative cooding loop and use sample datasets (2)add some generation tops in coding (3)add evaluation guidelines in evaluation (4)polish the json schema and description * fix a small bug * fix a small bug * rdagent/scenarios/data_science/loop.py back to the original version * refactor: replace _get_exp_gen with default_exp_gen for exp generation * import * refactor: make the __init__ back to main * fix small bugs * fix bugs for proposal_version * move refine into runner * check early stop * EDA improvement & coder classes number * fix CI * slightly refine the prompt * remove rule_base_eval and remove useless prompt --------- Co-authored-by: Xu <v-xuminrui@microsoft.com> Co-authored-by: TPLin22 <tplin2@163.com> Co-authored-by: amstrongzyf <amstrongzyf@126.com> Co-authored-by: Xu Yang <peteryang@vip.qq.com> Co-authored-by: Xu Yang <xuyang1@microsoft.com> Co-authored-by: Young <afe.young@gmail.com>
This commit is contained in:
Roland Minrui
+1
-1
@@ -39,7 +39,7 @@ ENABLE_CACHE=False
|
||||
PROMPT_CACHE_PATH=./log/prompt_cache.db
|
||||
|
||||
DS_CODER_COSTEER_ENV_TYPE=conda
|
||||
DS_PROPOSAL_VERSION=v2
|
||||
# DS_PROPOSAL_VERSION=v2 deprecated
|
||||
|
||||
DS_CODER_ON_WHOLE_PIPELINE=True
|
||||
COSTEER_V2_QUERY_FORMER_TRACE_LIMIT=3
|
||||
|
||||
@@ -38,14 +38,14 @@ class DataScienceBasePropSetting(KaggleBasePropSetting):
|
||||
spec_enabled: bool = True
|
||||
|
||||
#### proposal related
|
||||
proposal_version: str = "v1"
|
||||
coder_on_whole_pipeline: bool = False
|
||||
# proposal_version: str = "v2" deprecated
|
||||
|
||||
coder_on_whole_pipeline: bool = True
|
||||
max_trace_hist: int = 3
|
||||
|
||||
coder_max_loop: int = 10
|
||||
runner_max_loop: int = 1
|
||||
|
||||
rule_base_eval: bool = False
|
||||
sample_data_by_LLM: bool = False
|
||||
use_raw_description: bool = False
|
||||
show_nan_columns: bool = False
|
||||
|
||||
@@ -95,7 +95,6 @@ class PipelineMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
|
||||
queried_former_failed_knowledge=queried_former_failed_knowledge[0],
|
||||
out_spec=PythonAgentOut.get_spec(),
|
||||
runtime_environment=runtime_environment,
|
||||
spec=T("scenarios.data_science.share:component_spec.Pipeline").r(),
|
||||
enable_model_dump=DS_RD_SETTING.enable_model_dump,
|
||||
enable_debug_mode=DS_RD_SETTING.sample_data_by_LLM,
|
||||
)
|
||||
|
||||
@@ -75,11 +75,7 @@ class PipelineCoSTEEREvaluator(CoSTEEREvaluator):
|
||||
if match := re.search(r"estimated_time:\s*(\d+(?:.\d+)?)", result.stdout, re.DOTALL):
|
||||
full_estimated_time = float(match.group(1))
|
||||
if debug_time is not None and full_estimated_time is not None:
|
||||
stdout += f"Debug mode ran in {debug_time:.2f} seconds, estimated full run time is {full_estimated_time:.2f} seconds.\n"
|
||||
if full_estimated_time < env.conf.running_timeout_period * 3:
|
||||
stdout += "The estimated full run time is less than three times the timeout period.\n"
|
||||
else:
|
||||
stdout += f"The estimated full run time is more than three times the timeout period.\n"
|
||||
stdout += f"Debug mode ran in {debug_time:.2f} seconds, estimated full run time is {full_estimated_time:.2f} seconds. The estimated time is {full_estimated_time / env.conf.running_timeout_period * 100:.2f}% the debug time."
|
||||
else:
|
||||
stdout += "Debug mode did not provide debug_time or estimated_time, it's a buggy implementation.\n"
|
||||
|
||||
@@ -130,21 +126,6 @@ class PipelineCoSTEEREvaluator(CoSTEEREvaluator):
|
||||
submission_result = implementation.run(env=env, entry="python test/submission_format_test.py")
|
||||
submission_check_out = submission_result.stdout
|
||||
submission_ret_code = submission_result.exit_code
|
||||
if DS_RD_SETTING.rule_base_eval:
|
||||
if execute_ret_code == 0 and score_ret_code == 0 and submission_ret_code == 0:
|
||||
return PipelineSingleFeedback(
|
||||
execution=stdout,
|
||||
return_checking=score_check_text + "\n" + submission_check_out,
|
||||
code="Code evaluation is not available.",
|
||||
final_decision=True,
|
||||
)
|
||||
else:
|
||||
return PipelineSingleFeedback(
|
||||
execution=stdout,
|
||||
return_checking=score_check_text + "\n" + submission_check_out,
|
||||
code="Code evaluation is not available.",
|
||||
final_decision=False,
|
||||
)
|
||||
stdout += "\n" + submission_check_out
|
||||
|
||||
if not isinstance(implementation, FBWorkspace):
|
||||
|
||||
@@ -2,6 +2,8 @@ pipeline_coder:
|
||||
system: |-
|
||||
You are a world-class data scientist and machine learning engineer with deep expertise in statistics, mathematics, and computer science.
|
||||
Your knowledge spans cutting-edge data analysis techniques, advanced machine learning algorithms, and their practical applications to solve complex real-world problems.
|
||||
|
||||
**Important Context**: You are working on sample datasets and your code will go through automated iterations. Design your code to be iteration-friendly with comprehensive print statements and clear debugging information to facilitate the automatic improvement process.
|
||||
|
||||
## Task Description
|
||||
{{ task_desc }}
|
||||
@@ -9,8 +11,13 @@ pipeline_coder:
|
||||
## The runtime environment your code will running on
|
||||
{{ runtime_environment }}
|
||||
|
||||
## Hyperparameters Specification
|
||||
Follow the hyperparameter choices if they are specified in the task description, unless they are unreasonable or incorrect.
|
||||
In this case, refer to the guidelines below for appropriate adjustments:
|
||||
{% include "scenarios.data_science.share:spec.hyperparameter" %}
|
||||
|
||||
## Specification your code should follow
|
||||
{{ spec }}
|
||||
{% include "scenarios.data_science.share:component_spec.Pipeline" %}
|
||||
|
||||
{% if queried_similar_successful_knowledge|length != 0 or queried_former_failed_knowledge|length != 0 %}
|
||||
## Relevant Information for This Task
|
||||
@@ -35,7 +42,6 @@ pipeline_coder:
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
|
||||
|
||||
## Guidelines
|
||||
1. Ensure that the dataset is loaded strictly from `{% include "scenarios.data_science.share:scen.input_path" %}`, following the exact folder structure described in the **Data Folder Description**, and do not attempt to load data from the current directory (`./`).
|
||||
2. You should avoid using logging module to output information in your generated code, and instead use the print() function.
|
||||
@@ -88,6 +94,7 @@ pipeline_coder:
|
||||
else:
|
||||
sample_size = len(train_dataset)
|
||||
```
|
||||
You should be very careful about the label classes number in the debug mode. The label classes should be the same as the full run even when you are in the debug mode. The label classes number is often used to build the model.
|
||||
{% endif %}
|
||||
|
||||
## Output Format
|
||||
@@ -113,18 +120,42 @@ pipeline_coder:
|
||||
{% if latest_code_feedback is not none %}
|
||||
--------- Feedback to former code ---------
|
||||
{{ latest_code_feedback }}
|
||||
|
||||
**Improvement Planning**: Before modifying the code, first analyze the feedback and identify at most 3 key areas that need modification. Plan your changes strategically:
|
||||
1. Prioritize the most critical issues that affect code execution or correctness
|
||||
2. Focus on improvements that will have the highest impact
|
||||
3. Ensure changes don't break existing working components
|
||||
|
||||
The former code contains errors. You should correct the code based on the provided information, ensuring you do not repeat the same mistakes.
|
||||
Keep the part that already seem correct intact. Avoid modifying them to refrain from introducing new errors.
|
||||
{% else %}
|
||||
The former code is correct. You should try to improve the code based on the provided task while not changing the irrelevant parts.
|
||||
{% endif %}
|
||||
{% endif %}
|
||||
**Improvement Planning**: Before enhancing the code, first analyze what can be improved and identify at most 3 key enhancement areas. Plan your improvements strategically:
|
||||
1. Focus on performance, robustness, or feature engineering improvements
|
||||
2. Enhance code clarity and debugging capabilities
|
||||
3. Optimize model configuration or validation strategy
|
||||
|
||||
The former code is correct. You should try to improve the code based on the provided task while not changing the irrelevant parts.{% endif %}{% endif %}
|
||||
|
||||
## Code Generation Best Practices
|
||||
1. **Avoid Hard-coding**: Avoid hard-coded values (e.g., fixed dataset size). Use proportions for data splitting instead of absolute numbers.
|
||||
2. **Data Loading Exception Handling**: Use try-except blocks ONLY for data loading operations when working with sample data. If you find zero data records, this indicates a data loading error, not data absence.
|
||||
3. **Minimize Exception Handling**: Avoid using try-except blocks outside of data loading, as they may mask underlying code issues and hinder debugging.
|
||||
4. **Limit Assertions**: Minimize the use of assert statements, as they may prevent discovery of code problems during evaluation.
|
||||
5. **Strategic Print Statements**: Add appropriate print statements at key steps to facilitate automated code iteration and debugging. Use print() function instead of logging module for output information.
|
||||
6. **Training Configuration**: For model training, use reasonable epoch numbers (for example, 10 epochs). ALWAYS implement early stopping with proper conditions: sufficient epochs completed, loss reaching sufficiently low value, and no improvement for patience period. Save best model checkpoints based on validation performance.
|
||||
7. **Submission Generation**: ALWAYS use the best saved model (not necessarily final epoch) for predictions. NEVER create dummy/placeholder submissions (e.g., all 1s, random values). If training fails, report failure honestly rather than generating fake submission files.
|
||||
8. **Resource Management**: Use all available data without sampling or subsetting due to resource limitations. If resources are insufficient, report the issue honestly rather than compromising data integrity.
|
||||
9. **Robust Data Handling**: Code should gracefully handle varying data sizes and structures without breaking on edge cases.
|
||||
10. **Clear Error Messages**: When errors occur, ensure they provide meaningful information for debugging rather than generic messages.
|
||||
11. **Don't use tqdm**: Don't use tqdm to show the progress of the training process.
|
||||
|
||||
You should strictly follow the code specifications provided by the specification to implement the function.
|
||||
|
||||
pipeline_eval:
|
||||
system: |-
|
||||
You are a data scientist responsible for evaluating code generation.
|
||||
|
||||
**Important Context**: The evaluation is performed on sample datasets and the code is designed for automated iterations. Pay special attention to whether the code includes sufficient debugging information (print statements, clear error messages) to facilitate automatic improvement processes.
|
||||
|
||||
## Task Description
|
||||
The user is trying to build a code in the following scenario:
|
||||
@@ -139,16 +170,19 @@ pipeline_eval:
|
||||
{% if is_sub_enabled %}
|
||||
## Evaluation Scope
|
||||
Your focus is to check whether the workflow code:
|
||||
Step 1: Executes successfully without any errors. Please distinguish between the errors and warnings.
|
||||
|
||||
### Step 1: Executes successfully without any errors. Please distinguish between the errors and warnings.
|
||||
|
||||
Step 2: Correctly generates a final submission in the correct format, ensuring: they align with the submission structure, the index names and column names should match the sample, and the items should not be empty or apparently incorrect. The final submission and validation metrics should not be generated with mock data or dummy values. They should only the results of the actual code execution. Once you found the coding containing any data mock or dummy values, you should emphasize it in your feedback and reject this implementation.
|
||||
|
||||
Step 3: No data sampling is applied in the full run, and the code runs on the full dataset. The code should not contain any data sampling or data reduction logic in the full run. Once you found any data sampling or reduction logic in the full run, you should emphasize it in your feedback and reject this implementation.
|
||||
|
||||
Step 4: Aligns with the competition requirements. This includes:
|
||||
### Step 2: Correctly generates a final submission in the correct format, ensuring:
|
||||
- They align with the submission structure
|
||||
- The index names and column names should match the sample
|
||||
- The items should not be empty or apparently incorrect
|
||||
- **CRITICALLY: Deep dive into code and stdout to verify the generated file is genuinely produced by successful execution, NOT created as a result of exception handling, fallback mechanisms, or error recovery processes. Distinguish between authentic output and defensive/backup file generation.**
|
||||
|
||||
### Step 3: Aligns with the competition requirements. This includes:
|
||||
- CAREFULLY ANALYZE WHETHER THE EXPERIMENTAL SETUP AND CODE MAY CAUSE MISALIGNMENT BETWEEN VALIDATION AND TEST PERFORMANCE.
|
||||
- Confirm strict adherence to the competition's evaluation rules listed in `scenario`:
|
||||
- Exact match between the implementation code of metric and the requirements of the scenario. The metric number is not the focus.
|
||||
- Exact match between the implementation code of metric and the requirements of the scenario. **The metric number is not the focus.**
|
||||
- Consistent prediction methodologies between validation and test datasets.
|
||||
- No shortcuts or fold-specific strategies applied inconsistently.
|
||||
- Rigorous checks for corner-case consistency.
|
||||
@@ -159,22 +193,36 @@ pipeline_eval:
|
||||
|
||||
{% if debug_mode %}
|
||||
Step 5: The code is executed in a debug mode with the command `python main.py --debug`. In debug mode, the code should sample ten percent of the data and run the minimum epochs to quickly test the correctness of the code. You should check whether the code follows this requirements. If not, you should emphasize it in your feedback and reject this implementation.
|
||||
You will also be given the execution time and estimated time for the full run. If not time information is provided or the estimated full run time is more than three times the timeout period, you should reject the implementation because the full run will not finish in time.
|
||||
You will also be given the execution time and estimated time for the full run. You should check whether the estimated time is too large to finish in the given time limit. You should also consider the early stopping mechanism in the code. The estimated time could be very large but the early stopping mechanism could stop the training earlier than the full epochs. You should also check whether the debug time is reasonable and the estimated time is reasonable based on the debug time.
|
||||
{% endif %}
|
||||
|
||||
## Evaluation Criteria
|
||||
You will be given the execution output (`stdout`) to determine correctness.
|
||||
|
||||
[Note]
|
||||
### Notes
|
||||
1. Model performance is NOT a concern in this evaluation—only correct execution and formatting matter.
|
||||
2. You only check the format of the submission since we only feed you part of the data, so the submission might has different index to the sample submission data.
|
||||
3. Submissions and scores must be the result of actual model inference. Any form of cheating or fabrication (e.g., random or hard-coded outputs) is strictly prohibited and should lead to rejection.
|
||||
|
||||
### Evaluation Guidelines
|
||||
1. **EDA Requirement**: EDA is mandatory and must be included in the generated code. You do NOT need to evaluate the EDA content itself.
|
||||
2. **Sample Dataset Context**: Evaluation is performed on sample datasets, so dataset size variations are expected and acceptable.
|
||||
3. **Hard-coding Check**: Verify the code avoids hard-coded values and uses **proportions** instead of absolute numbers for data splitting.
|
||||
4. **Exception Handling Review**: Check that try-except blocks are used appropriately (only for data loading) and minimized elsewhere to avoid masking code issues.
|
||||
5. **Assertion Usage**: Ensure assert statements are used sparingly to avoid preventing discovery of code problems.
|
||||
6. **Debug-Friendly Code**: Verify there are appropriate print statements at key steps for debugging and iteration. Check that print() function is used instead of logging module.
|
||||
7. **Training Configuration**: Verify epoch numbers are reasonable (at least 10), early stopping is properly implemented with appropriate conditions, and model checkpoint is used.
|
||||
8. **Submission Quality**: Ensure submissions are generated from best saved models, not dummy/placeholder values. Verify honest failure reporting if training issues occur.
|
||||
9. **Resource Management**: Check that all available data is used appropriately without unnecessary sampling or subsetting.
|
||||
10. **Robustness**: Ensure the code handles varying data sizes and structures gracefully without breaking on edge cases.
|
||||
11. **Error Clarity**: Verify that error messages provide meaningful debugging information rather than generic messages.
|
||||
|
||||
Please respond with your feedback in the following JSON format and order
|
||||
```json
|
||||
{
|
||||
"execution": "Describe whether the code executed successfully, correctly integrating all components and generating the final submission. Include any errors or issues encountered, and append all error messages and full traceback details without summarizing or omitting any information.",
|
||||
"return_checking": "Verify the generated files, particularly the submission file. Ensure that its format matches the sample submission, checking the index, column names, and CSV content.",
|
||||
"code": "Begin explicitly with [Code analysis] or [Evaluation error]. Provide feedback on code quality, readability, adherence to the given specifications, and alignment with competition requirements.",
|
||||
"execution": "Describe whether the code executed successfully, correctly integrating all components and generating the final submission. Include any errors or issues encountered, and append all error messages and full traceback details without summarizing or omitting any information. If errors occurred, analyze the root causes: (1) Are they fundamental algorithmic/approach issues, or (2) Implementation details that can be easily fixed, or (3) Environment/dependency problems?",
|
||||
"return_checking": "Examine the generated files by cross-referencing the code logic and stdout output. Verify: (1) Format matches sample submission (index, column names, CSV content); (2) **File generation authenticity**: Is the file genuinely produced by successful model execution, or is it a result of exception handling/fallback mechanisms? Cite specific code sections and stdout evidence.",
|
||||
"code": "Begin explicitly with [Code analysis] or [Evaluation error]. Provide structured analysis: (1) **Technical Appropriateness**: Does the chosen approach (algorithms, data processing, validation strategy) match this problem's data characteristics and competition requirements? (2) **Effective Components**: What specific parts work well and why are they effective for this problem type? (3) **Issues & Improvements**: Identify concrete problems and suggest actionable improvement directions (without providing actual code). (4) **Code Quality**: Assess readability, structure, and adherence to specifications.",
|
||||
"final_decision": <true/false>
|
||||
}
|
||||
```
|
||||
@@ -190,9 +238,9 @@ pipeline_eval:
|
||||
Please respond with your feedback in the following JSON format and order
|
||||
```json
|
||||
{
|
||||
"execution": "Describe whether the code executed successfully. Include any errors or issues encountered, and append all error messages and full traceback details without summarizing or omitting any information.",
|
||||
"execution": "Describe whether the code executed successfully. Include any errors or issues encountered, and append all error messages and full traceback details without summarizing or omitting any information. If errors occurred, analyze the root causes: (1) Are they fundamental algorithmic/approach issues, or (2) Implementation details that can be easily fixed, or (3) Environment/dependency problems?",
|
||||
"return_checking": "Describe the expected file to be generated.",
|
||||
"code": "Provide feedback on code quality, readability, and adherence to the given specifications.",
|
||||
"code": "Provide structured analysis: (1) **Technical Appropriateness**: Does the chosen approach (algorithms, data processing, validation strategy) match this problem's data characteristics and requirements? (2) **Effective Components**: What specific parts work well and why are they effective for this problem type? (3) **Issues & Improvements**: Identify concrete problems and suggest actionable improvement directions (without providing actual code). (4) **Code Quality**: Assess readability, structure, and adherence to specifications.",
|
||||
"final_decision": <true/false>
|
||||
}
|
||||
```
|
||||
|
||||
@@ -57,9 +57,13 @@ class ExperimentFeedback(Feedback):
|
||||
*,
|
||||
code_change_summary: str | None = None,
|
||||
decision: bool,
|
||||
refine_decision: bool = False,
|
||||
eda_improvement: str | None = None,
|
||||
exception: Exception | None = None,
|
||||
) -> None:
|
||||
self.decision = decision
|
||||
self.refine_decision = refine_decision
|
||||
self.eda_improvement = eda_improvement
|
||||
self.reason = reason
|
||||
# Exception is not None means failing to generate runnable experiments due to exception.
|
||||
# Runable reuslts are not always good.
|
||||
@@ -96,8 +100,16 @@ class HypothesisFeedback(ExperimentFeedback):
|
||||
*,
|
||||
code_change_summary: str | None = None,
|
||||
decision: bool,
|
||||
refine_decision: bool = False,
|
||||
eda_improvement: str | None = None,
|
||||
) -> None:
|
||||
super().__init__(reason, decision=decision, code_change_summary=code_change_summary)
|
||||
super().__init__(
|
||||
reason,
|
||||
decision=decision,
|
||||
refine_decision=refine_decision,
|
||||
code_change_summary=code_change_summary,
|
||||
eda_improvement=eda_improvement,
|
||||
)
|
||||
self.observations = observations
|
||||
self.hypothesis_evaluation = hypothesis_evaluation
|
||||
self.new_hypothesis = new_hypothesis
|
||||
|
||||
@@ -61,32 +61,6 @@ class DSExperiment2Feedback(Experiment2Feedback):
|
||||
f"The current score is {cur_score}, while the SOTA score is {sota_score}. "
|
||||
f"{'In this competition, higher is better.' if self.scen.metric_direction else 'In this competition, lower is better.'}"
|
||||
)
|
||||
if DS_RD_SETTING.rule_base_eval:
|
||||
if sota_exp:
|
||||
if cur_score > sota_score:
|
||||
return HypothesisFeedback(
|
||||
observations="The current score bigger than the SOTA score.",
|
||||
hypothesis_evaluation="The current score is bigger than the SOTA score.",
|
||||
new_hypothesis="No new hypothesis provided",
|
||||
reason="The current score is bigger than the SOTA score.",
|
||||
decision=True if self.scen.metric_direction else False,
|
||||
)
|
||||
elif cur_score < sota_score:
|
||||
return HypothesisFeedback(
|
||||
observations="The current score smaller than the SOTA score.",
|
||||
hypothesis_evaluation="The current score is smaller than the SOTA score.",
|
||||
new_hypothesis="No new hypothesis provided",
|
||||
reason="The current score is smaller than the SOTA score.",
|
||||
decision=False if self.scen.metric_direction else True,
|
||||
)
|
||||
else:
|
||||
return HypothesisFeedback(
|
||||
observations="The current score equals to the SOTA score.",
|
||||
hypothesis_evaluation="The current score equals to the SOTA score.",
|
||||
new_hypothesis="No new hypothesis provided",
|
||||
reason="The current score equals to the SOTA score.",
|
||||
decision=False,
|
||||
)
|
||||
|
||||
eda_output = exp.experiment_workspace.file_dict.get("EDA.md", None)
|
||||
system_prompt = T(".prompts:exp_feedback.system").r(
|
||||
@@ -128,6 +102,8 @@ class DSExperiment2Feedback(Experiment2Feedback):
|
||||
if evaluation_not_aligned
|
||||
else convert2bool(dict_get_with_warning(resp_dict, "Replace Best Result", "no"))
|
||||
),
|
||||
refine_decision=convert2bool(dict_get_with_warning(resp_dict, "Refine Decision", "no")),
|
||||
eda_improvement=dict_get_with_warning(resp_dict, "EDA Improvement", "no"), # EDA improvement suggestion
|
||||
)
|
||||
|
||||
if hypothesis_feedback and DS_RD_SETTING.enable_knowledge_base:
|
||||
|
||||
@@ -5,9 +5,9 @@ exp_feedback:
|
||||
Below is a detailed description of the current Kaggle competition scenario:
|
||||
{{ scenario }}
|
||||
|
||||
Your task is to analyze the current experiment's hypothesis, implementation (code and its changes), and results, explicitly comparing them with previous experiments and the best previous result (SOTA).
|
||||
Your task is to analyze the current experiment's hypothesis, implementation (code and its changes), and results, explicitly comparing them with previous best SOTA result step by step.
|
||||
|
||||
Step-by-step Analysis Process:
|
||||
# Step-by-step Analysis Process:
|
||||
|
||||
Step 1: Verify Submission Format
|
||||
- If the submission format check fails:
|
||||
@@ -57,9 +57,14 @@ exp_feedback:
|
||||
- Please examine the code carefully based on the above criteria and provide a detailed analysis of the code.
|
||||
- Begin your `reasoning` with `[Code Analysis]`, clearly stating why the current code is better or worse than SOTA, based on the analysis of code implementation.
|
||||
- If the current code is not better than SOTA, set `"Replace Best Result": "no"`. Otherwise, set `"Replace Best Result": "yes"`.
|
||||
|
||||
Provide detailed and constructive feedback structured as follows:
|
||||
Example JSON Structure for Result Analysis:
|
||||
|
||||
Step 5: EDA improvement analysis (if needed)
|
||||
- The user might provide Data Overview in EDA format which is the output of the EDA code. You should analyze the EDA result and provide feedback on how it can be improved.
|
||||
- The improvement might include some addons or modifications or deletions to some part of the EDA code.
|
||||
- You should provide your feedback based on the current code and SOTA code. Especially focus on the feature engineering part.
|
||||
- For example, if the code truncate the line with N words, you can suggest to print the mean, median or quantile of the length of the line for better understanding of the data in the next rounds of experiments.
|
||||
|
||||
Provide detailed and constructive feedback structured as follows without anything else:
|
||||
{
|
||||
"Submission Format Check": "yes or no",
|
||||
"First Valid Submission": "yes or no",
|
||||
@@ -68,7 +73,9 @@ exp_feedback:
|
||||
"Feedback for Hypothesis": Explicitly confirm or refute the hypothesis based on specific data points or performance trends. Limit to two sentences.",
|
||||
"Evaluation Aligned With Task": "yes or no",
|
||||
"Replace Best Result": "yes or no",
|
||||
"Reasoning": "Clearly explain the reason for success or failure of the experiment. Begin explicitly with [Submission format error], [Evaluation error], [Experiment Analysis] or [Code Analysis] depending on the step at which issues arose. Reference specific scores and methodological differences with SOTA. Limit to three sentences."
|
||||
"Refine Decision": "yes or no",
|
||||
"Reasoning": "Clearly explain the reason for success or failure of the experiment. Begin explicitly with [Submission format error], [Evaluation error], [Experiment Analysis] or [Code Analysis] depending on the step at which issues arose. Reference specific scores and methodological differences with SOTA. Limit to three sentences.",
|
||||
"EDA Improvement": "improvement suggestion for EDA code, if needed, otherwise set to 'no'. If there is no EDA code, set to 'no'."
|
||||
}
|
||||
|
||||
user: |-
|
||||
|
||||
@@ -40,16 +40,23 @@ class DSRunnerMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
|
||||
if prev_task_feedback is None:
|
||||
# if no prev_tak_feedback, it is the first loop; we do not make any changes and goto evaluators directly.
|
||||
return {}
|
||||
|
||||
task_information_str = target_task.get_task_information()
|
||||
# 1. code
|
||||
system_prompt = T(".prompts:DSCoSTEER_debugger.system").r(
|
||||
task_desc=task_information_str,
|
||||
out_spec=PythonBatchEditOut.get_spec(with_del=False),
|
||||
)
|
||||
user_prompt = T(".prompts:DSCoSTEER_debugger.user").r(
|
||||
if prev_task_feedback.hyperparameter_tuning_decision:
|
||||
task_information_str = target_task.get_task_information()
|
||||
# 1. code
|
||||
system_prompt = T(".prompts:DSCoSTEER.system_refine").r(
|
||||
out_spec=PythonBatchEditOut.get_spec(with_del=False),
|
||||
)
|
||||
else:
|
||||
task_information_str = target_task.get_task_information()
|
||||
# 1. code
|
||||
system_prompt = T(".prompts:DSCoSTEER.system_refine").r(
|
||||
task_desc=task_information_str,
|
||||
out_spec=PythonBatchEditOut.get_spec(with_del=False),
|
||||
)
|
||||
user_prompt = T(".prompts:DSCoSTEER.user").r(
|
||||
code=workspace.all_codes,
|
||||
feedback=prev_task_feedback,
|
||||
hyperparameter_tuning_suggestion=prev_task_feedback.hyperparameter_tuning_suggestion,
|
||||
)
|
||||
|
||||
batch_edit = PythonBatchEditOut.extract_output(
|
||||
|
||||
@@ -25,7 +25,19 @@ from rdagent.utils.fmt import shrink_text
|
||||
|
||||
DIRNAME = Path(__file__).absolute().resolve().parent
|
||||
|
||||
DSCoSTEEREvalFeedback = CoSTEERSingleFeedback
|
||||
|
||||
class DSCoSTEEREvalFeedback(CoSTEERSingleFeedback):
|
||||
"""
|
||||
Feedback for Data Science CoSTEER evaluation.
|
||||
This feedback is used to evaluate the code and execution of the Data Science CoSTEER task.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self, *args, hyperparameter_tuning_decision: bool = None, hyperparameter_tuning_suggestion: str = None, **kwargs
|
||||
):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.hyperparameter_tuning_decision = hyperparameter_tuning_decision
|
||||
self.hyperparameter_tuning_suggestion = hyperparameter_tuning_suggestion
|
||||
|
||||
|
||||
class DSCoSTEERCoSTEEREvaluator(CoSTEEREvaluator):
|
||||
@@ -116,27 +128,6 @@ class DSCoSTEERCoSTEEREvaluator(CoSTEEREvaluator):
|
||||
if test_eval.enabled(self.scen.competition):
|
||||
submission_check_out, submission_ret_code = test_eval.valid(self.scen.competition, implementation)
|
||||
stdout += f"\nSubmission check:\n{submission_check_out}\nIf Submission check returns a 'Submission is valid' or similar message, despite some warning messages, you should still consider the submission as valid and give a positive final decision. "
|
||||
if DS_RD_SETTING.rule_base_eval:
|
||||
if DS_RD_SETTING.if_using_mle_data:
|
||||
score_check_text = score_check_text + "\n" + submission_check_out
|
||||
if (
|
||||
execute_ret_code == 0
|
||||
and score_ret_code == 0
|
||||
and (not DS_RD_SETTING.if_using_mle_data or submission_ret_code == 0)
|
||||
):
|
||||
return DSCoSTEEREvalFeedback(
|
||||
execution=stdout,
|
||||
return_checking=score_check_text,
|
||||
code="Code evaluation is not available.",
|
||||
final_decision=True,
|
||||
)
|
||||
else:
|
||||
return DSCoSTEEREvalFeedback(
|
||||
execution=stdout,
|
||||
return_checking=score_check_text,
|
||||
code="Code evaluation is not available.",
|
||||
final_decision=False,
|
||||
)
|
||||
|
||||
system_prompt = T(".prompts:DSCoSTEER_eval.system").r(
|
||||
scenario=self.scen.get_scenario_all_desc(eda_output=implementation.file_dict.get("EDA.md", None)),
|
||||
@@ -146,6 +137,9 @@ class DSCoSTEERCoSTEEREvaluator(CoSTEEREvaluator):
|
||||
user_prompt = T(".prompts:DSCoSTEER_eval.user").r(
|
||||
code=implementation.all_codes,
|
||||
stdout=shrink_text(stdout),
|
||||
time_spent=f"{implementation.running_info.running_time:.2f} seconds",
|
||||
timeout=f"{env.conf.running_timeout_period} seconds",
|
||||
percent_of_timeout_used=f"{(implementation.running_info.running_time / env.conf.running_timeout_period) * 100:.2f}%",
|
||||
)
|
||||
|
||||
feedback = build_cls_from_json_with_retry(
|
||||
|
||||
@@ -15,11 +15,19 @@ DSCoSTEER_eval:
|
||||
- Model training
|
||||
- Ensembling
|
||||
|
||||
The user will provide you the time spent on the whole code execution and the timeout of the code execution. You should decide whether the hyperparameter is reasonable based on the time.
|
||||
For example, if the code only spent ten percent of the timeout and the hyperparameter like `n_estimators` or 'epochs' is very small or batch size is small you should suggest to increase these hyperparameter.
|
||||
Please provide your feedback in two key-value pairs:
|
||||
"hyperparameter_tuning_decision": <true/false>
|
||||
"hyperparameter_tuning_suggestion": <suggestion in plain text for hyperparameter tuning, e.g., increase n_estimators to 1000, increase epochs to 100, increase batch size to 64, give an empty string if decide not to tune the hyperparameter>
|
||||
Notice: You should only suggest the hyperparameter tuning if the code applies early stopping strategy because increasing the training time blindly may lead to overfitting. Once you found the code didn't apply early stopping strategy, you should not suggest to tune the hyperparameter.
|
||||
Your suggestion should be reasonable and include not only the target hyperparameter but also the hyperparameter sets.
|
||||
Once you decide to tune the hyperparameter you should set "final_decision" to false.
|
||||
|
||||
{% if is_sub_enabled %}
|
||||
The user will provide you the whole code base, some logs generated during the execution of the whole workflow. Your evaluation scope includes whether the workflow code:
|
||||
1. Executes successfully, correctly organizing components and generating a final submission.
|
||||
2. Generates predictions in the correct format, ensuring they align with the **sample submission** structure!
|
||||
|
||||
|
||||
Please respond with your feedback in the following JSON format and order
|
||||
```json
|
||||
@@ -27,7 +35,9 @@ DSCoSTEER_eval:
|
||||
"execution": "Describe whether the whole code base executed successfully and generating the final submission. Include any errors or issues encountered, and retain all error messages and traceback details.",
|
||||
"return_checking": "Verify the generated files, particularly the submission file. Ensure that its format matches the sample submission",
|
||||
"code": "Provide feedback on code quality, readability, and adherence to the given specifications.",
|
||||
"final_decision": <true/false>
|
||||
"final_decision": <true/false>,
|
||||
"hyperparameter_tuning_decision": <true/false>,
|
||||
"hyperparameter_tuning_suggestion": <suggestion in plain text for hyperparameter tuning>,
|
||||
}
|
||||
```
|
||||
{% else %}
|
||||
@@ -40,7 +50,9 @@ DSCoSTEER_eval:
|
||||
"execution": "Describe whether the code executed successfully. Include any errors or issues encountered, and append all error messages and full traceback details without summarizing or omitting any information.",
|
||||
"return_checking": "Describe the expected file to be generated.",
|
||||
"code": "Provide feedback on code quality, readability, and adherence to the given specifications.",
|
||||
"final_decision": <true/false>
|
||||
"final_decision": <true/false>,
|
||||
"hyperparameter_tuning_decision": <true/false>,
|
||||
"hyperparameter_tuning_suggestion": <suggestion in plain text for hyperparameter tuning>,
|
||||
}
|
||||
```
|
||||
{% endif %}
|
||||
@@ -51,9 +63,15 @@ DSCoSTEER_eval:
|
||||
{{ code }}
|
||||
--------- the stdout of code execution and testing ---------
|
||||
{{ stdout }}
|
||||
--------- the time spent on code execution ---------
|
||||
{{ time_spent }}
|
||||
--------- the timeout of code execution ---------
|
||||
{{ timeout }}
|
||||
--------- the percent of timeout used ---------
|
||||
{{ percent_of_timeout_used }}
|
||||
|
||||
DSCoSTEER_debugger:
|
||||
system: |-
|
||||
DSCoSTEER:
|
||||
system_debugger: |-
|
||||
You are a world-class data scientist and machine learning engineer with deep expertise in statistics, mathematics, and computer science.
|
||||
You have finished the implementation of the whole workflow which has executed well on a sampled dataset. However, the user has reported that the workflow failed to execute on the full dataset.
|
||||
|
||||
@@ -65,6 +83,23 @@ DSCoSTEER_debugger:
|
||||
|
||||
Your modified code should follow the minimal changes principle. You should only modify the code that is necessary to fix the issues but not affect any other parts of the code. Try to correct as less files as possible since files are interdependent.
|
||||
|
||||
## Output Format
|
||||
{% if out_spec %}
|
||||
{{ out_spec }}
|
||||
{% else %}
|
||||
Please response the code in the following json format. Here is an example structure for the JSON output:
|
||||
{
|
||||
"code": "The Python code as a string."
|
||||
}
|
||||
{% endif %}
|
||||
system_refine: |-
|
||||
You are a world-class data scientist and machine learning engineer with deep expertise in statistics, mathematics, and computer science.
|
||||
You have finished the implementation of the whole workflow which has executed well on a sampled dataset. However, the user has reported that the hyperparameters are not reasonable and the code didn't make the best use of the time limit.
|
||||
Your current job is to refine the whole code base, try to refine the hyperparameters.
|
||||
|
||||
The user will provide your the whole code base, some feedback generated during the execution of the whole workflow and some suggestions for hyperparameter tuning.
|
||||
Your modified code should follow the minimal changes principle. Only modify the hyperparameters that is necessary.
|
||||
|
||||
## Output Format
|
||||
{% if out_spec %}
|
||||
{{ out_spec }}
|
||||
@@ -79,3 +114,7 @@ DSCoSTEER_debugger:
|
||||
{{ code }}
|
||||
--------- feedback ---------
|
||||
{{ feedback }}
|
||||
{% if hyperparameter_tuning_suggestion is not none %}
|
||||
--------- hyperparameter tuning suggestions ---------
|
||||
{{ hyperparameter_tuning_suggestion }}
|
||||
{% endif %}
|
||||
|
||||
@@ -30,8 +30,9 @@ from rdagent.log import rdagent_logger as logger
|
||||
from rdagent.scenarios.data_science.dev.feedback import DSExperiment2Feedback
|
||||
from rdagent.scenarios.data_science.dev.runner import DSCoSTEERRunner
|
||||
from rdagent.scenarios.data_science.experiment.experiment import DSExperiment
|
||||
from rdagent.scenarios.data_science.proposal.exp_gen import DSTrace
|
||||
from rdagent.scenarios.data_science.proposal.exp_gen import DSExpGen, DSTrace
|
||||
from rdagent.scenarios.data_science.proposal.exp_gen.idea_pool import DSKnowledgeBase
|
||||
from rdagent.scenarios.data_science.proposal.exp_gen.proposal import DSProposalV2ExpGen
|
||||
from rdagent.utils.workflow.misc import wait_retry
|
||||
|
||||
|
||||
@@ -80,26 +81,8 @@ class DataScienceRDLoop(RDLoop):
|
||||
skip_loop_error = (CoderError, RunnerError)
|
||||
withdraw_loop_error = (PolicyError,)
|
||||
|
||||
@staticmethod
|
||||
def _get_exp_gen(class_uri: str, scen: Scenario):
|
||||
"""
|
||||
Just for compatibility with the old version of the code.
|
||||
"""
|
||||
# TODO: remove me in the future. I don't have to be this complicated.
|
||||
# It is just for compatibility with the old version of the code and configuration.
|
||||
from rdagent.scenarios.data_science.proposal.exp_gen.proposal import (
|
||||
DSProposalV1ExpGen,
|
||||
DSProposalV2ExpGen,
|
||||
)
|
||||
|
||||
if class_uri == "rdagent.scenarios.data_science.proposal.exp_gen.DSExpGen":
|
||||
if DS_RD_SETTING.proposal_version not in ["v1", "v2"]:
|
||||
return import_class(DS_RD_SETTING.proposal_version)(scen=scen)
|
||||
if DS_RD_SETTING.proposal_version == "v1":
|
||||
return DSProposalV1ExpGen(scen=scen)
|
||||
if DS_RD_SETTING.proposal_version == "v2":
|
||||
return DSProposalV2ExpGen(scen=scen)
|
||||
return import_class(class_uri)(scen)
|
||||
# when using more advanced proposals(merged, parallel, etc.), we provide a default exp_gen for convinience.
|
||||
default_exp_gen: type[ExpGen] = DSProposalV2ExpGen
|
||||
|
||||
def __init__(self, PROP_SETTING: BasePropSetting):
|
||||
logger.log_object(PROP_SETTING.competition, tag="competition")
|
||||
@@ -115,8 +98,7 @@ class DataScienceRDLoop(RDLoop):
|
||||
|
||||
self.ckp_selector = import_class(PROP_SETTING.selector_name)()
|
||||
self.sota_exp_selector = import_class(PROP_SETTING.sota_exp_selector_name)()
|
||||
|
||||
self.exp_gen: ExpGen = self._get_exp_gen(PROP_SETTING.hypothesis_gen, scen)
|
||||
self.exp_gen: ExpGen = import_class(PROP_SETTING.hypothesis_gen)(scen)
|
||||
|
||||
# coders
|
||||
self.data_loader_coder = DataLoaderCoSTEER(scen)
|
||||
|
||||
@@ -1,3 +1,38 @@
|
||||
from rdagent.scenarios.data_science.proposal.exp_gen.base import DSTrace
|
||||
from rdagent.app.data_science.conf import DS_RD_SETTING
|
||||
from rdagent.core.proposal import ExpGen
|
||||
from rdagent.core.scenario import Scenario
|
||||
from rdagent.log import rdagent_logger as logger
|
||||
from rdagent.oai.llm_utils import APIBackend, md5_hash
|
||||
from rdagent.scenarios.data_science.experiment.experiment import DSExperiment
|
||||
from rdagent.scenarios.data_science.proposal.exp_gen.base import DSHypothesis, DSTrace
|
||||
from rdagent.scenarios.data_science.proposal.exp_gen.draft import DSDraftExpGen
|
||||
from rdagent.scenarios.data_science.proposal.exp_gen.proposal import (
|
||||
DSProposalV1ExpGen,
|
||||
DSProposalV2ExpGen,
|
||||
)
|
||||
from rdagent.scenarios.data_science.scen import DataScienceScen
|
||||
from rdagent.utils.agent.tpl import T
|
||||
|
||||
__all__ = ["DSTrace"]
|
||||
|
||||
class DSExpGen(ExpGen):
|
||||
"""
|
||||
Data Science Task Generator.
|
||||
This is a experiment router generator;
|
||||
"""
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
def gen(self, trace: DSTrace) -> DSExperiment:
|
||||
# sota_exp = trace.sota_experiment()
|
||||
|
||||
# # Draft
|
||||
# # TODO: draft here
|
||||
# if sota_exp is None:
|
||||
# pass
|
||||
|
||||
# Propose
|
||||
if DS_RD_SETTING.proposal_version == "v1":
|
||||
return DSProposalV1ExpGen(scen=self.scen).gen(trace=trace)
|
||||
if DS_RD_SETTING.proposal_version == "v2":
|
||||
return DSProposalV2ExpGen(scen=self.scen).gen(trace=trace)
|
||||
|
||||
@@ -226,7 +226,7 @@ class DSTrace(Trace[DataScienceScen, KnowledgeBase]):
|
||||
# the sota exp should be accepted decision and all required components are completed.
|
||||
if ef.decision:
|
||||
return exp, ef
|
||||
return None
|
||||
return None, None
|
||||
|
||||
def sota_experiment(
|
||||
self,
|
||||
@@ -241,11 +241,12 @@ class DSTrace(Trace[DataScienceScen, KnowledgeBase]):
|
||||
def last_successful_exp(
|
||||
self,
|
||||
search_type: Literal["all", "ancestors"] = "ancestors",
|
||||
selection: tuple[int, ...] | None = None,
|
||||
) -> DSExperiment | None:
|
||||
"""
|
||||
Access the last successful experiment even part of the components are not completed.
|
||||
"""
|
||||
search_list = self.retrieve_search_list(search_type)
|
||||
search_list = self.retrieve_search_list(search_type, selection=selection)
|
||||
|
||||
for exp, ef in search_list[::-1]:
|
||||
if ef.decision:
|
||||
|
||||
@@ -229,9 +229,7 @@ class ExpGen2TraceAndMerge(ExpGen):
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.merge_exp_gen = MergeExpGen(self.scen)
|
||||
self.exp_gen = DataScienceRDLoop._get_exp_gen(
|
||||
"rdagent.scenarios.data_science.proposal.exp_gen.DSExpGen", self.scen
|
||||
)
|
||||
self.exp_gen = DataScienceRDLoop.default_exp_gen(self.scen)
|
||||
|
||||
def gen(self, trace: DSTrace) -> DSExperiment:
|
||||
timer: RDAgentTimer = RD_Agent_TIMER_wrapper.timer
|
||||
@@ -337,17 +335,17 @@ class ExpGen2TraceAndMergeV2(ExpGen):
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.merge_exp_gen = MergeExpGen_MultiTrace(self.scen)
|
||||
self.exp_gen = DataScienceRDLoop._get_exp_gen(
|
||||
"rdagent.scenarios.data_science.proposal.exp_gen.DSExpGen", self.scen
|
||||
)
|
||||
self.exp_gen = DataScienceRDLoop.default_exp_gen(self.scen)
|
||||
self.flag_start_merge = False
|
||||
|
||||
def reset_exp_gen_version(self, version: str = "v2"):
|
||||
DS_RD_SETTING.proposal_version = version
|
||||
logger.info(f"ExpGen2TraceAndMergeV2: Resetting proposal version to {version}")
|
||||
self.exp_gen = DataScienceRDLoop._get_exp_gen(
|
||||
f"rdagent.scenarios.data_science.proposal.exp_gen.DSExpGen", self.scen
|
||||
)
|
||||
# AFAIK, this class is not used anymore (because v3 & v1 is deprecated); So we just leave a NotImplementedError instead of refine it.
|
||||
# DS_RD_SETTING.proposal_version = version
|
||||
# logger.info(f"ExpGen2TraceAndMergeV2: Resetting proposal version to {version}")
|
||||
# self.exp_gen = DataScienceRDLoop._get_exp_gen(
|
||||
# f"rdagent.scenarios.data_science.proposal.exp_gen.DSExpGen", self.scen
|
||||
# )
|
||||
raise NotImplementedError("You should not switch version with proposal_version")
|
||||
|
||||
def gen(self, trace: DSTrace, selection: tuple[int, ...] = (-1,)) -> DSExperiment:
|
||||
timer: RDAgentTimer = RD_Agent_TIMER_wrapper.timer
|
||||
@@ -402,9 +400,7 @@ class ExpGen2TraceAndMergeV3(ExpGen):
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.merge_exp_gen = ExpGen2Hypothesis(self.scen)
|
||||
self.exp_gen = DataScienceRDLoop._get_exp_gen(
|
||||
"rdagent.scenarios.data_science.proposal.exp_gen.DSExpGen", self.scen
|
||||
)
|
||||
self.exp_gen = DataScienceRDLoop.default_exp_gen(self.scen)
|
||||
|
||||
def gen(self, trace: DSTrace) -> DSExperiment:
|
||||
timer: RDAgentTimer = RD_Agent_TIMER_wrapper.timer
|
||||
|
||||
@@ -34,9 +34,7 @@ class ParallelMultiTraceExpGen(ExpGen):
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
# The underlying generator for creating a single experiment
|
||||
self.exp_gen = DataScienceRDLoop._get_exp_gen(
|
||||
"rdagent.scenarios.data_science.proposal.exp_gen.DSExpGen", self.scen
|
||||
)
|
||||
self.exp_gen = DataScienceRDLoop.default_exp_gen(self.scen)
|
||||
self.merge_exp_gen = ExpGen2Hypothesis(self.scen)
|
||||
self.trace_scheduler: TraceScheduler = RoundRobinScheduler()
|
||||
self.max_trace_num = DS_RD_SETTING.max_trace_num
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
auto_sota_selector:
|
||||
system: |-
|
||||
|
||||
You are a data scientist and a top Kaggle competitor. The user is working on improving a solution for a Kaggle competition. The user has already conducted a series of successful experiments (SOAT trails during the exploration) and collected feedbacks.
|
||||
You are a data scientist and a top Kaggle competitor. The user is working on improving a solution for a Kaggle competition. The user has already conducted a series of successful experiments (SOTA trails during the exploration) and collected feedbacks.
|
||||
|
||||
You are tasked with reviewing the list of SOTA experiments and feedbacks, and select the most promising experiment to submit.
|
||||
|
||||
|
||||
@@ -114,7 +114,7 @@ scenario_description: |-
|
||||
====== Background ======
|
||||
{{ background }}
|
||||
|
||||
{% if eda_output is not none %}
|
||||
{% if eda_output is not none %}The following is the output of the exploratory data analysis (EDA) performed on the dataset, You should carefully analyze it to better craft your feature engineering and model training strategies.
|
||||
====== Data Overview (EDA) ======
|
||||
{{ eda_output }}
|
||||
{% endif %}
|
||||
@@ -206,7 +206,6 @@ hypothesis_gen:
|
||||
- *Good Example (Efficiency)*: "To resolve the 'timeout during training' challenge, reduce `NUM_EPOCHS` from 5 to 2 and `N_SPLITS` for cross-validation from 5 to 3 in the main training loop, aiming to complete execution within the 1-hour limit while minimizing impact on the F1-score."
|
||||
- *Poor Example*: "Tune the model for better results."
|
||||
- If the hypothesis is about establishing the first solution, it should clearly outline the expected outcome -- RUNNABILITY and CORRECTNESS. Prioritize getting a valid submission out, even with a very basic model or pipeline.
|
||||
- *Good Example*: "Implement a simple RandomForest classifier with default parameters, using 5-fold cross-validation for model evaluation. This will lead to a decent baseline model that can run to completion and generate a valid submission file."
|
||||
3. **Align with Current SOTA and Identified Challenges**:
|
||||
- The hypothesis must be directly relevant to improving the *current* State-of-the-Art (SOTA) implementation or establishing a new SOTA if none exists.
|
||||
- It must directly address one of the `Identified Challenges` provided as input.
|
||||
@@ -280,7 +279,7 @@ task_gen:
|
||||
|
||||
Your primary goal is to generate a detailed, step-by-step **sketch or refinement plan** for a new data processing and modeling pipeline, specifically for the main workflow script (`main.py`), that effectively implements the `Proposed Hypothesis`. This sketch will guide a developer to write the code correctly.
|
||||
|
||||
### BACKGROUND CONTEXT: Pipeline Implementation Standards & Constraints ###
|
||||
# BACKGROUND CONTEXT: Pipeline Implementation Standards & Constraints
|
||||
|
||||
The `main.py` sketch you generate should lead to a pipeline implementation that adheres to the following standards. These are guiding principles for the final *outcome* of your sketch:
|
||||
|
||||
@@ -309,15 +308,13 @@ task_gen:
|
||||
- Prevent data leakage from test/validation sets into any training stage.
|
||||
7. **Resource Utilization**: Leverage GPU and multiprocessing where appropriate and beneficial, if consistent with the hypothesis and efficiency goals.
|
||||
8. **Metric Calculation and Storage (`scores.csv`)**:
|
||||
- Calculate the official competition metric on a proper validation set (e.g., K-fold CV, typically 3-5 folds unless efficiency dictates fewer). Save results to `scores.csv`.
|
||||
- Calculate the official competition metric on a proper validation set. Save results to `scores.csv`.
|
||||
- The sketch must ensure this step is included. A successful run should always produce scores.
|
||||
- `scores.csv` must have an index with model names and the literal string "ensemble" (lowercase). Columns should be "Model" (the name of the model or the ensemble strategy), and the exact metric name (e.g., "AUC").
|
||||
- `scores.csv` must have an index with model names and the literal string "ensemble" (lowercase). Columns should be the exact metric name (e.g., "AUC").
|
||||
- When only one model is used, its score should be present, and an "ensemble" score (which would be the same as the single model's score in this case) must also be recorded.
|
||||
- Ensure validation metrics and processes are consistent across all parts of the pipeline. Avoid changes that would alter how validation metrics are calculated unless that is part of the hypothesis.
|
||||
9. **Submission File (`submission.csv`)**: Generate `submission.csv` in the **exact format** required (column names, order, data types), as detailed by `sample_submission.csv` in the `Competition Scenario Description`. This is a critical step.
|
||||
|
||||
### END OF BACKGROUND CONTEXT ###
|
||||
|
||||
# Guidelines for Sketching the `main.py` Workflow
|
||||
|
||||
YOUR TASK IS TO create a conceptual sketch for drafting or updating the `main.py` workflow. This is a plan, not code.
|
||||
@@ -353,7 +350,13 @@ task_gen:
|
||||
- Handle file I/O robustly, especially for zipped data or large files, to prevent `FileNotFoundError` or `BadZipFile` issues.
|
||||
- Confirm no `tqdm` or other progress bars are in the final script.
|
||||
- Double-check that validation scores are saved correctly to `scores.csv` with specified 'Model' and metric columns, even for a single model run (include 'ensemble' row).
|
||||
|
||||
8. **EDA improvement**: The user might provide you some EDA improvement suggestions based on the previous EDA output. If so, you should also include the EDA improvement in your sketch.
|
||||
|
||||
# Hyperparameters Specification
|
||||
Follow the hyperparameters specification below when approaching hyperparameter selection.
|
||||
If you are confident in a specific value based on strong evidence, prior experiments, or clear rationale, specify the value clearly.
|
||||
{% include "scenarios.data_science.share:spec.hyperparameter" %}
|
||||
|
||||
{% if task_output_format is not none %}
|
||||
## [Partial Response Format 1] Task Output Format:
|
||||
{{ task_output_format }}
|
||||
@@ -392,8 +395,11 @@ task_gen:
|
||||
|
||||
{% endfor %}
|
||||
# Previous Failed Experiments & Feedback (e.g., experiments that did not pass evaluation, encountered bugs, or failed to surpass SOTA performance)
|
||||
|
||||
{{ failed_exp_and_feedback_list_desc }}
|
||||
|
||||
{% if eda_improvement is not none %}
|
||||
{{ eda_improvement }}
|
||||
{% endif %}
|
||||
|
||||
idea_sample:
|
||||
system: |-
|
||||
|
||||
@@ -1,283 +0,0 @@
|
||||
scenario_problem:
|
||||
system: |-
|
||||
{% include "scenarios.data_science.share:scen.role" %}
|
||||
The user is improving a Kaggle competition implementation iteratively. Each new iteration (trace) is typically a modification of the current overall State-of-the-Art (SOTA) solution. If a new trace's performance surpasses the current SOTA, it establishes a new SOTA. Otherwise, it is considered a failed experiment.
|
||||
|
||||
You will be provided with:
|
||||
1. A detailed competition scenario description;
|
||||
2. A history of previous SOTA experiments and their associated feedbacks, typically indexed or ordered from oldest to newest;
|
||||
3. A history of previous failed experiments and their associated feedbacks, chronologically ordered, where each failed experiment did not surpass the SOTA that was current at the time of its execution;
|
||||
|
||||
Your task is to analyze the provided information (primarily the scenario and current SOTA, if available) and identify a concise list of **Key Challenges** or **Core Problems** relevant to achieving success in this competition and improving the target metric. Aim for **FEWER BUT BETTER** challenges (e.g., 2-3 critical challenges), focusing on the most impactful aspects that can be methodically addressed.
|
||||
|
||||
### Core Analysis Dimensions for Identifying Challenges
|
||||
1. **SOTA Alignment Analysis**: (If SOTA is provided) Systematically compare the current SOTA implementation against dataset properties and domain knowledge to identify discrepancies or areas representing core challenges to overcome for enhancement.
|
||||
2. **Gap Identification**: (If successful past solutions or common winning strategies are known/inferred) Examine what implicitly addressed problems or unexploited avenues these successful approaches highlight. These gaps can represent current challenges.
|
||||
3. **Domain-Implementation Coherence Check**: Identify instances where technical approaches might violate domain constraints, oversimplify complex relationships, or miss domain-specific nuances. These incoherencies are challenges.
|
||||
4. **Scenario-First Focus (No SOTA)**: If no SOTA implementation is available, the **primary identified challenge** should be foundational. It should focus on establishing a **simple, robust, and efficient baseline** that directly addresses the core task and evaluation metric. Avoid overly complex initial challenges.
|
||||
|
||||
## Key Challenges / Core Problems
|
||||
You **MUST** categorize each identified challenge into one of the following two types. This categorization should be based on the primary driver or nature of the challenge:
|
||||
1. **Dataset-Driven Challenge**: Challenges primarily derived from addressing or leveraging inherent structural or statistical properties of the dataset (e.g., mitigating imbalance, managing high dimensionality, specific feature engineering needs for data types like text or time-series, handling missing data, transforming skewed distributions, accounting for collinearity or outliers).
|
||||
2. **Domain-Informed Challenge**: Challenges primarily derived from correctly applying actionable knowledge specific to the competition's domain. This includes the correct interpretation of data patterns based on domain context, domain-specific feature engineering, adhering to known domain constraints, or avoiding invalid assumptions that data analysis alone might not reveal.
|
||||
|
||||
### Specification for each Identified Challenge
|
||||
1. The challenge should be specific and fine-grained. Avoid general or vague statements.
|
||||
2. The challenge should be technical or methodological. Focus on design and implementation strategies that need to be solved, not simple runtime bugs (unless the bug points to a deeper architectural challenge or a persistent efficiency problem).
|
||||
3. The challenge must be strictly aligned with the improvement of the target metric.
|
||||
4. If no SOTA is available, at least one identified challenge must guide the creation of the simplest possible, yet potentially competitive, baseline model that can run to completion.
|
||||
|
||||
user: |-
|
||||
# Scenario Description
|
||||
{{ scenario_desc }}
|
||||
|
||||
# Current SOTA Implementation
|
||||
{{ sota_exp_desc }}
|
||||
|
||||
feedback_problem:
|
||||
system: |-
|
||||
{% include "scenarios.data_science.share:scen.role" %}
|
||||
The user is improving a Kaggle competition implementation iteratively through traces. Each new trace is a modification of the State-of-the-Art (SOTA) implementation that was current at the time that trace was initiated. If a new trace's performance surpasses the SOTA it aimed to improve upon, it becomes the new SOTA. If not, it is considered a failed experiment.
|
||||
|
||||
You will be provided with:
|
||||
1. A detailed competition scenario description;
|
||||
2. A history of previous SOTA experiments and their associated feedbacks, typically indexed or ordered from oldest to newest;
|
||||
3. A history of previous failed experiments and their associated feedbacks, chronologically ordered, where each failed experiment did not surpass the SOTA that was current at the time of its execution;
|
||||
4. The overall current SOTA implementation and its associated feedback, which represents the best-performing experiment from the entire history provided up to this point.
|
||||
|
||||
Your task is to analyze all this provided historical information and extract **Key Learnings and Unresolved Challenges** from the experiment history. These should guide concrete improvements in subsequent iterations.
|
||||
|
||||
## Key Learnings and Unresolved Challenges
|
||||
### Definition
|
||||
Key Learnings and Unresolved Challenges are specific, fine-grained technical or methodological observations, persistent issues, or patterns identified within previous experiments or the current SOTA implementation. These are primarily derived from explicit feedback, code analysis, or patterns in the trace history, and should highlight problems that need solving or learnings that should inform future hypotheses.
|
||||
|
||||
### Guidelines for Identification
|
||||
Here are guidelines to help you identify these Learnings and Challenges:
|
||||
|
||||
1. **Feedback Analysis**:
|
||||
- **Explicit Issues/Suggestions as Challenges**: Extract critical issues, errors (especially those pointing to deeper problems like resource limits or incorrect submission formats if not easily fixed), or direct suggestions from feedback that represent unresolved problems.
|
||||
- **Implicit Gaps as Challenges**: Infer unaddressed points, shortcomings, or areas for improvement implied by feedback that constitute ongoing challenges.
|
||||
- **Time/Memory Constraints as Critical Challenges**: If previous experiments indicate failures due to time/memory limitations, or inefficient resource usage, this **MUST** be listed as a critical challenge. This includes identifying if the current SOTA or failed experiments are too complex for the given time limits.
|
||||
|
||||
2. **Implementation Review (of SOTA or relevant past experiments)**:
|
||||
- **Suboptimal Design as Challenges**: Identify potentially suboptimal feature selection, model architecture, hyperparameters, ensemble strategy, training/validation processes that appear as recurring problems or limit performance, framing them as challenges to be addressed.
|
||||
- **Common Implementation Issues**: Note the coding issues that are blocking for receiving a reasonable result. For example, the submission format was repeatedly incorrect despite attempts to fix it, this is an unresolved challenge related to the implementation.
|
||||
|
||||
3. **Trace History Analysis (Trends & Patterns as Challenges)**:
|
||||
- **Persistent Issues/Errors as Challenges**: Flag unresolved negative patterns, errors (e.g., recurrent `zipfile.BadZipFile`, CUDA label errors, submission format mismatches if they persist after attempts to fix), or suboptimal outcomes that recur across multiple experiment traces. These represent core unresolved challenges.
|
||||
- **Ineffective/Partial Fixes**: Highlight if previous changes intended to solve a problem were only partially successful or ineffective, meaning the core challenge remains.
|
||||
- **Unexplored Promising Directions**: Identify potentially valuable approaches (e.g., alternative feature sets, different model families, advanced optimization techniques) that were hinted at by feedback, briefly tried without full exploration, or represent logical next steps given the trajectory of past experiments.
|
||||
- **Constraint Violations/Inefficiencies as Challenges**: Explicitly note any unaddressed time or memory constraint violations or significant computational inefficiencies as critical challenges that need strategic solutions.
|
||||
|
||||
### Specification for each Learning/Challenge
|
||||
1. The Learning/Challenge must be specific, actionable, and evidence-based (tied to feedback, code, or trace history).
|
||||
2. It should focus on technical or methodological problems that need solving.
|
||||
3. Clearly state the learning or articulate the challenge.
|
||||
4. Addressing the challenge or applying the learning should have a plausible positive impact on the target metric or successful execution.
|
||||
5. The challenge must be strictly aligned with the improvement of the target metric.
|
||||
|
||||
user: |-
|
||||
# Scenario Description
|
||||
{{ scenario_desc }}
|
||||
|
||||
# Previous Experiments and Feedbacks
|
||||
{{ exp_and_feedback_list_desc }}
|
||||
|
||||
# Current SOTA Implementation
|
||||
{{ sota_exp_desc }}
|
||||
|
||||
scenario_description: |-
|
||||
{% include "scenarios.data_science.proposal.exp_gen.prompts_v2:scenario_description" %}
|
||||
|
||||
hypothesis_gen:
|
||||
system: |-
|
||||
{% include "scenarios.data_science.share:scen.role" %}
|
||||
The user is iteratively improving a Kaggle competition implementation. Each new iteration (trace) is a modification of the current State-of-the-Art (SOTA). If a new trace surpasses the current SOTA, it becomes the new SOTA. Otherwise, it's a failed experiment.
|
||||
You will be provided with:
|
||||
1. A detailed competition scenario description.
|
||||
2. Previous SOTA experiments and feedback (chronologically ordered, oldest to newest).
|
||||
3. Previous failed experiments and feedback (ordered attempts that did not improve SOTA).
|
||||
4. The current SOTA implementation and feedback (the latest successful experiment).
|
||||
5. A list of identified **Challenges** from history), which we will refer to as "Identified Challenges" below.
|
||||
|
||||
Your task is to perform two main steps:
|
||||
1. **Hypothesis Proposal**: For each relevant Identified Challenge, propose one specific, testable hypothesis.
|
||||
2. **Hypothesis Evaluation**: Evaluate each proposed hypothesis across multiple dimensions.
|
||||
|
||||
{% if enable_idea_pool %}
|
||||
To help you propose hypotheses, the user may provide a list of ideas for each Identified Challenge. These ideas are methods or techniques from successful SOTA implementations in other competitions.
|
||||
Evaluate these ideas: they might help address the Identified Challenges and improve the current SOTA. You must decide whether to use them. If you adapt a provided idea for a specific Challenge into your hypothesis, ensure you clearly state this by setting the 'inspired' flag to True for that hypothesis.
|
||||
{% endif %}
|
||||
|
||||
# Task 1: Hypothesis Proposal
|
||||
First note that the user might provide a list of challenges containing duplicates. You should only propose one hypothesis for each unique challenge. If a challenge is a duplicate of a previous one, you can skip it.
|
||||
For each Identified Challenge, propose one hypothesis corresponding to the Challenge, aimed at improving the current SOTA implementation or establishing a robust initial SOTA.
|
||||
|
||||
## 1.1. Steps to Hypothesize
|
||||
Follow these steps to formulate effective hypotheses:
|
||||
|
||||
1. **Understanding the Challenge**:
|
||||
- Analyze the Identified Challenge to understand its root cause and potential impact on the competition's target metric or successful execution.
|
||||
- If the Challenge stems from past experiments (SOTA or failed), review the specifics of those experiments to ensure the proposed hypothesis offers a novel, more effective, or correctly implemented solution.
|
||||
- If the Challenge relates to persistent problems from failed experiments (e.g., experiments consistently failed due to time/memory constraints, or recurrent errors like incorrect data loading or submission formats), your hypothesis MUST propose a direct and robust tentative solution.
|
||||
2. **Drafting the First Implementation (if no SOTA exists)**:
|
||||
- If there is no SOTA implementation yet (i.e., you are drafting the first implementation based on a foundational Challenge identified in the previous step), your primary hypothesis **MUST** focus on a plan creating the **simplest possible, yet potentially competitive, baseline model** that directly addresses this foundational Challenge and can run to completion reliably.
|
||||
- This initial hypothesis should define the core data processing, feature engineering, model choice, and submission generation steps in their most straightforward form. Avoid proposing complex, multi-faceted solutions or combining multiple distinct techniques for the very first run.
|
||||
3. **Actionable Changes**:
|
||||
- If a Challenge relates to **time/memory constraints**, the hypothesis must propose concrete changes (e.g., simpler model architecture, reduced number of cross-validation folds or training epochs, data sub-sampling, more efficient data loading, optimized code). The aim is to create a solution that can run to completion reliably. While performance is key, a non-completing experiment has zero performance. Strive for the best possible performance *within* the operational constraints.
|
||||
- If a Challenge involves underperforming models (e.g., in an ensemble), propose specific actions like removing or replacing those models.
|
||||
- If a Challenge relates to hyperparameter tuning, recommend a specific method or strategy (e.g., "Implement Bayesian optimization for RandomForest's learning_rate and num_leaves to address the 'suboptimal hyperparameter' challenge").
|
||||
- If a Challenge points to data loading, preprocessing, or submission format errors, the hypothesis must detail the exact changes required to rectify these issues.
|
||||
{% if enable_idea_pool %}
|
||||
4. **Idea Reference**: Provided ideas are methods, techniques, or tricks from high-performing implementations in other competitions addressing similar problems. Use them as inspiration if you find them suitable for the current Challenge.
|
||||
{% endif %}
|
||||
|
||||
## 1.2. Guidelines for Writing Hypotheses
|
||||
|
||||
1. **Be Specific and Decisive**:
|
||||
- Clearly state the exact, unambiguous change(s) being proposed. Avoid vague goals like "improve the model" or "optimize the pipeline."
|
||||
- The hypothesis must propose a single, clear course of action. Do not suggest alternatives (e.g., "try method A or method B").
|
||||
- The hypothesis statement must be direct and definitive, without phrases like "for example," "e.g.," or "might involve."
|
||||
- The hypothesis must be more informative and decisive than the Challenge it addresses. It should not simply restate the Challenge or suggest a general approach without specifics.
|
||||
2. **Ensure Testability and Actionability**:
|
||||
- The hypothesis must describe an action or change that can be practically implemented and tested.
|
||||
- If the hypothesis is about improving SOTA, it should clearly state the expected improvement, typically related to a measurable performance metric or successful execution.
|
||||
- *Good Example (Optimizer)*: "Changing the optimizer from Adam to AdamW and applying a learning rate of 1e-4 for the BERT-based text classifier will increase F1-score by at least 2% on the validation set.""
|
||||
- *Good Example (Format)*: "To address the 'incorrect submission format' challenge, modify the submission generation step to create a CSV file with columns 'Id' and 'Category', mapping predicted class indices to original category labels, which is expected to pass submission validation."
|
||||
- *Good Example (Efficiency)*: "To resolve the 'timeout during training' challenge, reduce `NUM_EPOCHS` from 5 to 2 and `N_SPLITS` for cross-validation from 5 to 3 in the main training loop, aiming to complete execution within the 1-hour limit while minimizing impact on the F1-score."
|
||||
- *Poor Example*: "Tune the model for better results."
|
||||
- If the hypothesis is about establishing the first solution, it should clearly outline the expected outcome -- RUNNABILITY and CORRECTNESS. Prioritize getting a valid submission out, even with a very basic model or pipeline.
|
||||
- *Good Example*: "Implement a simple RandomForest classifier with default parameters, using 5-fold cross-validation for model evaluation. This will lead to a decent baseline model that can run to completion and generate a valid submission file."
|
||||
3. **Align with Current SOTA and Identified Challenges**:
|
||||
- The hypothesis must be directly relevant to improving the *current* State-of-the-Art (SOTA) implementation or establishing a new SOTA if none exists.
|
||||
- It must directly address one of the `Identified Challenges` provided as input.
|
||||
4. **Maintain Singular Focus within Hypothesis**:
|
||||
- If a hypothesis involves multiple adjustments, these must be tightly correlated and contribute to a single, unified conceptual change addressing the core of the Identified Challenge.
|
||||
- Avoid bundling multiple independent or unrelated ideas into a single hypothesis. Each hypothesis should test one core concept.
|
||||
5. **Address the Overall Pipeline (for Pipeline-Focused Tasks)**:
|
||||
- The hypothesis should address improvements to the end-to-end pipeline.
|
||||
- It can propose coordinated changes across multiple parts of the SOTA implementation if these are necessary to achieve a significant pipeline-level improvement to address the Challenge. (Note: Even for pipeline-focused hypotheses, you will still select the single *most relevant* primary component tag during the evaluation task.)
|
||||
|
||||
# Task 2: Hypothesis Evaluation
|
||||
After proposing one hypothesis for each relevant Identified Challenge, evaluate each one.
|
||||
|
||||
## 2.1. Evaluation Instruction
|
||||
For each individual hypothesis you proposed in Task 1, perform the following two evaluation steps:
|
||||
|
||||
1. **Assign a Component Tag:** Assign a single component tag to the hypothesis. Choose the **single most relevant** tag from the official list below, even if the hypothesis appears to touch upon multiple areas. Use the following detailed descriptions to understand the scope and boundaries of each component.
|
||||
|
||||
- **`DataLoadSpec`**: Responsible for loading raw competition data, ensuring data is converted to the correct types, and potentially providing an initial exploratory data analysis (EDA) summary. (e.g., fixing `zipfile.BadZipFile` by improving loading logic).
|
||||
- **`FeatureEng`**: Focuses on transforming raw data into meaningful features suitable for model consumption. Key responsibilities include maintaining data shape consistency, preventing data leakage during feature creation, and optimizing features for model performance. Feature engineering should be model-agnostic.
|
||||
- **`Model`**: Involves model building (developing new models to address the problem), model tuning (optimizing existing models for better performance), or model removal. This component also handles data operations or augmentations closely tied to a specific model framework (e.g., PyTorch `Datasets` & `DataLoaders`, TensorFlow `tf.data`, or fixing CUDA label errors by ensuring correct label mapping before loss calculation).
|
||||
- **`Ensemble`**: Combines predictions from multiple models using various ensemble strategies.
|
||||
- **`Workflow`**: Integrates all pipeline components, orchestrating the flow from data loading through to final output generation (e.g., correcting `submission.csv` column names or structure, managing overall pipeline execution logic for efficiency).
|
||||
|
||||
2. **Score the Hypothesis:** For each hypothesis, provide a score from 1 (lowest/worst) to 10 (highest/best) on each of the following five dimensions. Base your scores on all provided information.
|
||||
|
||||
- **Challenge-Hypothesis Alignment (Score: 1-10):** How directly and effectively does the hypothesis address the core issues of the `Identified Challenge` it targets? A higher score means a stronger, more direct alignment.
|
||||
- **Expected Impact (Score: 1-10):** What is the estimated magnitude of improvement (e.g., in the primary competition metric, efficiency, robustness, or successful execution) if this hypothesis is successfully implemented? Higher scores for greater positive impact.
|
||||
- **Novelty (Score: 1-10):** How innovative or original is this hypothesis when compared to the approaches and ideas evident in the `previous SOTA experiments` and `previous failed experiments`? Assign a score of 1 if the hypothesis is a repeat or substantially similar to a previously attempted hypothesis (whether successful or failed), UNLESS the previous attempt clearly failed due to a trivial implementation bug and the current hypothesis proposes the correct implementation of the same core idea.
|
||||
- **Feasibility (Score: 1-10):** How easily and practically can this hypothesis be implemented and *run to completion* within the existing SOTA codebase and operational constraints (e.g., allowed time for training/inference, available compute resources, overall complexity)? Higher scores for easier implementation and higher likelihood of successful execution.
|
||||
- **Risk-Reward Balance (Score: 1-10):** Considering the potential for significant improvement (reward) versus the probability of failure, negative side-effects, or excessive resource consumption (risk), how optimal is this balance? A high score indicates a favorable balance.
|
||||
- **Prioritization for Critical Challenges:** If a hypothesis directly and credibly addresses a **critical Challenge that caused prior experiment failures** (e.g., timeout, persistent data loading errors, incorrect submission format preventing any score), its **Expected Impact** and **Risk-Reward Balance** should generally be scored highly (e.g., 8-10), and **Feasibility** should also be high if the proposed solution is indeed simpler, more direct, or more efficient. This ensures such critical hypotheses are prioritized.
|
||||
|
||||
user: |-
|
||||
# Scenario Description
|
||||
{{ scenario_desc }}
|
||||
|
||||
# Previous Experiments and Feedbacks
|
||||
{{ exp_and_feedback_list_desc }}
|
||||
|
||||
# Current SOTA Implementation
|
||||
{{ sota_exp_desc }}
|
||||
|
||||
# Identified Challenges{% if enable_idea_pool %} with Sampled Ideas{% endif %}
|
||||
{{ problems }}
|
||||
|
||||
task_gen:
|
||||
system: |-
|
||||
{% include "scenarios.data_science.share:scen.role" %}
|
||||
The user is iteratively developing a Kaggle competition solution. Each new iteration aims to improve upon the current State-of-the-Art (SOTA) implementation by applying a specific hypothesis that addresses an identified challenge. The new trace is based on the current SOTA; the SOTA itself evolves.
|
||||
|
||||
You will be provided with the following inputs:
|
||||
1. **Competition Scenario Description**: Details about the competition (task type, data, evaluation metric, time limits, etc.).
|
||||
2. **Previous SOTA Experiments & Feedback**: (If available) A history of successful implementations, ordered chronologically.
|
||||
3. **Previous Failed Experiments & Feedback**: (If available) A history of unsuccessful attempts, which are crucial for learning.
|
||||
4. **Current SOTA Implementation & Feedback**: (If available) Details of the best-performing solution so far. **If no SOTA implementation is provided, your primary task is to sketch the initial, simplest possible, end-to-end `main.py` workflow.**
|
||||
5. **Proposed Hypothesis**: One, or more specific hypotheses aimed at improving the current SOTA or forming the basis of an initial SOTA. This hypothesis directly addresses an "Identified Challenge" from a previous analysis step.
|
||||
|
||||
Your primary goal is to generate a detailed, step-by-step **sketch or refinement plan** for a new data processing and modeling pipeline, specifically for the main workflow script (`main.py`), that effectively implements the `Proposed Hypothesis`. This sketch will guide a developer to write the code correctly.
|
||||
|
||||
### BACKGROUND CONTEXT: Pipeline Implementation Standards & Constraints ###
|
||||
|
||||
The `main.py` sketch you generate should lead to a pipeline implementation that adheres to the following standards. These are guiding principles for the final *outcome* of your sketch:
|
||||
|
||||
1. **Program Execution**: The resulting `main.py` script must be executable via `python main.py` without command-line parameters. Configurations should be hardcoded for simplicity.
|
||||
2. **File Handling**:
|
||||
- Implement robust handling of file encodings and delimiters.
|
||||
- Input files are under `{% include "scenarios.data_science.share:scen.input_path" %}`. The sketch must detail how they are loaded and, if multiple, combined or processed.
|
||||
- Test indices must be determined from a dedicated test index file (if available) or by the order in the test data file. **Crucially, DO NOT use the sample submission file to infer test indices or the number of test samples.**
|
||||
- Ensure actual data (not just filenames) is loaded during the data loading phase.
|
||||
- If data is in zip files, the sketch should advise on robust loading, e.g., pre-extraction or careful handling if using multiprocessing in data loaders.
|
||||
3. **Data Preprocessing**:
|
||||
- Convert data to correct types (numeric, categorical, parse dates).
|
||||
- Optimize memory usage (e.g., downcasting, chunk processing if essential and the hypothesis supports it).
|
||||
- Implement domain-specific preprocessing relevant to the hypothesis (e.g., text tokenization, image resizing/augmentation).
|
||||
4. **Code Standards**:
|
||||
- The pipeline must **NOT** use progress bars (e.g., `tqdm`) in the submission code.
|
||||
- Reiterate: **DO NOT** use the sample submission file to extract test indices or any other information beyond the required column names and format for the output file.
|
||||
- Ensure no features are inadvertently excluded during processing.
|
||||
5. **Preferred Technologies & Methodological Notes**:
|
||||
- NN models: Prefer PyTorch (over TensorFlow) if no SOTA or hypothesis dictates otherwise. Prioritize fine-tuning pre-trained models.
|
||||
- Decision Tree models: Prefer XGBoost or RandomForest over LightGBM unless SOTA or hypothesis dictates otherwise.
|
||||
6. **General Data Science Considerations**:
|
||||
- Design for scalability.
|
||||
- Handle missing values and outliers appropriately as guided by the hypothesis or SOTA.
|
||||
- Ensure consistency between feature data types and any transformations applied.
|
||||
- Prevent data leakage from test/validation sets into any training stage.
|
||||
7. **Resource Utilization**: Leverage GPU and multiprocessing where appropriate and beneficial, if consistent with the hypothesis and efficiency goals.
|
||||
8. **Metric Calculation and Storage (`scores.csv`)**:
|
||||
- Calculate the official competition metric on a proper validation set (e.g., K-fold CV, typically 3-5 folds unless efficiency dictates fewer). Save results to `scores.csv`.
|
||||
- The sketch must ensure this step is included. A successful run should always produce scores.
|
||||
- `scores.csv` must have an index with model names and the literal string "ensemble" (lowercase). Columns should be "Model" (the name of the model or the ensemble strategy), and the exact metric name (e.g., "AUC").
|
||||
- When only one model is used, its score should be present, and an "ensemble" score (which would be the same as the single model's score in this case) must also be recorded.
|
||||
- Ensure validation metrics and processes are consistent across all parts of the pipeline. Avoid changes that would alter how validation metrics are calculated unless that is part of the hypothesis.
|
||||
9. **Submission File (`submission.csv`)**: Generate `submission.csv` in the **exact format** required (column names, order, data types), as detailed by `sample_submission.csv` in the `Competition Scenario Description`. This is a critical step.
|
||||
|
||||
### END OF BACKGROUND CONTEXT ###
|
||||
|
||||
# Guidelines for Sketching the `main.py` Workflow
|
||||
|
||||
YOUR TASK IS TO create a conceptual sketch for drafting or updating the `main.py` workflow. This is a plan, not code.
|
||||
|
||||
1. **No Code**: The sketch **MUST NOT** contain any programming code, specific library calls, or pseudo-code. Describe steps conceptually (e.g., "Load training data from {% include "scenarios.data_science.share:scen.input_path" %}/train.csv"). List specific algorithm names where appropriate (e.g., "Apply XGBoost classifier," "Use Isotonic Regression for calibration").
|
||||
2. **Structure and Conciseness**:
|
||||
- If SOTA exists, understand its structure first.
|
||||
- If no SOTA, outline a clear, logical sequence of steps for the new `main.py`.
|
||||
3. **Leverage SOTA or Design a New One**:
|
||||
- **If a `Current SOTA Implementation` is provided**: Your sketch must primarily detail the **minimal and targeted changes, additions, or replacements** needed to integrate the `Proposed Hypothesis` into that SOTA. Focus only on what needs to change.
|
||||
- **If NO `Current SOTA Implementation` is provided (Initial Version)**: This is critical. Your sketch **MUST** describe a **COMPLETE, END-TO-END, YET SIMPLEST POSSIBLE baseline pipeline**.
|
||||
- It must cover: Data loading (from specified paths), essential preprocessing (as per hypothesis or minimal viable), a basic model implementation (as per hypothesis), a simple validation strategy (e.g., a single train-validation split or fewer folds if CV is too complex initially), generation of `scores.csv`, and `submission.csv` in the correct format.
|
||||
- The overriding goal for this initial sketch is **RUNNABILITY and CORRECTNESS of the pipeline structure**. Prioritize getting a valid submission out, even with a very basic model. Avoid any complexity not absolutely mandated by the core hypothesis or competition basics.
|
||||
4. **Learn from Past Failures**:
|
||||
- If `Previous Failed Experiments & Feedback` are provided, analyze them meticulously. Design the sketch to explicitly avoid repeating similar mistakes, especially if failures relate to the current hypothesis, data handling, submission format, or resource usage (timeouts).
|
||||
- If a hypothesis aims to fix a past failure, the sketch should detail precisely how the fix is implemented.
|
||||
5. **Specificity and Clarity**:
|
||||
- Be unambiguous. Instead of "select model," if the hypothesis implies "Train an EfficientNet-B0 model," state that.
|
||||
- The sketch must be definitive. No open-ended options or phrases like "for example," or "e.g.," within a step's action.
|
||||
6. **Resource Constraints & Efficiency**:
|
||||
- Always design the workflow to execute within the competition `Time Limit`.
|
||||
- If `Previous Failed Experiments` explicitly state time/memory constraint issues, your sketch **MUST** make efficiency the **TOP PRIORITY**. Clearly state `[EFFICIENCY AS TOP PRIORITY]` at the beginning of your sketch.
|
||||
- The sketch must then detail *specific measures* to achieve this (e.g., "Reduce CV folds to 2," "Limit training to 3 epochs," "Use a smaller pre-trained model like MobileNetV2," "Subsample training data to 50% if full dataset causes timeout").
|
||||
- Even if the `Proposed Hypothesis` is not about efficiency, if past experiments failed due to timeouts or the dataset/model is complex, the sketch **must still incorporate measures to improve overall pipeline efficiency**. This might involve simplifying aspects unrelated to the core hypothesis (e.g., reducing image resolution, simpler feature engineering) to ensure the hypothesis can be tested within limits.
|
||||
- The goal is a workflow that successfully implements and validates the `Proposed Hypothesis` effectively, balancing performance with strict resource constraints. An experiment that times out provides no information.
|
||||
7. **Reminders of Common Mistakes (Especially for New `main.py`)**: At the end of your sketch, include a "Key Reminders for Developer" section. Add the following reminders if appropriate.
|
||||
- Ensure all input files are loaded from their exact paths under `{% include "scenarios.data_science.share:scen.input_path" %}` (e.g., `{% include "scenarios.data_science.share:scen.input_path" %}<competition_name>/train.csv`)."
|
||||
- Verify `submission.csv` strictly adheres to format: columns, correct data types, and no extra index.
|
||||
- "Implement correct label mapping for classification tasks (e.g., 0-indexed, contiguous integers for loss functions like PyTorch's CrossEntropyLoss) to prevent runtime errors."
|
||||
- Handle file I/O robustly, especially for zipped data or large files, to prevent `FileNotFoundError` or `BadZipFile` issues.
|
||||
- Confirm no `tqdm` or other progress bars are in the final script.
|
||||
- Double-check that validation scores are saved correctly to `scores.csv` with specified 'Model' and metric columns, even for a single model run (include 'ensemble' row).
|
||||
|
||||
user: |-
|
||||
{% include "scenarios.data_science.proposal.exp_gen.prompts_v2:task_gen.user" %}
|
||||
@@ -16,10 +16,12 @@ from rdagent.core.proposal import ExpGen
|
||||
from rdagent.core.scenario import Scenario
|
||||
from rdagent.log import rdagent_logger as logger
|
||||
from rdagent.oai.llm_utils import APIBackend, md5_hash
|
||||
from rdagent.scenarios.data_science.dev.feedback import ExperimentFeedback
|
||||
from rdagent.scenarios.data_science.experiment.experiment import DSExperiment
|
||||
from rdagent.scenarios.data_science.proposal.exp_gen.base import DSHypothesis, DSTrace
|
||||
from rdagent.scenarios.data_science.proposal.exp_gen.draft import DSDraftExpGen
|
||||
from rdagent.scenarios.data_science.proposal.exp_gen.idea_pool import DSIdea
|
||||
from rdagent.scenarios.data_science.proposal.exp_gen.refine import DSRefineExpGen
|
||||
from rdagent.utils.agent.tpl import T
|
||||
from rdagent.utils.repo.diff import generate_diff_from_dict
|
||||
from rdagent.utils.workflow import wait_retry
|
||||
@@ -719,6 +721,7 @@ class DSProposalV2ExpGen(ExpGen):
|
||||
hypotheses: list[DSHypothesis],
|
||||
pipeline: bool,
|
||||
failed_exp_feedback_list_desc: str,
|
||||
sota_exp_fb: ExperimentFeedback | None = None,
|
||||
) -> DSExperiment:
|
||||
if pipeline:
|
||||
component_info = get_component("Pipeline")
|
||||
@@ -737,6 +740,7 @@ class DSProposalV2ExpGen(ExpGen):
|
||||
sota_exp_desc=sota_exp_desc,
|
||||
hypotheses=hypotheses,
|
||||
failed_exp_and_feedback_list_desc=failed_exp_feedback_list_desc,
|
||||
eda_improvement=sota_exp_fb.eda_improvement if sota_exp_fb else None,
|
||||
)
|
||||
response = APIBackend().build_messages_and_create_chat_completion(
|
||||
user_prompt=user_prompt,
|
||||
@@ -806,7 +810,6 @@ class DSProposalV2ExpGen(ExpGen):
|
||||
self,
|
||||
trace: DSTrace,
|
||||
) -> DSExperiment:
|
||||
|
||||
pipeline = DS_RD_SETTING.coder_on_whole_pipeline
|
||||
if not pipeline and (draft_exp := draft_exp_in_decomposition(self.scen, trace)):
|
||||
return draft_exp
|
||||
@@ -821,7 +824,7 @@ class DSProposalV2ExpGen(ExpGen):
|
||||
]
|
||||
)
|
||||
|
||||
sota_exp = trace.sota_experiment()
|
||||
sota_exp, sota_exp_fb = trace.sota_experiment_fb()
|
||||
if not isinstance(sota_exp, DSExperiment):
|
||||
eda_output = None
|
||||
else:
|
||||
@@ -919,4 +922,5 @@ class DSProposalV2ExpGen(ExpGen):
|
||||
),
|
||||
pipeline=pipeline,
|
||||
failed_exp_feedback_list_desc=failed_exp_feedback_list_desc,
|
||||
sota_exp_fb=sota_exp_fb,
|
||||
)
|
||||
|
||||
@@ -0,0 +1,105 @@
|
||||
import json
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
from rdagent.app.data_science.conf import DS_RD_SETTING
|
||||
from rdagent.core.proposal import ExpGen
|
||||
from rdagent.log import rdagent_logger as logger
|
||||
from rdagent.oai.llm_utils import APIBackend
|
||||
from rdagent.scenarios.data_science.experiment.experiment import DSExperiment
|
||||
from rdagent.scenarios.data_science.proposal.exp_gen.base import DSHypothesis, DSTrace
|
||||
from rdagent.scenarios.data_science.proposal.exp_gen.utils import (
|
||||
CodingSketch,
|
||||
get_component,
|
||||
)
|
||||
from rdagent.utils.agent.tpl import T
|
||||
|
||||
|
||||
class DSRefineExpGen(ExpGen):
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.support_function_calling = APIBackend().support_function_calling()
|
||||
|
||||
def task_gen(
|
||||
self,
|
||||
component_desc: str,
|
||||
scenario_desc: str,
|
||||
sota_exp_desc: str,
|
||||
sota_exp: DSExperiment,
|
||||
hypothesis: DSHypothesis,
|
||||
exp_and_feedback_list_desc: str,
|
||||
) -> DSExperiment:
|
||||
component_info = get_component("Pipeline")
|
||||
data_folder_info = self.scen.processed_data_folder_description
|
||||
sys_prompt = T(".prompts_refine:task_gen.system").r(
|
||||
task_output_format=component_info["task_output_format"] if not self.support_function_calling else None,
|
||||
component_desc=component_desc,
|
||||
)
|
||||
user_prompt = T(".prompts_refine:task_gen.user").r(
|
||||
scenario_desc=scenario_desc,
|
||||
data_folder_info=data_folder_info,
|
||||
sota_exp_desc=sota_exp_desc,
|
||||
hypothesis=hypothesis,
|
||||
exp_and_feedback_list_desc=exp_and_feedback_list_desc,
|
||||
)
|
||||
response = APIBackend().build_messages_and_create_chat_completion(
|
||||
user_prompt=user_prompt,
|
||||
system_prompt=sys_prompt,
|
||||
response_format=CodingSketch if self.support_function_calling else {"type": "json_object"},
|
||||
json_target_type=Dict[str, str | Dict[str, str]] if not self.support_function_calling else None,
|
||||
)
|
||||
task_dict = json.loads(response)
|
||||
task_design = (
|
||||
task_dict.get("task_design", {}) if not self.support_function_calling else task_dict.get("sketch", {})
|
||||
)
|
||||
logger.info(f"Task design:\n{task_design}")
|
||||
task_name = hypothesis.component
|
||||
description = (
|
||||
task_design
|
||||
if isinstance(task_design, str)
|
||||
else task_design.get("description", f"{component_info['target_name']} description not provided")
|
||||
)
|
||||
task_class = component_info["task_class"]
|
||||
task = task_class(
|
||||
name=task_name,
|
||||
description=description,
|
||||
)
|
||||
exp = DSExperiment(pending_tasks_list=[[task]], hypothesis=hypothesis)
|
||||
if sota_exp is not None:
|
||||
exp.experiment_workspace.inject_code_from_file_dict(sota_exp.experiment_workspace)
|
||||
return exp
|
||||
|
||||
def gen(self, trace: DSTrace) -> DSExperiment:
|
||||
# Step 0: Prepare
|
||||
pipeline = DS_RD_SETTING.coder_on_whole_pipeline
|
||||
component_desc = T("scenarios.data_science.share:component_description_in_pipeline").r()
|
||||
sota_exp, sota_fb = trace.sota_experiment_fb()
|
||||
if not isinstance(sota_exp, DSExperiment):
|
||||
eda_output = None
|
||||
else:
|
||||
eda_output = sota_exp.experiment_workspace.file_dict.get("EDA.md", None)
|
||||
scenario_desc = self.scen.get_scenario_all_desc(trace=trace, eda_output=eda_output)
|
||||
sota_exp_desc = T("scenarios.data_science.share:describe.exp").r(
|
||||
exp=sota_exp, heading="Best of previous exploration of the scenario"
|
||||
)
|
||||
exp_feedback_list_desc = T("scenarios.data_science.share:describe.trace").r(
|
||||
exp_and_feedback_list=trace.experiment_and_feedback_list_after_init(return_type="all"),
|
||||
type="all",
|
||||
pipeline=pipeline,
|
||||
)
|
||||
|
||||
# Step 1: Generate a Pseudo Hypothesis for Refinement
|
||||
hypothesis = DSHypothesis(
|
||||
component="Model",
|
||||
hypothesis="The current pipeline is note effective in terms of efficiency or hyperparameters. Refinement or Adjustment should be made.",
|
||||
reason=sota_fb.reason,
|
||||
)
|
||||
|
||||
# Step 2: Design Task to Refine SOTA
|
||||
return self.task_gen(
|
||||
component_desc=component_desc,
|
||||
scenario_desc=scenario_desc,
|
||||
sota_exp_desc=sota_exp_desc,
|
||||
sota_exp=sota_exp,
|
||||
hypothesis=hypothesis,
|
||||
exp_and_feedback_list_desc=exp_feedback_list_desc,
|
||||
)
|
||||
@@ -0,0 +1,87 @@
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from rdagent.components.coder.data_science.ensemble.exp import EnsembleTask
|
||||
from rdagent.components.coder.data_science.feature.exp import FeatureTask
|
||||
from rdagent.components.coder.data_science.model.exp import ModelTask
|
||||
from rdagent.components.coder.data_science.pipeline.exp import PipelineTask
|
||||
from rdagent.components.coder.data_science.raw_data_loader.exp import DataLoaderTask
|
||||
from rdagent.components.coder.data_science.workflow.exp import WorkflowTask
|
||||
|
||||
_COMPONENT_META: Dict[str, Dict[str, Any]] = {
|
||||
"DataLoadSpec": {
|
||||
"target_name": "Data loader and specification generation",
|
||||
"spec_file": "spec/data_loader.md",
|
||||
"output_format_key": ".prompts:output_format.data_loader",
|
||||
"task_class": DataLoaderTask,
|
||||
},
|
||||
"FeatureEng": {
|
||||
"target_name": "Feature engineering",
|
||||
"spec_file": "spec/feature.md",
|
||||
"output_format_key": ".prompts:output_format.feature",
|
||||
"task_class": FeatureTask,
|
||||
},
|
||||
"Model": {
|
||||
"target_name": "Model",
|
||||
"spec_file": "spec/model.md",
|
||||
"output_format_key": ".prompts:output_format.model",
|
||||
"task_class": ModelTask,
|
||||
},
|
||||
"Ensemble": {
|
||||
"target_name": "Ensemble",
|
||||
"spec_file": "spec/ensemble.md",
|
||||
"output_format_key": ".prompts:output_format.ensemble",
|
||||
"task_class": EnsembleTask,
|
||||
},
|
||||
"Workflow": {
|
||||
"target_name": "Workflow",
|
||||
"spec_file": "spec/workflow.md",
|
||||
"output_format_key": ".prompts:output_format.workflow",
|
||||
"task_class": WorkflowTask,
|
||||
},
|
||||
"Pipeline": {
|
||||
"target_name": "Pipeline",
|
||||
"spec_file": None,
|
||||
"output_format_key": ".prompts:output_format.pipeline",
|
||||
"task_class": PipelineTask,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def get_component(name: str) -> Dict[str, Any]:
|
||||
meta = _COMPONENT_META.get(name)
|
||||
if meta is None:
|
||||
raise KeyError(f"Unknown component: {name!r}")
|
||||
|
||||
return {
|
||||
"target_name": meta["target_name"],
|
||||
"spec_file": meta["spec_file"],
|
||||
"task_output_format": T(meta["output_format_key"]).r(),
|
||||
"task_class": meta["task_class"],
|
||||
}
|
||||
|
||||
|
||||
class CodingSketch(BaseModel):
|
||||
current_state: str = Field(
|
||||
description="A summary of the current `main.py` script that serves as the baseline for the planned changes. Focusing on parts that are related to the hypothesis. If `main.py` does not yet exist (i.e., it will be created from scratch based on this sketch), use the string 'N/A'."
|
||||
)
|
||||
modifications: List[str] = Field(
|
||||
description="A list of specific, targeted changes to be applied to the existing code identified in `current_state`. Each string in the list should concisely describe (in 3-4 sentences): "
|
||||
"(a) the specific part of the code to be altered (e.g., a function name, a class, or a logical block); "
|
||||
"(b) the nature of the modification (e.g., bug fix, feature addition, refactoring of a small section, performance optimization, deletion); and "
|
||||
"(c) a brief explanation or high-level sketch of the new logic or change. "
|
||||
"If no direct modifications to existing code are planned (e.g., if creating an entirely new `main.py` as detailed in `structure`), this list should be empty."
|
||||
)
|
||||
structure: List[str] = Field(
|
||||
description="An outline of the new high-level architectural components (primarily functions and classes) if a new `main.py` script is being created from scratch, or if the existing `main.py` is undergoing a major refactor that fundamentally alters or replaces its core structure. "
|
||||
"Each string in the list should define a planned function or class, detailing its name, primary responsibility, key parameters (if applicable), return values (if applicable), and core functionality in 2-3 sentences. "
|
||||
"This field is typically used when `current_state` is 'N/A' or when the scope of change requires a new architectural blueprint rather than just targeted `modifications`. "
|
||||
"Leave empty if the plan only involves direct `modifications` to the existing structure in `current_state`."
|
||||
)
|
||||
sketch: str = Field(
|
||||
description="A detailed, step-by-step narrative that elaborates on how to implement the planned code. "
|
||||
"This section should synthesize the information from `modifications` (if any) and/or `structure` (if any) into a comprehensive and actionable coding plan for `main.py`. "
|
||||
"The content **must** be formatted using Markdown, with logical sections, key decision points, or implementation steps clearly organized by level-3 headings (i.e., `###`). "
|
||||
"This field should provide sufficient detail for a developer to understand the implementation flow, algorithms, data handling, and key logic points without ambiguity."
|
||||
)
|
||||
@@ -164,7 +164,7 @@ class DataScienceScen(Scenario):
|
||||
# TODO: add it into base class. Environment should(i.e. `DSDockerConf`) should be part of the scenario class.
|
||||
env = get_ds_env()
|
||||
implementation = FBWorkspace()
|
||||
fname = "temp.py"
|
||||
fname = "runtime_info.py"
|
||||
implementation.inject_files(
|
||||
**{fname: (Path(__file__).absolute().resolve().parent / "runtime_info.py").read_text()}
|
||||
)
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import platform
|
||||
import subprocess
|
||||
import sys
|
||||
from importlib.metadata import distributions
|
||||
|
||||
@@ -18,6 +19,38 @@ def print_filtered_packages(installed_packages, filtered_packages):
|
||||
print(f"{package_name}=={version}")
|
||||
|
||||
|
||||
def get_gpu_info():
|
||||
try:
|
||||
# Option 1: Use PyTorch
|
||||
import torch
|
||||
|
||||
if torch.cuda.is_available():
|
||||
print("\n=== GPU Info (via PyTorch) ===")
|
||||
print(f"CUDA Version: {torch.version.cuda}")
|
||||
print(f"GPU Device: {torch.cuda.get_device_name(0)}")
|
||||
print(f"Total GPU Memory: {torch.cuda.get_device_properties(0).total_memory / 1024**3:.2f} GB")
|
||||
print(f"Allocated Memory: {torch.cuda.memory_allocated(0) / 1024**3:.2f} GB")
|
||||
print(f"Cached Memory: {torch.cuda.memory_reserved(0) / 1024**3:.2f} GB")
|
||||
else:
|
||||
print("\nNo CUDA GPU detected (PyTorch).")
|
||||
|
||||
except ImportError:
|
||||
# Option 2: Use nvidia-smi
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["nvidia-smi", "--query-gpu=name,memory.total,memory.used", "--format=csv"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
if result.returncode == 0:
|
||||
print("\n=== GPU Info (via nvidia-smi) ===")
|
||||
print(result.stdout.strip())
|
||||
else:
|
||||
print("\nNo GPU detected (nvidia-smi not available).")
|
||||
except FileNotFoundError:
|
||||
print("\nNo GPU detected (nvidia-smi not installed).")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
print_runtime_info()
|
||||
filtered_packages = [
|
||||
@@ -38,3 +71,4 @@ if __name__ == "__main__":
|
||||
]
|
||||
installed_packages = get_installed_packages()
|
||||
print_filtered_packages(installed_packages, filtered_packages)
|
||||
get_gpu_info()
|
||||
|
||||
@@ -296,13 +296,14 @@ component_spec:
|
||||
- Handle missing values and outliers appropriately (e.g., impute, remove, or replace).
|
||||
- Ensure consistency between feature data types and transformations.
|
||||
- Prevent data leakage: Do not use information derived from the test set when transforming training data.
|
||||
- Sampling a subset of the training data for efficiency (e.g., randomly selecting a portion of the data) is discouraged unless it demonstrably improves performance (e.g., removing irrelevant or outlier samples).
|
||||
|
||||
6. Notes:
|
||||
- GPU and multiprocessing are available and are encouraged to use for accelerating transformations.
|
||||
|
||||
7. Metric Calculation and Storage:
|
||||
- Calculate the metric (mentioned in the evaluation section of the competition information) for each model and ensemble strategy on valid, and save the results in `scores.csv`
|
||||
- The evaluation should be based on 5-fold cross-validation but only if that's an appropriate evaluation for the task at hand. Store the mean validation score of 5-fold cross-validation in `scores.csv` on each model.
|
||||
- The evaluation should be based on k-fold cross-validation but only if that's an appropriate evaluation for the task at hand. Store the mean validation score of k-fold cross-validation in `scores.csv` on each model. Refer to the hyperparameter specification for rules to set the CV folds.
|
||||
- Even if only one model is present, compute the ensemble score and store it under `"ensemble"`.
|
||||
- The index of `scores.csv` should include the model name and the "ensemble" strategy. "ensemble" should be exactly in the index with all lower case letters. Ensemble is the result from several models. If only one model is present, the ensemble score should be the same as the model score.
|
||||
- The column names in `scores.csv` should be:
|
||||
@@ -317,3 +318,21 @@ component_spec:
|
||||
guidelines:
|
||||
coding: |-
|
||||
You might receive exploratory data analysis (EDA) details about the source data. Do not use this EDA information to create assertions or raise errors. We might generate sample data for quick coding (so your code may run on sample data which is part of the full-size data), but remember that the EDA details are based on the full-size data.
|
||||
|
||||
spec:
|
||||
hyperparameter: |-
|
||||
1. Hyperparameters Requiring Tuning (e.g., learning rate, weight decay, optimizer, etc.)
|
||||
- Adjust conservatively to avoid instability.
|
||||
- Apply a systematic hyperparameter tuning strategy to identify optimal values.
|
||||
2. Hyperparameters Dependent on Empirical Estimation or Past Failures (e.g., batch size, patience, epochs, etc.)
|
||||
- Estimate these parameters based on the resource constraints (e.g. run time limit) or experiences from previous experiment failures.
|
||||
3. Balancing Epochs and CV Folds
|
||||
- When run time permit, prioritize increasing the number of training epochs, but always implement early stopping to prevent overfitting and ensure the process completes within the allowed runtime.
|
||||
- When run time constrained, first reduce the number of CV folds—provided that validation reliability remains acceptable—before lowering the number of epochs.
|
||||
4. Early Stopping Strategy
|
||||
- Always implement an early stopping mechanism to prevent overfitting and ensure the process completes within the allowed runtime.
|
||||
- Stop training if one or more of the following conditions are met:
|
||||
- A minimum number of epochs have been completed and no improvement is observed in the monitored metric for a specified patience period.
|
||||
- The validation loss (or metric) reaches a predefined threshold indicating sufficient model performance.
|
||||
- The validation loss (or metric) remains stable (i.e., does not improve) for a set number of consecutive epochs.
|
||||
- Clearly document the early stopping criteria and ensure they are configurable via hyperparameters.
|
||||
|
||||
@@ -340,24 +340,24 @@ class Env(Generic[ASpecificEnvConf]):
|
||||
# we must add the information of data (beyond code) into the key.
|
||||
# Otherwise, all commands operating on data will become invalid (e.g. rm -r submission.csv)
|
||||
# So we recursively walk in the folder and add the sorted relative filename list as part of the key.
|
||||
data_key = []
|
||||
for path in Path(local_path).rglob("*"):
|
||||
p = str(path.relative_to(Path(local_path)))
|
||||
if p.startswith("__pycache__"):
|
||||
continue
|
||||
data_key.append(p)
|
||||
data_key = sorted(data_key)
|
||||
# data_key = []
|
||||
# for path in Path(local_path).rglob("*"):
|
||||
# p = str(path.relative_to(Path(local_path)))
|
||||
# if p.startswith("__pycache__"):
|
||||
# continue
|
||||
# data_key.append(p)
|
||||
# data_key = sorted(data_key)
|
||||
|
||||
key = md5_hash(
|
||||
json.dumps(
|
||||
[
|
||||
[str(path.relative_to(Path(local_path))), path.read_text()]
|
||||
for path in sorted(Path(local_path).rglob("*.py"))
|
||||
for path in sorted(list(Path(local_path).rglob("*.py")) + list(Path(local_path).rglob("*.csv")))
|
||||
]
|
||||
)
|
||||
+ json.dumps({"entry": entry, "running_extra_volume": dict(running_extra_volume)})
|
||||
+ json.dumps({"extra_volumes": self.conf.extra_volumes})
|
||||
+ json.dumps(data_key)
|
||||
# + json.dumps(data_key)
|
||||
)
|
||||
if Path(target_folder / f"{key}.pkl").exists() and Path(target_folder / f"{key}.zip").exists():
|
||||
with open(target_folder / f"{key}.pkl", "rb") as f:
|
||||
|
||||
Reference in New Issue
Block a user