diff --git a/rdagent/components/coder/data_science/ensemble/eval_tests/ensemble_test.txt b/rdagent/components/coder/data_science/ensemble/eval_tests/ensemble_test.txt index c1d17742..ce38c2a6 100644 --- a/rdagent/components/coder/data_science/ensemble/eval_tests/ensemble_test.txt +++ b/rdagent/components/coder/data_science/ensemble/eval_tests/ensemble_test.txt @@ -16,6 +16,21 @@ from load_data import load_data from feature import feat_eng from ensemble import ensemble_workflow +def print_preds_info(model_name, data_type, preds): + if preds is None: + print(f"Model {model_name} {data_type} predictions: None") + else: + print(f"Model {model_name} {data_type} predictions shape: {preds.shape}") + + if isinstance(preds, pd.DataFrame): + print(preds.head()) + elif isinstance(preds, (np.ndarray, torch.Tensor, tf.Tensor)): + print(pd.DataFrame(preds).head()) + elif isinstance(preds, list): + print(pd.DataFrame(preds[:5])) + else: + print(f"Unknown prediction type: {type(preds)}") + X, y, test_X, test_ids = load_data() X, y, test_X = feat_eng(X, y, test_X) train_X, val_X, train_y, val_y = train_test_split(X, y, test_size=0.2, random_state=42) @@ -34,6 +49,8 @@ val_preds_dict["{{mn}}"], test_preds_dict["{{mn}}"], _ = {{mn}}_workflow( val_y=val_y, test_X=test_X ) + +print_preds_info("{{mn}}", "test", test_preds_dict["{{mn}}"]) {% endfor %} for key in val_preds_dict.keys(): @@ -54,6 +71,8 @@ for key in val_preds_dict.keys(): # Run ensemble final_pred = ensemble_workflow(test_preds_dict, val_preds_dict, val_y) +print_preds_info("ensemble", "test", final_pred) + # Check type pred_type = type(next(iter(test_preds_dict.values()))) assert isinstance(final_pred, pred_type), ( diff --git a/rdagent/components/coder/data_science/ensemble/prompts.yaml b/rdagent/components/coder/data_science/ensemble/prompts.yaml index 7e15f058..2e0d0197 100644 --- a/rdagent/components/coder/data_science/ensemble/prompts.yaml +++ b/rdagent/components/coder/data_science/ensemble/prompts.yaml @@ -9,7 +9,7 @@ ensemble_coder: Your specific task as follows: {{ task_desc }} - Here is the competition information for this task: + ## Competition Information for This Task {{ competition_info }} {% if queried_similar_successful_knowledge|length != 0 or queried_former_failed_knowledge|length != 0 %} @@ -97,7 +97,7 @@ ensemble_eval: { "execution": "Describe how well the ensemble executed, including any errors or issues encountered. Retain all error messages and traceback details.", "return_checking": "Detail the checks performed on the ensemble results, including shape and value validation.", - "code": "Assess code quality, readability, and adherence to specifications. Consider efficiency, including whether the code utilizes multi-threading or GPU acceleration for optimization.", + "code": "Assess code quality, readability, and adherence to specifications.", "final_decision": } ``` diff --git a/rdagent/components/coder/data_science/raw_data_loader/prompts.yaml b/rdagent/components/coder/data_science/raw_data_loader/prompts.yaml index 1513b769..a51e4e7b 100644 --- a/rdagent/components/coder/data_science/raw_data_loader/prompts.yaml +++ b/rdagent/components/coder/data_science/raw_data_loader/prompts.yaml @@ -1,3 +1,4 @@ + spec: system: |- You are a world-class data scientist and machine learning engineer with deep expertise in statistics, mathematics, and computer science. @@ -23,14 +24,14 @@ spec: Your specification should consists two parts: 1. The function definition in code format with detailed annotation to each parameter and return value. - 2. A detailed docstring to the function that explains the purpose of the function, the input parameters, and the output. + 2. A correct, concise, and complete docstring to the function that explains the purpose of the function, the input parameters, and the output. 3. Additional information or notes that the coder should consider while implementing the function. Your specifications should not include any code implementation, only the function definition and docstring. - -----------Competition Information----------- + ----------- Competition Information ----------- {{ competition_info }} - -----------Folder Description---------(All path are relative to the data folder) + -----------Folder Description (All path are relative to the data folder) --------- - Ensure that all columns in sample_submission can be generated. {{ folder_spec }} @@ -53,11 +54,11 @@ spec: 2. Precautions for Data Loading and Preprocessing: - File Handling: - - Ensure proper file encoding (e.g., UTF-8) and delimiters (e.g., CSV comma-separated). + - Ensure proper file encoding and delimiters. - Combine or process multiple files if necessary. - Data Preprocessing: - Convert data types correctly (e.g., numeric, categorical, date parsing). - - Handle missing values appropriately (e.g., impute, drop rows/columns). + - Handle missing values appropriately. - Optimize memory usage for large datasets using techniques like downcasting or reading data in chunks if necessary. - Domain-Specific Handling: - Apply competition-specific preprocessing steps as needed (e.g., text tokenization, image resizing). @@ -67,7 +68,6 @@ spec: 4. Notes: - Update `DT` (data type) based on the specific competition dataset. This can include `pd.DataFrame`, `np.array`, `torch.Tensor`, etc. - - Extend domain-specific handling steps based on the competition information. - Never use sample submission as the test index, as it may not be the same as the test data. Use the test index file or test data source to get the test index. {% if latest_spec %} @@ -76,8 +76,6 @@ spec: You should follow the provided specifications to improve this task. {% endif %} - - Please respond with a JSON structure as follows: { "spec": "The function definition in code format, tailored to the Competition Information, with detailed explanations provided in the docstring." @@ -160,7 +158,7 @@ spec: - Inferred data shape to each input and output data variables. To uncertain dimension, use -1. 2. Code Standards: - - Avoid using progress bars (e.g., `tqdm`) in the implementation. + - Do not use progress bars (e.g., `tqdm`) in the implementation. 3. Precautions: - Ensure input arrays (`X`, `y`, `val_X`, `val_y`, `test_X`) have consistent dimensions and shapes. @@ -168,7 +166,6 @@ spec: - Train the model on `X` and `y`. - Evaluate the model using `val_X` and `val_y` if validation data is available. - If `test_X` is provided, generate predictions for it. - - Do not use progress bars (e.g., `tqdm`) in the implementation. 4. Notes: - Align `DT` (data type) with the definitions used in Feature Engineering specifications. @@ -206,25 +203,24 @@ spec: - Ensure all predictions in `test_preds_dict` and `val_preds_dict` have consistent shapes and dimensions. - Verify that `val_label` is provided and matches the length of `val_preds_dict` predictions. - Handle empty or invalid inputs gracefully with appropriate error messages. - - You should calculate the metric for each model and ensemble strategy, and save the results in `scores.csv`, e.g.: - ```python - scores = {} - for model_name, val_pred in val_preds_dict.items(): - scores[model_name] = calculate_metric(val_label, val_pred) - - ... some code about ensemble strategy ... - ensemble_score = calculate_metric(val_label, ensemble_pred) - scores[] = ensemble_score - - scores_df = pd.DataFrame(scores.items(), columns=['Model', ]) - scores_df.to_csv("scores.csv", index=False) - ``` - - Consensus Strategy: - - Clearly define how the ensemble predictions are aggregated (e.g., majority voting, weighted average). - - Avoid introducing biases or overfitting during decision-making. - + - Metric Calculation and Storage: + - Calculate the metric for each model and ensemble strategy, and save the results in `scores.csv`, e.g.: + ```python + scores = {} + for model_name, val_pred in val_preds_dict.items(): + scores[model_name] = calculate_metric(val_label, val_pred) + + # ... some code about ensemble strategy ... + ensemble_score = calculate_metric(val_label, ensemble_pred) + scores["ensemble"] = ensemble_score # Ensure "ensemble" is explicitly stored + + scores_df = pd.DataFrame(scores.items(), columns=["Model", ]) + scores_df.to_csv("scores.csv", index=False) + ``` + - If there is only one model, still compute the ensemble score and store it under "ensemble". + 3. Code Standards: - - Avoid using progress bars (e.g., `tqdm`) in the implementation. + - Do not use progress bars (e.g., tqdm) in the code. 4. Notes: - Align `DT` (data type) definitions with those used in model specifications. @@ -263,16 +259,13 @@ spec: 3. Dataset Splitting - The dataset returned by `load_data` is not split into training and testing sets, so the dataset splitting should happen after calling `feat_eng`. - - By default, split the dataset into 80% for training and 20% for testing. - - You can also use cross-validation or other splitting methods as you deem more useful and appropriate based on the Competition Information. + - Choose the splitting methods you deem useful and appropriate based on the `Competition Information`. 4. Submission File: - - Save the final predictions as `submission.csv` in the format required by the competition. + - Save the final predictions as `submission.csv`, ensuring the format matches the competition requirements (refer to `sample_submission` in the Folder Description for the correct structure). - Present the required submission format explicitly and ensure the output adheres to it. 5. Code Standards: - - Use consistent naming conventions and type annotations. - - Document the workflow with clear comments and docstring. - Do not use progress bars (e.g., tqdm) in the code. 6. Ensemble Strategy: @@ -336,7 +329,6 @@ data_loader_coder: ## Guidelines 1. Ensure that the dataset is loaded strictly from `/kaggle/input/`, following the exact folder structure described in the **Data Folder Description**, and do not attempt to load data from the current directory (`./`). - 2. You should avoid using logging module to output information in your generated code, and instead use the print() function. ## Output Format Please response the code in the following json format. Here is an example structure for the JSON output: diff --git a/rdagent/scenarios/data_science/dev/prompts.yaml b/rdagent/scenarios/data_science/dev/prompts.yaml index d0314f50..8bfb4c97 100644 --- a/rdagent/scenarios/data_science/dev/prompts.yaml +++ b/rdagent/scenarios/data_science/dev/prompts.yaml @@ -37,7 +37,7 @@ exp_feedback: Current solution to be evaluated: ### Task of Current solution - {{cur_exp.pending_tasks_list[0][0].get_task_information()}} + {{ cur_exp.pending_tasks_list[0][0].get_task_information() }} {% if cur_exp.hypothesis %} the experiment is designed based on hypothesis: {{ cur_exp.hypothesis }} @@ -50,18 +50,21 @@ exp_feedback: {{ de }} {% endfor %} - Final results of the current solution: + ### Final results of the current solution + 1. Pay close attention to the performance of ensemble, as it represents the final score for this iteration. + 2. If any individual model significantly outperforms the ensemble, this may indicate an issue in the ensemble method. If the final ensemble score surpasses the current SOTA, you may update the SOTA record. However, it seems that there are noticeable issues in the ensemble component, be sure to highlight them explicitly. + Below are the results for this experiment: {{ cur_exp.result }} {% if cur_exp.format_check_result is not none %} - Submission format check to current solution: + ### Submission format check to current solution: {{ cur_exp.format_check_result }} {% endif %} ### Complete Code of current solution - {{cur_exp.experiment_workspace.all_codes}} + {{ cur_exp.experiment_workspace.all_codes }} - {{feedback_desc}} + {{ feedback_desc }} Please refer to these hypotheses and feedback to help you recommend new experiment and hypothesis