diff --git a/rdagent/scenarios/kaggle/experiment/prompts.yaml b/rdagent/scenarios/kaggle/experiment/prompts.yaml index eb09fe7b..e1f67208 100644 --- a/rdagent/scenarios/kaggle/experiment/prompts.yaml +++ b/rdagent/scenarios/kaggle/experiment/prompts.yaml @@ -10,6 +10,7 @@ kg_description_template: "Target Description": "A description of the target variable to be predicted", "Competition Features": "Two-line description of the overall features involved within the competition as background." "Submission Specifications": "The submission specification & sample submission csv descriptions for the model to output." + "Submission channel number to each sample": "The number of channels in the output for each sample, e.g., 1 for regression, N for N class classification with probabilities, etc. A Integer. If not specified, it is 1." } Since these might be very similar column names in data like one_hot_encoded columns, you can use some regex to group them together. @@ -314,12 +315,15 @@ kg_feature_simulator: |- kg_model_output_format: |- For feature related tasks, the output should be a pandas DataFrame with the new features. The columns should be the new features, and the rows should correspond to the number of samples in the input DataFrame. - For model related tasks: - 1. the output should be an np.ndarray with the appropriate number of predictions & the appropriate values within each prediction - 2. the output should be a 2D array with dimensions corresponding to the number of predictions and the number of things to output. Eg, if 4 predictions, each prediction needs to predict 3 probabilities, then (4,3). Or (8, 1) if there are 8 predictions but each prediction is only one value. - 3. please reference the competition's submission requirement and align with that. - Submission Requirements here:\n: {{submission_specifications}} + For model related tasks, the output should be an np.ndarray with the appropriate number of predictions. + {% if channel == 1 %} + For each sample, the output should be a single value (e.g., (8, 1) if there are 8 samples). + {% else %} + For each sample, the output should be multiple values with {{ channel }} numbers (e.g., (8, {{ channel }}) if there are 8 samples). + {% endif %} kg_model_simulator: |- The models will be trained on the competition dataset and evaluated on their ability to predict the target. Metrics like accuracy and AUC-ROC is used to evaluate the model performance. Model performance will be iteratively improved based on feedback from evaluation results. + Your output should follow some requirements to submit to the competition: + {{ submission_specifications }} \ No newline at end of file diff --git a/rdagent/scenarios/kaggle/experiment/scenario.py b/rdagent/scenarios/kaggle/experiment/scenario.py index 45feb04c..d7883f10 100644 --- a/rdagent/scenarios/kaggle/experiment/scenario.py +++ b/rdagent/scenarios/kaggle/experiment/scenario.py @@ -32,16 +32,15 @@ class KGScenario(Scenario): self.target_description = None self.competition_features = None self.submission_specifications = None + self.model_output_channel = None self._analysis_competition_description() + self.if_action_choosing_based_on_UCB = KAGGLE_IMPLEMENT_SETTING.if_action_choosing_based_on_UCB - # Move these assignments after _analysis_competition_description self._output_format = self.output_format self._interface = self.interface self._simulator = self.simulator self._background = self.background - self.if_action_choosing_based_on_UCB = KAGGLE_IMPLEMENT_SETTING.if_action_choosing_based_on_UCB - def _analysis_competition_description(self): sys_prompt = ( Environment(undefined=StrictUndefined) @@ -64,25 +63,15 @@ class KGScenario(Scenario): json_mode=True, ) - try: - response_json_analysis = json.loads(response_analysis) - self.competition_type = response_json_analysis.get("Competition Type", "No type provided") - self.competition_description = response_json_analysis.get( - "Competition Description", "No description provided" - ) - self.target_description = response_json_analysis.get("Target Description", "No target provided") - self.competition_features = response_json_analysis.get("Competition Features", "No features provided") - self.submission_specifications = response_json_analysis.get( - "Submission Specifications", "No submission requirements provided" - ) - except json.JSONDecodeError: - print(f"Failed to parse JSON response: {response_analysis}") - # Set default values if JSON parsing fails - self.competition_type = "Unknown" - self.competition_description = "No description available" - self.target_description = "No target available" - self.competition_features = "No features available" - self.submission_specifications = "No submission requirements available" + response_json_analysis = json.loads(response_analysis) + self.competition_type = response_json_analysis.get("Competition Type", "No type provided") + self.competition_description = response_json_analysis.get("Competition Description", "No description provided") + self.target_description = response_json_analysis.get("Target Description", "No target provided") + self.competition_features = response_json_analysis.get("Competition Features", "No features provided") + self.submission_specifications = response_json_analysis.get( + "Submission Specifications", "No submission requirements provided" + ) + self.model_output_channel = response_json_analysis.get("Submission channel number to each sample", 1) def get_competition_full_desc(self) -> str: return f"""Competition Type: {self.competition_type} @@ -155,7 +144,7 @@ class KGScenario(Scenario): return ( Environment(undefined=StrictUndefined) .from_string(prompt_dict["kg_model_output_format"]) - .render(submission_specifications=self.submission_specifications) + .render(channel=self.model_output_channel) ) @property @@ -168,10 +157,15 @@ The model code should follow the interface: @property def simulator(self) -> str: + kg_model_simulator = ( + Environment(undefined=StrictUndefined) + .from_string(prompt_dict["kg_model_simulator"]) + .render(submission_specifications=self.submission_specifications) + ) return f"""The feature code should follow the simulator: {prompt_dict['kg_feature_simulator']} The model code should follow the simulator: -{prompt_dict['kg_model_simulator']} +{kg_model_simulator} """ @property