From 39e25d28a1d09b967906ff46a537bc4f58e8f7c7 Mon Sep 17 00:00:00 2001
From: Yuante Li <104308117+WinstonLiyt@users.noreply.github.com>
Date: Tue, 22 Apr 2025 18:47:24 +0800
Subject: [PATCH] fix: improve eval alignment check (e.g. small-scale
 finetuning) (#802)

* fix

* fix
---
 rdagent/scenarios/data_science/dev/prompts.yaml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/rdagent/scenarios/data_science/dev/prompts.yaml b/rdagent/scenarios/data_science/dev/prompts.yaml
index b90d84a6..42e8a0e9 100644
--- a/rdagent/scenarios/data_science/dev/prompts.yaml
+++ b/rdagent/scenarios/data_science/dev/prompts.yaml
@@ -18,13 +18,14 @@ exp_feedback:
     - If submission passes, proceed to Step 2.
 
     Step 2: Evaluate Alignment with Competition Requirements (if format correct)
-    - GOAL: TRY YOUR BEST TO AVOID SIGNIFICANT DISCREPANCIES BETWEEN VALIDATION AND TEST SET SCORES.
+    - GOAL: CAREFULLY ANALYZE WHETHER THE EXPERIMENTAL SETUP AND CODE MAY CAUSE MISALIGNMENT BETWEEN VALIDATION AND TEST PERFORMANCE.
     - Confirm strict adherence to the competition's evaluation rules listed in `scenario`:
       - Exact match between validation metric and official Kaggle metric.
       - Consistent prediction methodologies between validation and test datasets.
       - No shortcuts or fold-specific strategies applied inconsistently.
       - Rigorous checks for corner-case consistency.
-    - If discrepancies are identified:
+    - Additionally, detect whether the setup introduces structural risks, such as overfitting-prone finetuning strategies or domain adaptation on insufficient data.
+    - If such discrepancies or risks are found:
       - Clearly document these issues in `Reasoning`.
       - Set `"Evaluation Aligned With Task": "no"` and `"Replace Best Result": "no"`.
       - Begin your `reasoning` with `[Evaluation error]`, explicitly stating the evaluation alignment issues causing experiment failure.