gavindiaz/NexQuant
1
0
Fork 0
mirror of https://github.com/NicolasBohn/NexQuant.git synced 2026-07-27 23:47:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: add scores.csv checking in ensemble_test (#567)

Browse Source 
* add scores.csv checking in ensemble_test

* text modify
This commit is contained in:
XianBW
2025-02-08 19:03:29 +08:00
committed by GitHub
parent 2d5cd556e5
commit 1b193fa2e7
2 changed files with 19 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rdagent/components/coder/data_science/ensemble/eval_tests/ensemble_test.txt
+8 -1
View File
@@ -67,8 +67,15 @@ if isinstance(final_pred, (list, np.ndarray, pd.DataFrame, torch.Tensor, tf.Tens
f"vs. len(test_X)={len(test_X)}"
)
# check if scores.csv is generated
# check scores.csv
assert Path("scores.csv").exists(), "scores.csv is not generated"
score_df = pd.read_csv("scores.csv", index_col=0)
model_set_in_scores = set(score_df.index)
for model in {{model_names}}:
if model not in model_set_in_scores:
print(f"\nModel {model} is not evaluated in the scores.csv.")
print("Please check scores dataframe's format, it's:")
print(score_df)
print("Ensemble test passed successfully.")
print(f"Output shape: {final_pred.shape}")
rdagent/components/coder/data_science/workflow/eval.py
+11 -10
View File
@@ -80,16 +80,17 @@ class WorkflowGeneralCaseSpecEvaluator(CoSTEEREvaluator):
if not score_fp.exists():
stdout += "\nMetrics file (scores.csv) is not generated."
else:
score_df = pd.read_csv(score_fp, index_col=0)
model_set_in_scores = set(score_df.index)
model_set_in_folder = set(
f[:-3] for f in implementation.file_dict.keys() if re.match(r"^model_(?!test)\w+\.py$", f)
)
for model in model_set_in_folder:
if model not in model_set_in_scores:
stdout += (
f"\nModel {model} is not evaluated in the scores.csv. The scores.csv has {model_set_in_scores}."
)
try:
score_df = pd.read_csv(score_fp, index_col=0)
model_set_in_scores = set(score_df.index)
model_set_in_folder = set(
f[:-3] for f in implementation.file_dict.keys() if re.match(r"^model_(?!test)\w+\.py$", f)
)
for model in model_set_in_folder:
if model not in model_set_in_scores:
stdout += f"\nModel {model} is not evaluated in the scores.csv. The scores.csv has {model_set_in_scores}."
except Exception as e:
stdout += f"\nError in checking the scores.csv file: {e}\nscores.csv's content:\n-----\n{score_fp.read_text()}\n-----"
# Check submission file
submission_fp = implementation.workspace_path / "submission.csv"