diff --git a/rdagent/components/coder/data_science/ensemble/eval_tests/ensemble_test.txt b/rdagent/components/coder/data_science/ensemble/eval_tests/ensemble_test.txt index 06837e6e..de85f6d5 100644 --- a/rdagent/components/coder/data_science/ensemble/eval_tests/ensemble_test.txt +++ b/rdagent/components/coder/data_science/ensemble/eval_tests/ensemble_test.txt @@ -70,8 +70,31 @@ for key in val_preds_dict.keys(): print(f"val_y.shape: {val_y.shape}" if not isinstance(val_y, list) else f"val_y(list)'s length: {len(val_y)}") +import sys +import reprlib +def debug_info_print(func): + aRepr = reprlib.Repr() + aRepr.maxother=300 + def wrapper(*args, **kwargs): + def local_trace(frame, event, arg): + if event == "return" and frame.f_code == func.__code__: + print("\n" + "="*20 + "Running ensemble code, local variable values:" + "="*20) + for k, v in frame.f_locals.items(): + printed = aRepr.repr(v) + print(f"{k}:\n {printed}") + print("="*20 + "Local variable values end" + "="*20) + return local_trace + + sys.settrace(local_trace) + try: + return func(*args, **kwargs) + finally: + sys.settrace(None) + return wrapper + + # Run ensemble -final_pred = ensemble_workflow(test_preds_dict, val_preds_dict, val_y) +final_pred = debug_info_print(ensemble_workflow)(test_preds_dict, val_preds_dict, val_y) print_preds_info("ensemble", "test", final_pred) diff --git a/rdagent/components/coder/data_science/ensemble/prompts.yaml b/rdagent/components/coder/data_science/ensemble/prompts.yaml index 73ba14b6..47b30110 100644 --- a/rdagent/components/coder/data_science/ensemble/prompts.yaml +++ b/rdagent/components/coder/data_science/ensemble/prompts.yaml @@ -95,6 +95,7 @@ ensemble_eval: ## Evaluation Criteria - You will be given the standard output (`stdout`) from the ensemble test and, if applicable, the workflow test. - Code should have no try-except blocks because they can hide errors. + - The stdout includes the local variable values from the ensemble code execution. Check whether the validation score is calculated correctly. Please respond with your feedback in the following JSON format and order ```json diff --git a/rdagent/components/coder/data_science/feature/eval_tests/feature_test.txt b/rdagent/components/coder/data_science/feature/eval_tests/feature_test.txt index 947fa617..bb029aa0 100644 --- a/rdagent/components/coder/data_science/feature/eval_tests/feature_test.txt +++ b/rdagent/components/coder/data_science/feature/eval_tests/feature_test.txt @@ -23,7 +23,29 @@ print(f"test_ids length: {len(test_ids)}") X_loaded = deepcopy(X) y_loaded = deepcopy(y) X_test_loaded = deepcopy(X_test) -X, y, X_test = feat_eng(X, y, X_test) + +import sys +import reprlib +def debug_info_print(func): + aRepr = reprlib.Repr() + aRepr.maxother=300 + def wrapper(*args, **kwargs): + def local_trace(frame, event, arg): + if event == "return" and frame.f_code == func.__code__: + print("\n" + "="*20 + "Running feat_eng code, local variable values:" + "="*20) + for k, v in frame.f_locals.items(): + printed = aRepr.repr(v) + print(f"{k}:\n {printed}") + print("="*20 + "Local variable values end" + "="*20) + return local_trace + + sys.settrace(local_trace) + try: + return func(*args, **kwargs) + finally: + sys.settrace(None) + return wrapper +X, y, X_test = debug_info_print(feat_eng)(X, y, X_test) def get_length(data): diff --git a/rdagent/components/coder/data_science/model/eval_tests/model_test.txt b/rdagent/components/coder/data_science/model/eval_tests/model_test.txt index e2a8361c..5dea8be0 100644 --- a/rdagent/components/coder/data_science/model/eval_tests/model_test.txt +++ b/rdagent/components/coder/data_science/model/eval_tests/model_test.txt @@ -25,10 +25,32 @@ print(f"train_y.shape: {train_y.shape}" if not isinstance(train_y, list) else f" print(f"val_X.shape: {val_X.shape}") print(f"val_y.shape: {val_y.shape}" if not isinstance(val_y, list) else f"val_y(list)'s length: {len(val_y)}") +import sys +import reprlib +def debug_info_print(func): + aRepr = reprlib.Repr() + aRepr.maxother=300 + def wrapper(*args, **kwargs): + def local_trace(frame, event, arg): + if event == "return" and frame.f_code == func.__code__: + print("\n" + "="*20 + "Running model training code, local variable values:" + "="*20) + for k, v in frame.f_locals.items(): + printed = aRepr.repr(v) + print(f"{k}:\n {printed}") + print("="*20 + "Local variable values end" + "="*20) + return local_trace + + sys.settrace(local_trace) + try: + return func(*args, **kwargs) + finally: + sys.settrace(None) + return wrapper + # First execution print("The first execution begins.\n") start_time = time.time() -val_pred, test_pred, hypers = model_workflow( +val_pred, test_pred, hypers = debug_info_print(model_workflow)( X=train_X, y=train_y, val_X=val_X, @@ -40,7 +62,7 @@ log_execution_results(start_time, val_pred, test_pred, hypers, "The first execut # Second execution print("The second execution begins.\n") start_time = time.time() -val_pred, test_pred, final_hypers = model_workflow( +val_pred, test_pred, final_hypers = debug_info_print(model_workflow)( X=train_X, y=train_y, val_X=None, diff --git a/rdagent/components/coder/data_science/raw_data_loader/eval_tests/data_loader_test.txt b/rdagent/components/coder/data_science/raw_data_loader/eval_tests/data_loader_test.txt index 750dfe11..427066de 100644 --- a/rdagent/components/coder/data_science/raw_data_loader/eval_tests/data_loader_test.txt +++ b/rdagent/components/coder/data_science/raw_data_loader/eval_tests/data_loader_test.txt @@ -12,7 +12,29 @@ import pickle import pandas as pd from load_data import load_data -X, y, X_test, test_ids = load_data() +import sys +import reprlib +def debug_info_print(func): + aRepr = reprlib.Repr() + aRepr.maxother=300 + def wrapper(*args, **kwargs): + def local_trace(frame, event, arg): + if event == "return" and frame.f_code == func.__code__: + print("\n" + "="*20 + "Running data_load code, local variable values:" + "="*20) + for k, v in frame.f_locals.items(): + printed = aRepr.repr(v) + print(f"{k}:\n {printed}") + print("="*20 + "Local variable values end" + "="*20) + return local_trace + + sys.settrace(local_trace) + try: + return func(*args, **kwargs) + finally: + sys.settrace(None) + return wrapper + +X, y, X_test, test_ids = debug_info_print(load_data)() def get_length(data):