reporeformat V2 (#23)

* reformat factor implement process

* move some code to more reasonable place

* fix the bug

* add test function in factor_extract_and_implement.py

* change select factor number to ratio , add some factor implement setting and fix some bug while using knowledgebase

* change evoagent

* add abstract class EvoAgent

* add benchmark workflow

* fix some bug in llm_utils

* run wenjun's code

* fix the knowledgebase instance check

---------

Co-authored-by: xuyang1 <xuyang1@microsoft.com>

rdagent/app/factor_extraction_and_implementation/factor_extract_and_implement.py

@@ -8,8 +8,13 @@ from rdagent.document_process.document_analysis import (
     deduplicate_factors_by_llm,
     extract_factors_from_report_dict,
     merge_file_to_factor_dict_to_factor_dict,
+    classify_report_from_dict,
 )
 from rdagent.document_process.document_reader import load_and_process_pdfs_by_langchain
+from rdagent.factor_implementation.share_modules.factor_implementation_utils import load_data_from_dict
+from rdagent.factor_implementation.CoSTEER import CoSTEERFG
+import pickle
+from dotenv import load_dotenv
 
 
 def extract_factors_and_implement(report_file_path: str) -> None:
@@ -24,6 +29,15 @@ def extract_factors_and_implement(report_file_path: str) -> None:
 
     factor_dict, duplication_names_list = deduplicate_factors_by_llm(factor_dict, factor_viability)
 
+    factor_tasks = load_data_from_dict(factor_dict)
+
+    factor_generate_method = CoSTEERFG()
+
+    result = factor_generate_method.generate(factor_tasks)
+
+    return result
+
 
 if __name__ == "__main__":
     extract_factors_and_implement("/home/xuyang1/workspace/report.pdf")
+    # test_implement()

rdagent/app/factor_implementation/eval_implement.py

@@ -0,0 +1,30 @@
+from rdagent.core.conf import BenchmarkSettings
+from rdagent.core.utils import import_class
+from rdagent.benchmark.eval_method import FactorImplementEval
+from rdagent.benchmark.data_process import load_eval_data
+
+# 1.read the settings
+bs = BenchmarkSettings()
+
+# 2.read and prepare the eval_data
+test_cases = load_eval_data(bs.bench_version)
+
+# 3.declare the method to be tested and pass the arguments.
+# TODO: Whether it is necessary to define two Eval method classes for two data type?
+method_cls = import_class(bs.bench_method_cls)
+generate_method = method_cls(bs.bench_method_extra_kwargs)
+
+# 4.declare the eval method and pass the arguments.
+eval_method = FactorImplementEval(
+    method=generate_method,
+    test_cases=test_cases,
+    catch_eval_except=True,
+    test_round=bs.bench_test_round,
+)
+
+# 5.run the eval
+eval_method.eval()
+
+# 6.save the result
+eval_method.save(output_path = bs.bench_result_path)
+