CI checks that can be automatically repaired (#119)

* fix isort & black & toml-sort & sphinx error

* fix ci error

* fix ci error

* add comments

* Update Makefile

* change sphinx build command

* add auto-lint

* add black args

* format with black

* Auto Linting document

* fix ci error

---------

Co-authored-by: you-n-g <you-n-g@users.noreply.github.com>
Co-authored-by: Young <afe.young@gmail.com>

rdagent/app/data_mining/conf.py

@@ -1,6 +1,5 @@
 from pathlib import Path
 
-
 from rdagent.components.workflow.conf import BasePropSetting
 
 
@@ -19,11 +18,11 @@ class PropSetting(BasePropSetting):
 
     evolving_n: int = 10
 
-    # 2) Extra config for the scenario 
+    # 2) Extra config for the scenario
     # physionet account
     # NOTE: You should apply the account in https://physionet.org/
-    username: str = ''
-    password: str = ''
+    username: str = ""
+    password: str = ""
 
 
 PROP_SETTING = PropSetting()

rdagent/app/data_mining/model.py

@@ -1,8 +1,10 @@
 import fire
+
 from rdagent.app.data_mining.conf import PROP_SETTING
 from rdagent.components.workflow.rd_loop import RDLoop
 from rdagent.core.exception import ModelEmptyError
 
+
 class ModelRDLoop(RDLoop):
     skip_loop_error = (ModelEmptyError,)
 

rdagent/app/qlib_rd_loop/RDAgent.py

@@ -1,4 +1,5 @@
 import pickle
+
 from rdagent.app.qlib_rd_loop.conf import PROP_SETTING
 from rdagent.core.developer import Developer
 from rdagent.core.exception import ModelEmptyError
@@ -12,6 +13,7 @@ from rdagent.core.scenario import Scenario
 from rdagent.core.utils import import_class
 from rdagent.log import rdagent_logger as logger
 
+
 # TODO: we can design a workflow that can automatically save session and traceback in the future
 class Model_RD_Agent:
     def __init__(self):
@@ -20,50 +22,55 @@ class Model_RD_Agent:
         self.hypothesis2experiment: Hypothesis2Experiment = import_class(PROP_SETTING.model_hypothesis2experiment)()
         self.qlib_model_coder: Developer = import_class(PROP_SETTING.model_coder)(self.scen)
         self.qlib_model_runner: Developer = import_class(PROP_SETTING.model_runner)(self.scen)
-        self.qlib_model_summarizer: HypothesisExperiment2Feedback = import_class(PROP_SETTING.model_summarizer)(self.scen)
+        self.qlib_model_summarizer: HypothesisExperiment2Feedback = import_class(PROP_SETTING.model_summarizer)(
+            self.scen
+        )
         self.trace = Trace(scen=self.scen)
 
     def generate_hypothesis(self):
         hypothesis = self.hypothesis_gen.gen(self.trace)
-        self.dump_objects(hypothesis=hypothesis, trace=self.trace, filename='step_hypothesis.pkl')
+        self.dump_objects(hypothesis=hypothesis, trace=self.trace, filename="step_hypothesis.pkl")
         return hypothesis
 
     def convert_hypothesis(self, hypothesis):
         exp = self.hypothesis2experiment.convert(hypothesis, self.trace)
-        self.dump_objects(exp=exp, hypothesis=hypothesis, trace=self.trace, filename='step_experiment.pkl')
+        self.dump_objects(exp=exp, hypothesis=hypothesis, trace=self.trace, filename="step_experiment.pkl")
         return exp
 
     def generate_code(self, exp):
         exp = self.qlib_model_coder.develop(exp)
-        self.dump_objects(exp=exp, trace=self.trace, filename='step_code.pkl')
+        self.dump_objects(exp=exp, trace=self.trace, filename="step_code.pkl")
         return exp
 
     def run_experiment(self, exp):
         exp = self.qlib_model_runner.develop(exp)
-        self.dump_objects(exp=exp, trace=self.trace, filename='step_run.pkl')
+        self.dump_objects(exp=exp, trace=self.trace, filename="step_run.pkl")
         return exp
 
     def generate_feedback(self, exp, hypothesis):
         feedback = self.qlib_model_summarizer.generate_feedback(exp, hypothesis, self.trace)
-        self.dump_objects(exp=exp, hypothesis=hypothesis, feedback=feedback, trace=self.trace, filename='step_feedback.pkl')
+        self.dump_objects(
+            exp=exp, hypothesis=hypothesis, feedback=feedback, trace=self.trace, filename="step_feedback.pkl"
+        )
         return feedback
 
     def append_to_trace(self, hypothesis, exp, feedback):
         self.trace.hist.append((hypothesis, exp, feedback))
-        self.dump_objects(trace=self.trace, filename='step_trace.pkl')
+        self.dump_objects(trace=self.trace, filename="step_trace.pkl")
 
-    def dump_objects(self, exp=None, hypothesis=None, feedback=None, trace=None, filename='dumped_objects.pkl'):
-        with open(filename, 'wb') as f:
+    def dump_objects(self, exp=None, hypothesis=None, feedback=None, trace=None, filename="dumped_objects.pkl"):
+        with open(filename, "wb") as f:
             pickle.dump((exp, hypothesis, feedback, trace or self.trace), f)
 
     def load_objects(self, filename):
-        with open(filename, 'rb') as f:
+        with open(filename, "rb") as f:
             return pickle.load(f)
 
+
 def process_steps(agent):
     # Load trace if available
     try:
-        _, _, _, trace = agent.load_objects('step_trace.pkl')
+        _, _, _, trace = agent.load_objects("step_trace.pkl")
         agent.trace = trace
         print(trace.get_sota_hypothesis_and_experiment())
     except FileNotFoundError:
@@ -99,6 +106,7 @@ def process_steps(agent):
     # # Step 6: Append to trace
     # agent.append_to_trace(hypothesis, exp, feedback)
 
+
 if __name__ == "__main__":
     agent = Model_RD_Agent()
     process_steps(agent)

rdagent/app/qlib_rd_loop/factor_from_report.py

@@ -1,34 +1,38 @@
 import json
-from pathlib import Path
 import pickle
+from pathlib import Path
+
+import pandas as pd
 from dotenv import load_dotenv
 from jinja2 import Environment, StrictUndefined
-import pandas as pd
 
 from rdagent.app.qlib_rd_loop.conf import PROP_SETTING
-from rdagent.components.document_reader.document_reader import load_and_process_pdfs_by_langchain
+from rdagent.components.document_reader.document_reader import (
+    load_and_process_pdfs_by_langchain,
+)
+from rdagent.core.developer import Developer
 from rdagent.core.prompts import Prompts
+from rdagent.core.proposal import (
+    Hypothesis,
+    Hypothesis2Experiment,
+    HypothesisExperiment2Feedback,
+    HypothesisGen,
+    Trace,
+)
 from rdagent.core.scenario import Scenario
 from rdagent.core.utils import import_class
 from rdagent.log import rdagent_logger as logger
 from rdagent.oai.llm_utils import APIBackend
 from rdagent.scenarios.qlib.developer.factor_coder import QlibFactorCoSTEER
-from rdagent.scenarios.qlib.experiment.factor_experiment import QlibFactorScenario, QlibFactorExperiment
+from rdagent.scenarios.qlib.experiment.factor_experiment import (
+    QlibFactorExperiment,
+    QlibFactorScenario,
+)
 from rdagent.scenarios.qlib.factor_experiment_loader.pdf_loader import (
     FactorExperimentLoaderFromPDFfiles,
     classify_report_from_dict,
 )
 
-from rdagent.core.proposal import (
-    Hypothesis2Experiment,
-    HypothesisExperiment2Feedback,
-    HypothesisGen,
-    Hypothesis,
-    Trace,
-)
-
-from rdagent.core.developer import Developer
-
 assert load_dotenv()
 
 scen: Scenario = import_class(PROP_SETTING.factor_scen)()
@@ -43,17 +47,21 @@ qlib_factor_runner: Developer = import_class(PROP_SETTING.factor_runner)(scen)
 
 qlib_factor_summarizer: HypothesisExperiment2Feedback = import_class(PROP_SETTING.factor_summarizer)(scen)
 
-with open(PROP_SETTING.report_result_json_file_path, 'r') as f:
+with open(PROP_SETTING.report_result_json_file_path, "r") as f:
     judge_pdf_data = json.load(f)
 
 prompts_path = Path(__file__).parent / "prompts.yaml"
 prompts = Prompts(file_path=prompts_path)
 
+
 def generate_hypothesis(factor_result: dict, report_content: str) -> str:
-    system_prompt = Environment(undefined=StrictUndefined).from_string(prompts["hypothesis_generation"]["system"]).render()
-    user_prompt = Environment(undefined=StrictUndefined).from_string(prompts["hypothesis_generation"]["user"]).render(
-        factor_descriptions=json.dumps(factor_result),
-        report_content=report_content
+    system_prompt = (
+        Environment(undefined=StrictUndefined).from_string(prompts["hypothesis_generation"]["system"]).render()
+    )
+    user_prompt = (
+        Environment(undefined=StrictUndefined)
+        .from_string(prompts["hypothesis_generation"]["user"])
+        .render(factor_descriptions=json.dumps(factor_result), report_content=report_content)
     )
 
     response = APIBackend().build_messages_and_create_chat_completion(
@@ -68,16 +76,16 @@ def generate_hypothesis(factor_result: dict, report_content: str) -> str:
 
     return Hypothesis(hypothesis=hypothesis_text, reason=reason_text)
 
+
 def extract_factors_and_implement(report_file_path: str) -> tuple:
     scenario = QlibFactorScenario()
 
     with logger.tag("extract_factors_and_implement"):
         with logger.tag("load_factor_tasks"):
-
             exp = FactorExperimentLoaderFromPDFfiles().load(report_file_path)
             if exp is None or exp.sub_tasks == []:
                 return None, None
-            
+
     docs_dict = load_and_process_pdfs_by_langchain(Path(report_file_path))
 
     factor_result = {
@@ -85,7 +93,7 @@ def extract_factors_and_implement(report_file_path: str) -> tuple:
             "description": task.factor_description,
             "formulation": task.factor_formulation,
             "variables": task.variables,
-            "resources": task.factor_resources
+            "resources": task.factor_resources,
         }
         for task in exp.sub_tasks
     }
@@ -95,6 +103,7 @@ def extract_factors_and_implement(report_file_path: str) -> tuple:
 
     return exp, hypothesis
 
+
 trace = Trace(scen=scen)
 
 for file_path, attributes in judge_pdf_data.items():

rdagent/app/qlib_rd_loop/factor_from_report_sh.py

@@ -1,35 +1,40 @@
 # TODO: we should have more advanced mechanism to handle such requirements for saving sessions.
 import json
-from pathlib import Path
 import pickle
+from pathlib import Path
+
+import pandas as pd
 from dotenv import load_dotenv
 from jinja2 import Environment, StrictUndefined
-import pandas as pd
 
 from rdagent.app.qlib_rd_loop.conf import PROP_SETTING
-from rdagent.components.document_reader.document_reader import extract_first_page_screenshot_from_pdf, load_and_process_pdfs_by_langchain
+from rdagent.components.document_reader.document_reader import (
+    extract_first_page_screenshot_from_pdf,
+    load_and_process_pdfs_by_langchain,
+)
+from rdagent.core.developer import Developer
 from rdagent.core.prompts import Prompts
+from rdagent.core.proposal import (
+    Hypothesis,
+    Hypothesis2Experiment,
+    HypothesisExperiment2Feedback,
+    HypothesisGen,
+    Trace,
+)
 from rdagent.core.scenario import Scenario
 from rdagent.core.utils import import_class
 from rdagent.log import rdagent_logger as logger
 from rdagent.oai.llm_utils import APIBackend
 from rdagent.scenarios.qlib.developer.factor_coder import QlibFactorCoSTEER
-from rdagent.scenarios.qlib.experiment.factor_experiment import QlibFactorScenario, QlibFactorExperiment
+from rdagent.scenarios.qlib.experiment.factor_experiment import (
+    QlibFactorExperiment,
+    QlibFactorScenario,
+)
 from rdagent.scenarios.qlib.factor_experiment_loader.pdf_loader import (
     FactorExperimentLoaderFromPDFfiles,
     classify_report_from_dict,
 )
 
-from rdagent.core.proposal import (
-    Hypothesis2Experiment,
-    HypothesisExperiment2Feedback,
-    HypothesisGen,
-    Hypothesis,
-    Trace,
-)
-
-from rdagent.core.developer import Developer
-
 assert load_dotenv()
 
 scen: Scenario = import_class(PROP_SETTING.factor_scen)()
@@ -44,27 +49,33 @@ qlib_factor_runner: Developer = import_class(PROP_SETTING.factor_runner)(scen)
 
 qlib_factor_summarizer: HypothesisExperiment2Feedback = import_class(PROP_SETTING.factor_summarizer)(scen)
 
-with open(PROP_SETTING.report_result_json_file_path, 'r') as f:
+with open(PROP_SETTING.report_result_json_file_path, "r") as f:
     judge_pdf_data = json.load(f)
 
 prompts_path = Path(__file__).parent / "prompts.yaml"
 prompts = Prompts(file_path=prompts_path)
 
+
 def save_progress(trace, current_index):
     with open(PROP_SETTING.progress_file_path, "wb") as f:
         pickle.dump((trace, current_index), f)
 
+
 def load_progress():
     if Path(PROP_SETTING.progress_file_path).exists():
         with open(PROP_SETTING.progress_file_path, "rb") as f:
             return pickle.load(f)
     return Trace(scen=scen), 0
 
+
 def generate_hypothesis(factor_result: dict, report_content: str) -> str:
-    system_prompt = Environment(undefined=StrictUndefined).from_string(prompts["hypothesis_generation"]["system"]).render()
-    user_prompt = Environment(undefined=StrictUndefined).from_string(prompts["hypothesis_generation"]["user"]).render(
-        factor_descriptions=json.dumps(factor_result),
-        report_content=report_content
+    system_prompt = (
+        Environment(undefined=StrictUndefined).from_string(prompts["hypothesis_generation"]["system"]).render()
+    )
+    user_prompt = (
+        Environment(undefined=StrictUndefined)
+        .from_string(prompts["hypothesis_generation"]["user"])
+        .render(factor_descriptions=json.dumps(factor_result), report_content=report_content)
     )
 
     response = APIBackend().build_messages_and_create_chat_completion(
@@ -79,12 +90,12 @@ def generate_hypothesis(factor_result: dict, report_content: str) -> str:
 
     return Hypothesis(hypothesis=hypothesis_text, reason=reason_text)
 
+
 def extract_factors_and_implement(report_file_path: str) -> tuple:
     scenario = QlibFactorScenario()
 
     with logger.tag("extract_factors_and_implement"):
         with logger.tag("load_factor_tasks"):
-
             exp = FactorExperimentLoaderFromPDFfiles().load(report_file_path)
             if exp is None or exp.sub_tasks == []:
                 return None, None
@@ -100,7 +111,7 @@ def extract_factors_and_implement(report_file_path: str) -> tuple:
             "description": task.factor_description,
             "formulation": task.factor_formulation,
             "variables": task.variables,
-            "resources": task.factor_resources
+            "resources": task.factor_resources,
         }
         for task in exp.sub_tasks
     }
@@ -110,6 +121,7 @@ def extract_factors_and_implement(report_file_path: str) -> tuple:
 
     return exp, hypothesis
 
+
 trace, start_index = load_progress()
 
 try:
@@ -122,7 +134,7 @@ try:
             report_file_path = Path(file_path.replace(PROP_SETTING.origin_report_path, PROP_SETTING.local_report_path))
             if report_file_path.exists():
                 logger.info(f"Processing {report_file_path}")
-                
+
                 with logger.tag("r"):
                     exp, hypothesis = extract_factors_and_implement(str(report_file_path))
                     if exp is None:
@@ -132,7 +144,7 @@ try:
                         exp.based_experiments.append(QlibFactorExperiment(sub_tasks=[]))
                     logger.log_object(hypothesis, tag="hypothesis generation")
                     logger.log_object(exp.sub_tasks, tag="experiment generation")
-                
+
                 with logger.tag("d"):
                     exp = qlib_factor_coder.develop(exp)
                     logger.log_object(exp.sub_workspace_list)
@@ -145,10 +157,10 @@ try:
                     logger.log_object(exp, tag="factor runner result")
                     feedback = qlib_factor_summarizer.generate_feedback(exp, hypothesis, trace)
                     logger.log_object(feedback, tag="feedback")
-                
+
                 trace.hist.append((hypothesis, exp, feedback))
                 logger.info(f"Processed {report_file_path}: Result: {exp}")
-                
+
                 # Save progress after processing each report
                 save_progress(trace, index + 1)
             else:

rdagent/app/qlib_rd_loop/factor_w_sc.py

@@ -30,4 +30,4 @@ def main(path=None, step_n=None):
 
 
 if __name__ == "__main__":
-    fire.Fire(main)
+    fire.Fire(main)

rdagent/app/quant_factor_benchmark/analysis.py

@@ -1,12 +1,14 @@
 import json
 import pickle
-import pandas as pd
 from pathlib import Path
+
 import matplotlib.pyplot as plt
+import pandas as pd
 import seaborn as sns
 
-from rdagent.components.benchmark.eval_method import FactorImplementEval
 from rdagent.components.benchmark.conf import BenchmarkSettings
+from rdagent.components.benchmark.eval_method import FactorImplementEval
+
 
 class BenchmarkAnalyzer:
     def __init__(self, settings):
@@ -25,10 +27,10 @@ class BenchmarkAnalyzer:
         file_path = Path(file_path)
         if not (file_path.is_file() and file_path.suffix == ".pkl"):
             raise ValueError("Invalid file path")
-        
+
         with file_path.open("rb") as f:
             res = pickle.load(f)
-        
+
         return res
 
     def process_results(self, results):
@@ -39,7 +41,7 @@ class BenchmarkAnalyzer:
             processed_data = self.analyze_data(summarized_data)
             final_res[experiment] = processed_data.iloc[-1, :]
         return final_res
-    
+
     def reformat_succ_rate(self, display_df):
         new_idx = []
         display_df = display_df[display_df.index.isin(self.index_map.keys())]
@@ -52,8 +54,10 @@ class BenchmarkAnalyzer:
         )
         display_df = display_df.swaplevel(0, 2).swaplevel(0, 1).sort_index(axis=0)
 
-        return display_df.sort_index(key=lambda x: [{"Easy": 0, "Medium": 1, "Hard": 2, "New Discovery": 3}.get(i, i) for i in x])
-    
+        return display_df.sort_index(
+            key=lambda x: [{"Easy": 0, "Medium": 1, "Hard": 2, "New Discovery": 3}.get(i, i) for i in x]
+        )
+
     def result_all_key_order(self, x):
         order_v = []
         for i in x:
@@ -92,9 +96,7 @@ class BenchmarkAnalyzer:
 
         sum_df_clean["FactorRowCountEvaluator"]
 
-        format_issue = (
-            sum_df_clean["FactorRowCountEvaluator"] & sum_df_clean["FactorIndexEvaluator"]
-        )
+        format_issue = sum_df_clean["FactorRowCountEvaluator"] & sum_df_clean["FactorIndexEvaluator"]
         eval_series = format_issue.unstack()
         succ_rate = eval_series.T.fillna(False).astype(bool)  # false indicate failure
         format_succ_rate = succ_rate.mean(axis=0).to_frame("success rate")
@@ -113,10 +115,7 @@ class BenchmarkAnalyzer:
         value_max_res = self.reformat_succ_rate(value_max)
 
         value_avg = (
-            (sum_df_clean["FactorMissingValuesEvaluator"] * format_issue)
-            .unstack()
-            .T.mean(axis=0)
-            .to_frame("avg_value")
+            (sum_df_clean["FactorMissingValuesEvaluator"] * format_issue).unstack().T.mean(axis=0).to_frame("avg_value")
         )
         value_avg_res = self.reformat_succ_rate(value_avg)
 
@@ -148,7 +147,6 @@ class BenchmarkAnalyzer:
         return df_w_mean
 
 
-
 class Plotter:
     @staticmethod
     def change_fs(font_size):
@@ -169,6 +167,7 @@ class Plotter:
         plt.title("Comparison of Different Methods")
         plt.savefig(file_name)
 
+
 if __name__ == "__main__":
     settings = BenchmarkSettings()
     benchmark = BenchmarkAnalyzer(settings)

rdagent/app/quant_factor_benchmark/eval.py

@@ -1,23 +1,20 @@
 import os
-from pathlib import Path
 import pickle
 import time
+from pathlib import Path
+from pprint import pprint
+
 from rdagent.app.qlib_rd_loop.conf import PROP_SETTING
+from rdagent.components.benchmark.conf import BenchmarkSettings
+from rdagent.components.benchmark.eval_method import FactorImplementEval
+from rdagent.core.scenario import Scenario
+from rdagent.core.utils import import_class
 from rdagent.log import rdagent_logger as logger
+from rdagent.scenarios.qlib.experiment.factor_experiment import QlibFactorScenario
 from rdagent.scenarios.qlib.factor_experiment_loader.json_loader import (
     FactorTestCaseLoaderFromJsonFile,
 )
 
-from rdagent.components.benchmark.conf import BenchmarkSettings
-from rdagent.components.benchmark.eval_method import FactorImplementEval
-from rdagent.core.utils import import_class
-
-from rdagent.core.utils import import_class
-from rdagent.core.scenario import Scenario
-from rdagent.scenarios.qlib.experiment.factor_experiment import QlibFactorScenario
-
-from pprint import pprint
-
 # 1.read the settings
 bs = BenchmarkSettings()
 
@@ -28,7 +25,7 @@ test_cases = FactorTestCaseLoaderFromJsonFile().load(bs.bench_data_path)
 
 scen: Scenario = import_class(PROP_SETTING.factor_scen)()
 generate_method = import_class(bs.bench_method_cls)(scen=scen)
- 
+
 # 4.declare the eval method and pass the arguments.
 eval_method = FactorImplementEval(
     method=generate_method,