fix: first round app folder cleaning (#166)

* first round app folder cleaning

* fix CI

rdagent/app/factor_extraction_and_code/factor_extract_and_implement.py

@@ -1,30 +0,0 @@
-# %%
-from dotenv import load_dotenv
-
-from rdagent.log import rdagent_logger as logger
-from rdagent.scenarios.qlib.developer.factor_coder import QlibFactorCoSTEER
-from rdagent.scenarios.qlib.experiment.factor_from_report_experiment import (
-    QlibFactorFromReportScenario,
-)
-from rdagent.scenarios.qlib.factor_experiment_loader.pdf_loader import (
-    FactorExperimentLoaderFromPDFfiles,
-)
-
-assert load_dotenv()
-
-
-def extract_factors_and_implement(report_file_path: str) -> None:
-    scenario = QlibFactorFromReportScenario()
-
-    with logger.tag("extract_factors_and_implement"):
-        with logger.tag("load_factor_tasks"):
-            exp = FactorExperimentLoaderFromPDFfiles().load(report_file_path)
-        with logger.tag("implement_factors"):
-            exp = QlibFactorCoSTEER(scenario).develop(exp)
-
-    # Qlib to run the implementation in rd loop
-    return exp
-
-
-if __name__ == "__main__":
-    extract_factors_and_implement("workspace/report.pdf")

rdagent/app/qlib_rd_loop/conf.py

@@ -33,11 +33,7 @@ class FactorBasePropSetting(BasePropSetting):
     evolving_n: int = 10
 
     # 2) sub task specific:
-    origin_report_path: str = "data/report_origin"
-    local_report_path: str = "data/report"
-    report_result_json_file_path: str = "git_ignore_folder/report_list_new.json"
-    progress_file_path: str = "git_ignore_folder/progress.pkl"
-    report_extract_result: str = "git_ignore_folder/hypo_exp_cache.pkl"
+    report_result_json_file_path: str = "git_ignore_folder/report_list.json"
     max_factor_per_report: int = 10000
 
 

rdagent/app/qlib_rd_loop/factor.py

@@ -1,62 +0,0 @@
-"""
-Factor Structure RD-Loop
-"""
-
-from dotenv import load_dotenv
-
-from rdagent.core.exception import FactorEmptyError
-from rdagent.core.scenario import Scenario
-from rdagent.log import rdagent_logger as logger
-
-load_dotenv(override=True)
-
-from rdagent.app.qlib_rd_loop.conf import PROP_SETTING
-from rdagent.core.developer import Developer
-from rdagent.core.proposal import (
-    Hypothesis2Experiment,
-    HypothesisExperiment2Feedback,
-    HypothesisGen,
-    Trace,
-)
-from rdagent.core.utils import import_class
-
-scen: Scenario = import_class(PROP_SETTING.factor_scen)()
-
-hypothesis_gen: HypothesisGen = import_class(PROP_SETTING.factor_hypothesis_gen)(scen)
-
-hypothesis2experiment: Hypothesis2Experiment = import_class(PROP_SETTING.factor_hypothesis2experiment)()
-
-qlib_factor_coder: Developer = import_class(PROP_SETTING.factor_coder)(scen)
-
-qlib_factor_runner: Developer = import_class(PROP_SETTING.factor_runner)(scen)
-
-qlib_factor_summarizer: HypothesisExperiment2Feedback = import_class(PROP_SETTING.factor_summarizer)(scen)
-
-
-trace = Trace(scen=scen)
-for _ in range(PROP_SETTING.evolving_n):
-    try:
-        with logger.tag("r"):
-            hypothesis = hypothesis_gen.gen(trace)
-            logger.log_object(hypothesis, tag="hypothesis generation")
-
-            exp = hypothesis2experiment.convert(hypothesis, trace)
-            logger.log_object(exp.sub_tasks, tag="experiment generation")
-
-        with logger.tag("d"):
-            exp = qlib_factor_coder.develop(exp)
-            logger.log_object(exp.sub_workspace_list)
-
-        with logger.tag("ef"):
-            exp = qlib_factor_runner.develop(exp)
-            if exp is None:
-                logger.error(f"Factor extraction failed.")
-                continue
-            logger.log_object(exp, tag="factor runner result")
-            feedback = qlib_factor_summarizer.generate_feedback(exp, hypothesis, trace)
-            logger.log_object(feedback, tag="feedback")
-
-        trace.hist.append((hypothesis, exp, feedback))
-    except FactorEmptyError as e:
-        logger.warning(e)
-        continue

rdagent/app/qlib_rd_loop/factor_from_report.py

@@ -1,130 +0,0 @@
-import json
-import pickle
-from pathlib import Path
-
-import pandas as pd
-from dotenv import load_dotenv
-from jinja2 import Environment, StrictUndefined
-
-from rdagent.app.qlib_rd_loop.conf import PROP_SETTING
-from rdagent.components.document_reader.document_reader import (
-    load_and_process_pdfs_by_langchain,
-)
-from rdagent.core.developer import Developer
-from rdagent.core.prompts import Prompts
-from rdagent.core.proposal import (
-    Hypothesis,
-    Hypothesis2Experiment,
-    HypothesisExperiment2Feedback,
-    HypothesisGen,
-    Trace,
-)
-from rdagent.core.scenario import Scenario
-from rdagent.core.utils import import_class
-from rdagent.log import rdagent_logger as logger
-from rdagent.oai.llm_utils import APIBackend
-from rdagent.scenarios.qlib.developer.factor_coder import QlibFactorCoSTEER
-from rdagent.scenarios.qlib.experiment.factor_experiment import (
-    QlibFactorExperiment,
-    QlibFactorScenario,
-)
-from rdagent.scenarios.qlib.factor_experiment_loader.pdf_loader import (
-    FactorExperimentLoaderFromPDFfiles,
-    classify_report_from_dict,
-)
-
-assert load_dotenv()
-
-scen: Scenario = import_class(PROP_SETTING.factor_scen)()
-
-hypothesis_gen: HypothesisGen = import_class(PROP_SETTING.factor_hypothesis_gen)(scen)
-
-hypothesis2experiment: Hypothesis2Experiment = import_class(PROP_SETTING.factor_hypothesis2experiment)()
-
-qlib_factor_coder: Developer = import_class(PROP_SETTING.factor_coder)(scen)
-
-qlib_factor_runner: Developer = import_class(PROP_SETTING.factor_runner)(scen)
-
-qlib_factor_summarizer: HypothesisExperiment2Feedback = import_class(PROP_SETTING.factor_summarizer)(scen)
-
-with open(PROP_SETTING.report_result_json_file_path, "r") as f:
-    judge_pdf_data = json.load(f)
-
-prompts_path = Path(__file__).parent / "prompts.yaml"
-prompts = Prompts(file_path=prompts_path)
-
-
-def generate_hypothesis(factor_result: dict, report_content: str) -> str:
-    system_prompt = (
-        Environment(undefined=StrictUndefined).from_string(prompts["hypothesis_generation"]["system"]).render()
-    )
-    user_prompt = (
-        Environment(undefined=StrictUndefined)
-        .from_string(prompts["hypothesis_generation"]["user"])
-        .render(factor_descriptions=json.dumps(factor_result), report_content=report_content)
-    )
-
-    response = APIBackend().build_messages_and_create_chat_completion(
-        user_prompt=user_prompt,
-        system_prompt=system_prompt,
-        json_mode=True,
-    )
-
-    response_json = json.loads(response)
-    hypothesis_text = response_json.get("hypothesis", "No hypothesis generated.")
-    reason_text = response_json.get("reason", "No reason provided.")
-
-    return Hypothesis(hypothesis=hypothesis_text, reason=reason_text)
-
-
-def extract_factors_and_implement(report_file_path: str) -> tuple:
-    scenario = QlibFactorScenario()
-
-    with logger.tag("extract_factors_and_implement"):
-        with logger.tag("load_factor_tasks"):
-            exp = FactorExperimentLoaderFromPDFfiles().load(report_file_path)
-            if exp is None or exp.sub_tasks == []:
-                return None, None
-
-    docs_dict = load_and_process_pdfs_by_langchain(Path(report_file_path))
-
-    factor_result = {
-        task.factor_name: {
-            "description": task.factor_description,
-            "formulation": task.factor_formulation,
-            "variables": task.variables,
-            "resources": task.factor_resources,
-        }
-        for task in exp.sub_tasks
-    }
-
-    report_content = "\n".join(docs_dict.values())
-    hypothesis = generate_hypothesis(factor_result, report_content)
-
-    return exp, hypothesis
-
-
-trace = Trace(scen=scen)
-
-for file_path, attributes in judge_pdf_data.items():
-    if attributes["class"] == 1:
-        report_file_path = Path(file_path.replace(PROP_SETTING.origin_report_path, PROP_SETTING.local_report_path))
-        if report_file_path.exists():
-            logger.info(f"Processing {report_file_path}")
-            exp, hypothesis = extract_factors_and_implement(str(report_file_path))
-            if exp is None:
-                continue
-            exp.based_experiments = [t[1] for t in trace.hist if t[2]]
-            if len(exp.based_experiments) == 0:
-                exp.based_experiments.append(QlibFactorExperiment(sub_tasks=[]))
-            exp = qlib_factor_coder.develop(exp)
-            exp = qlib_factor_runner.develop(exp)
-            if exp is None:
-                logger.error(f"Factor extraction failed for {report_file_path}. Skipping to the next report.")
-                continue
-            feedback = qlib_factor_summarizer.generate_feedback(exp, hypothesis, trace)
-
-            trace.hist.append((hypothesis, exp, feedback))
-            logger.info(f"Processed {report_file_path}: Result: {exp}")
-        else:
-            logger.error(f"File not found: {report_file_path}")

rdagent/app/qlib_rd_loop/factor_from_report_w_sc.py

@@ -6,12 +6,13 @@ from typing import Any, Tuple
 import fire
 from jinja2 import Environment, StrictUndefined
 
-from rdagent.app.qlib_rd_loop.conf import FACTOR_FROM_REPORT_PROP_SETTING
+from rdagent.app.qlib_rd_loop.conf import FactorBasePropSetting
 from rdagent.components.document_reader.document_reader import (
     extract_first_page_screenshot_from_pdf,
     load_and_process_pdfs_by_langchain,
 )
 from rdagent.components.workflow.conf import BasePropSetting
+from rdagent.components.workflow.rd_loop import RDLoop
 from rdagent.core.developer import Developer
 from rdagent.core.exception import FactorEmptyError
 from rdagent.core.prompts import Prompts
@@ -62,8 +63,6 @@ def generate_hypothesis(factor_result: dict, report_content: str) -> str:
 
 
 def extract_hypothesis_and_exp_from_reports(report_file_path: str) -> Tuple[QlibFactorExperiment, Hypothesis]:
-    scenario = QlibFactorFromReportScenario()
-
     with logger.tag("extract_factors_and_implement"):
         with logger.tag("load_factor_tasks"):
             exp = FactorExperimentLoaderFromPDFfiles().load(report_file_path)
@@ -88,30 +87,22 @@ def extract_hypothesis_and_exp_from_reports(report_file_path: str) -> Tuple[Qlib
 
     report_content = "\n".join(docs_dict.values())
     hypothesis = generate_hypothesis(factor_result, report_content)
-
     return exp, hypothesis
 
 
-class FactorReportLoop(LoopBase, metaclass=LoopMeta):
+class FactorReportLoop(RDLoop, metaclass=LoopMeta):
     skip_loop_error = (FactorEmptyError,)
 
-    def __init__(self, PROP_SETTING: BasePropSetting):
-        scen: Scenario = import_class(PROP_SETTING.scen)()
-
-        self.coder: Developer = import_class(PROP_SETTING.coder)(scen)
-        self.runner: Developer = import_class(PROP_SETTING.runner)(scen)
-
-        self.summarizer: HypothesisExperiment2Feedback = import_class(PROP_SETTING.summarizer)(scen)
-        self.trace = Trace(scen=scen)
-
-        self.judge_pdf_data_items = json.load(open(FACTOR_FROM_REPORT_PROP_SETTING.report_result_json_file_path, "r"))
+    def __init__(self, PROP_SETTING: FactorBasePropSetting):
+        self.judge_pdf_data_items = json.load(open(PROP_SETTING.report_result_json_file_path, "r"))
         self.pdf_file_index = 0
-        super().__init__()
+        self.valid_pdf_file_count = 0
+        super().__init__(PROP_SETTING=PROP_SETTING)
 
     def propose_hypo_exp(self, prev_out: dict[str, Any]):
         with logger.tag("r"):
             while True:
-                if self.pdf_file_index > 100:
+                if self.valid_pdf_file_count > 15:
                     break
                 report_file_path = self.judge_pdf_data_items[self.pdf_file_index]
                 logger.info(f"Processing number {self.pdf_file_index} report: {report_file_path}")
@@ -119,33 +110,21 @@ class FactorReportLoop(LoopBase, metaclass=LoopMeta):
                 exp, hypothesis = extract_hypothesis_and_exp_from_reports(str(report_file_path))
                 if exp is None:
                     continue
+                self.valid_pdf_file_count += 1
                 exp.based_experiments = [QlibFactorExperiment(sub_tasks=[])] + [t[1] for t in self.trace.hist if t[2]]
                 exp.sub_workspace_list = exp.sub_workspace_list[: FACTOR_FROM_REPORT_PROP_SETTING.max_factor_per_report]
                 exp.sub_tasks = exp.sub_tasks[: FACTOR_FROM_REPORT_PROP_SETTING.max_factor_per_report]
                 logger.log_object(hypothesis, tag="hypothesis generation")
                 logger.log_object(exp.sub_tasks, tag="experiment generation")
-                return hypothesis, exp
+                self.current_loop_hypothesis = hypothesis
+                self.current_loop_exp = exp
+                return None
 
-    def coding(self, prev_out: dict[str, Any]):
-        with logger.tag("d"):  # develop
-            exp = self.coder.develop(prev_out["propose_hypo_exp"][1])
-            logger.log_object(exp.sub_workspace_list, tag="coder result")
-        return exp
+    def propose(self, prev_out: dict[str, Any]):
+        return self.current_loop_hypothesis
 
-    def running(self, prev_out: dict[str, Any]):
-        with logger.tag("ef"):  # evaluate and feedback
-            exp = self.runner.develop(prev_out["coding"])
-            if exp is None:
-                logger.error(f"Factor extraction failed.")
-                raise FactorEmptyError("Factor extraction failed.")
-            logger.log_object(exp, tag="runner result")
-        return exp
-
-    def feedback(self, prev_out: dict[str, Any]):
-        feedback = self.summarizer.generate_feedback(prev_out["running"], prev_out["propose_hypo_exp"][0], self.trace)
-        with logger.tag("ef"):  # evaluate and feedback
-            logger.log_object(feedback, tag="feedback")
-        self.trace.hist.append((prev_out["propose_hypo_exp"][0], prev_out["running"], feedback))
+    def exp_gen(self, prev_out: dict[str, Any]):
+        return self.current_loop_exp
 
 
 def main(path=None, step_n=None):

rdagent/app/qlib_rd_loop/general_model.py

@@ -1,11 +1,12 @@
 # %%
 from dotenv import load_dotenv
 
+from rdagent.scenarios.general_model.scenario import GeneralModelScenario
+
 load_dotenv(override=True)
 
 import fire
 
-from rdagent.app.model_extraction_and_code.GeneralModel import GeneralModelScenario
 from rdagent.components.coder.model_coder.task_loader import (
     ModelExperimentLoaderFromPDFfiles,
 )
@@ -17,17 +18,18 @@ from rdagent.scenarios.qlib.developer.model_coder import QlibModelCoSTEER
 
 
 def extract_models_and_implement(
-    report_file_path: str = "/home/v-xisenwang/RD-Agent/rdagent/app/model_extraction_and_code/test_doc1.pdf",
+    report_file_path: str,
 ) -> None:
+    with logger.tag("init"):
+        scenario = GeneralModelScenario()
+        logger.log_object(scenario, tag="scenario")
     with logger.tag("r"):
         # Save Relevant Images
         img = extract_first_page_screenshot_from_pdf(report_file_path)
         logger.log_object(img, tag="pdf_image")
-        scenario = GeneralModelScenario()
-        logger.log_object(scenario, tag="scenario")
-    with logger.tag("d"):
         exp = ModelExperimentLoaderFromPDFfiles().load(report_file_path)
         logger.log_object(exp, tag="load_experiment")
+    with logger.tag("d"):
         exp = QlibModelCoSTEER(scenario).develop(exp)
         logger.log_object(exp, tag="developed_experiment")
     return exp

rdagent/app/qlib_rd_loop/model.py

@@ -1,53 +0,0 @@
-"""
-TODO: Model Structure RD-Loop
-TODO: move the following code to a new class: Model_RD_Agent
-"""
-
-# import_from
-
-from rdagent.app.qlib_rd_loop.conf import PROP_SETTING
-from rdagent.core.developer import Developer
-from rdagent.core.exception import ModelEmptyError
-from rdagent.core.proposal import (
-    Hypothesis2Experiment,
-    HypothesisExperiment2Feedback,
-    HypothesisGen,
-    Trace,
-)
-from rdagent.core.scenario import Scenario
-from rdagent.core.utils import import_class
-from rdagent.log import rdagent_logger as logger
-
-scen: Scenario = import_class(PROP_SETTING.model_scen)()
-
-hypothesis_gen: HypothesisGen = import_class(PROP_SETTING.model_hypothesis_gen)(scen)
-
-hypothesis2experiment: Hypothesis2Experiment = import_class(PROP_SETTING.model_hypothesis2experiment)()
-
-qlib_model_coder: Developer = import_class(PROP_SETTING.model_coder)(scen)
-qlib_model_runner: Developer = import_class(PROP_SETTING.model_runner)(scen)
-
-qlib_model_summarizer: HypothesisExperiment2Feedback = import_class(PROP_SETTING.model_summarizer)(scen)
-
-trace = Trace(scen=scen)
-with logger.tag("model.loop"):
-    for _ in range(PROP_SETTING.evolving_n):
-        try:
-            with logger.tag("r"):  # research
-                hypothesis = hypothesis_gen.gen(trace)
-                logger.log_object(hypothesis, tag="hypothesis generation")
-
-                exp = hypothesis2experiment.convert(hypothesis, trace)
-                logger.log_object(exp.sub_tasks, tag="experiment generation")
-            with logger.tag("d"):  # develop
-                exp = qlib_model_coder.develop(exp)
-                logger.log_object(exp.sub_workspace_list, tag="model coder result")
-            with logger.tag("ef"):  # evaluate and feedback
-                exp = qlib_model_runner.develop(exp)
-                logger.log_object(exp, tag="model runner result")
-                feedback = qlib_model_summarizer.generate_feedback(exp, hypothesis, trace)
-                logger.log_object(feedback, tag="feedback")
-            trace.hist.append((hypothesis, exp, feedback))
-        except ModelEmptyError as e:
-            logger.warning(e)
-            continue

rdagent/components/workflow/conf.py

@@ -3,7 +3,7 @@ from pydantic_settings import BaseSettings
 
 class BasePropSetting(BaseSettings):
     """
-    The common part of the config for RD Loop to propose and developement
+    The common part of the config for RD Loop to propose and development
     You can add following config in the subclass to distinguish the environment variables.
 
     .. code-block:: python

rdagent/components/workflow/rd_loop.py

@@ -21,18 +21,25 @@ from rdagent.utils.workflow import LoopBase, LoopMeta
 
 class RDLoop(LoopBase, metaclass=LoopMeta):
     def __init__(self, PROP_SETTING: BasePropSetting):
-        scen: Scenario = import_class(PROP_SETTING.scen)()
+        with logger.tag("init"):
+            scen: Scenario = import_class(PROP_SETTING.scen)()
+            logger.log_object(scen, tag="scenario")
 
-        self.hypothesis_gen: HypothesisGen = import_class(PROP_SETTING.hypothesis_gen)(scen)
+            self.hypothesis_gen: HypothesisGen = import_class(PROP_SETTING.hypothesis_gen)(scen)
+            logger.log_object(self.hypothesis_gen, tag="hypothesis generator")
 
-        self.hypothesis2experiment: Hypothesis2Experiment = import_class(PROP_SETTING.hypothesis2experiment)()
+            self.hypothesis2experiment: Hypothesis2Experiment = import_class(PROP_SETTING.hypothesis2experiment)()
+            logger.log_object(self.hypothesis2experiment, tag="hypothesis2experiment")
 
-        self.coder: Developer = import_class(PROP_SETTING.coder)(scen)
-        self.runner: Developer = import_class(PROP_SETTING.runner)(scen)
+            self.coder: Developer = import_class(PROP_SETTING.coder)(scen)
+            logger.log_object(self.coder, tag="coder")
+            self.runner: Developer = import_class(PROP_SETTING.runner)(scen)
+            logger.log_object(self.runner, tag="runner")
 
-        self.summarizer: HypothesisExperiment2Feedback = import_class(PROP_SETTING.summarizer)(scen)
-        self.trace = Trace(scen=scen)
-        super().__init__()
+            self.summarizer: HypothesisExperiment2Feedback = import_class(PROP_SETTING.summarizer)(scen)
+            logger.log_object(self.summarizer, tag="summarizer")
+            self.trace = Trace(scen=scen)
+            super().__init__()
 
     def propose(self, prev_out: dict[str, Any]):
         with logger.tag("r"):  # research

rdagent/log/ui/app.py

@@ -14,7 +14,6 @@ from st_btn_select import st_btn_select
 from streamlit import session_state as state
 from streamlit.delta_generator import DeltaGenerator
 
-from rdagent.app.model_extraction_and_code.GeneralModel import GeneralModelScenario
 from rdagent.components.coder.factor_coder.CoSTEER.evaluators import (
     FactorSingleFeedback,
 )
@@ -26,6 +25,7 @@ from rdagent.log.base import Message
 from rdagent.log.storage import FileStorage
 from rdagent.log.ui.qlib_report_figure import report_figure
 from rdagent.scenarios.data_mining.experiment.model_experiment import DMModelScenario
+from rdagent.scenarios.general_model.scenario import GeneralModelScenario
 from rdagent.scenarios.qlib.experiment.factor_experiment import (
     QlibFactorExperiment,
     QlibFactorScenario,

rdagent/scenarios/general_model/scenario.py

@@ -1,6 +1,5 @@
 from pathlib import Path
 
-from rdagent.components.coder.model_coder.model import ModelExperiment
 from rdagent.core.prompts import Prompts
 from rdagent.core.scenario import Scenario
 

test/utils/env_tpl/read_exp.py

@@ -8,7 +8,7 @@ from qlib.workflow import R
 
 # here is the documents of the https://qlib.readthedocs.io/en/latest/component/recorder.html
 
-# TODO: list all the recorder and metrics 
+# TODO: list all the recorder and metrics
 
 # Assuming you have already listed the experiments
 experiments = R.list_experiments()
@@ -30,7 +30,7 @@ for experiment in experiments:
 # TODO: get the latest recorder
 
 recorder_list = R.list_recorders(experiment_name="workflow")
-end_times = {key: value.info['end_time'] for key, value in recorder_list.items()}
+end_times = {key: value.info["end_time"] for key, value in recorder_list.items()}
 sorted_end_times = dict(sorted(end_times.items(), key=lambda item: item[1], reverse=True))
 
 latest_recorder_id = next(iter(sorted_end_times))