From faa2fb03ade3a5a13d662f2df444364433b6dbc7 Mon Sep 17 00:00:00 2001 From: Linlang <30293408+SunsetWolf@users.noreply.github.com> Date: Fri, 26 Jul 2024 12:12:16 +0800 Subject: [PATCH] CI checks that can be automatically repaired (#119) * fix isort & black & toml-sort & sphinx error * fix ci error * fix ci error * add comments * Update Makefile * change sphinx build command * add auto-lint * add black args * format with black * Auto Linting document * fix ci error --------- Co-authored-by: you-n-g Co-authored-by: Young --- .github/workflows/ci.yml | 7 +- Makefile | 29 +- docs/conf.py | 2 +- docs/development.rst | 8 +- docs/index.rst | 1 + docs/installation.rst | 15 +- docs/scenarios_and_quickstart.rst | 4 +- pyproject.toml | 7 +- rdagent/app/data_mining/conf.py | 7 +- rdagent/app/data_mining/model.py | 2 + rdagent/app/qlib_rd_loop/RDAgent.py | 30 +- .../app/qlib_rd_loop/factor_from_report.py | 53 ++- .../app/qlib_rd_loop/factor_from_report_sh.py | 62 +-- rdagent/app/qlib_rd_loop/factor_w_sc.py | 2 +- .../app/quant_factor_benchmark/analysis.py | 29 +- rdagent/app/quant_factor_benchmark/eval.py | 21 +- rdagent/components/benchmark/eval_method.py | 3 +- .../coder/factor_coder/CoSTEER/evaluators.py | 14 +- .../factor_coder/CoSTEER/evolving_strategy.py | 8 +- .../components/coder/factor_coder/factor.py | 6 +- rdagent/components/coder/model_coder/model.py | 23 +- .../document_reader/document_reader.py | 10 +- rdagent/components/proposal/model_proposal.py | 2 +- rdagent/components/workflow/conf.py | 2 + rdagent/components/workflow/rd_loop.py | 6 +- rdagent/core/evolving_agent.py | 11 +- rdagent/core/proposal.py | 10 +- rdagent/core/utils.py | 2 +- rdagent/log/base.py | 12 +- rdagent/log/logger.py | 6 +- rdagent/log/storage.py | 17 +- rdagent/log/ui/app.py | 14 +- rdagent/log/ui/st_fixed_container.py | 2 +- rdagent/log/ui/web.py | 410 ++++++++++-------- rdagent/log/utils.py | 3 +- .../data_mining/developer/feedback.py | 2 +- .../data_mining/developer/model_runner.py | 1 - .../experiment/model_experiment.py | 8 +- .../experiment/model_template/train.py | 48 +- .../data_mining/experiment/workspace.py | 5 +- .../data_mining/proposal/model_proposal.py | 7 +- rdagent/scenarios/qlib/developer/feedback.py | 39 +- .../qlib/experiment/factor_experiment.py | 4 +- .../factor_template/read_exp_res.py | 2 +- .../qlib/experiment/model_experiment.py | 8 +- .../experiment/model_template/read_exp_res.py | 1 - .../factor_experiment_loader/json_loader.py | 4 +- .../factor_experiment_loader/pdf_loader.py | 13 +- .../qlib/proposal/factor_proposal.py | 12 +- .../scenarios/qlib/proposal/model_proposal.py | 12 +- rdagent/utils/agent/ret.py | 2 +- rdagent/utils/agent/tpl.py | 20 +- rdagent/utils/env.py | 21 +- rdagent/utils/workflow.py | 25 +- test/utils/test_agent_infra.py | 4 - test/utils/test_misc.py | 1 - 56 files changed, 604 insertions(+), 475 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f47df5cc..75e3c44a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,16 +20,17 @@ jobs: - run: env | sort - run: make dev - env: + # Two environment variables here can cause sphinx to fail to build in ci. + # CHAT_MAX_TOKENS: ${{ secrets.CHAT_MAX_TOKENS }} + # CHAT_TEMPERATURE: ${{ secrets.CHAT_TEMPERATURE }} CHAT_AZURE_API_BASE: ${{ secrets.CHAT_AZURE_API_BASE }} CHAT_AZURE_API_VERSION: ${{ secrets.CHAT_AZURE_API_VERSION }} - CHAT_MAX_TOKENS: ${{ secrets.CHAT_MAX_TOKENS }} CHAT_MODEL: ${{ secrets.CHAT_MODEL }} - CHAT_TEMPERATURE: ${{ secrets.CHAT_TEMPERATURE }} EMBEDDING_AZURE_API_BASE: ${{ secrets.CHAT_AZURE_API_BASE }} EMBEDDING_AZURE_API_VERSION: ${{ secrets.CHAT_AZURE_API_VERSION }} EMBEDDING_MODEL: ${{ secrets.EMBEDDING_MODEL }} name: lint test docs and build - run: make lint # test docs build sphinx + run: make lint docs-gen # test docs build strategy: matrix: python-version: diff --git a/Makefile b/Makefile index 2f1ce9d3..b3618567 100644 --- a/Makefile +++ b/Makefile @@ -81,10 +81,7 @@ constraints: deepclean # Check lint with black. black: - $(PIPRUN) python -m black --check . --extend-exclude test/scripts --extend-exclude git_ignore_folder -l 120 - -sphinx: - $(PIPRUN) sphinx-build -W --keep-going -b html ./docs _build + $(PIPRUN) python -m black --check --diff . --extend-exclude test/scripts --extend-exclude git_ignore_folder -l 120 # Check lint with isort. isort: @@ -107,13 +104,33 @@ toml-sort: $(PIPRUN) toml-sort --check pyproject.toml # Check lint with all linters. -# lint: black isort mypy ruff toml-sort -lint: mypy ruff +# Prioritize fixing isort, then black, otherwise you'll get weird and unfixable black errors. +# lint: mypy ruff +lint: mypy ruff isort black toml-sort # Run pre-commit with autofix against all files. pre-commit: pre-commit run --all-files +######################################################################################## +# Auto Lint +######################################################################################## + +# Auto lint with black. +auto-black: + $(PIPRUN) python -m black . --extend-exclude test/scripts --extend-exclude git_ignore_folder -l 120 + +# Auto lint with isort. +auto-isort: + $(PIPRUN) python -m isort . -s git_ignore_folder -s test/scripts + +# Auto lint with toml-sort. +auto-toml-sort: + $(PIPRUN) toml-sort pyproject.toml + +# Auto lint with all linters. +auto-lint: auto-isort auto-black auto-toml-sort + ######################################################################################## # Test ######################################################################################## diff --git a/docs/conf.py b/docs/conf.py index 426a57cd..65fb9388 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -16,7 +16,7 @@ author = "Microsoft" # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration extensions = [ - 'sphinx.ext.autodoc', + "sphinx.ext.autodoc", ] autodoc_member_order = "bysource" diff --git a/docs/development.rst b/docs/development.rst index 4ab01b06..837f5689 100644 --- a/docs/development.rst +++ b/docs/development.rst @@ -11,12 +11,18 @@ For Development make dev ``` -- Run linting and formatting. +- Run linting and checking. ```bash make lint ``` +- Some linting issues can be fixed automatically. We have added a command in the Makefile for easy use. + + ```bash + make auto-lint + ``` + Code Structure ========================= diff --git a/docs/index.rst b/docs/index.rst index 1bba0c85..a8abb5a6 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -17,6 +17,7 @@ Welcome to RDAgent's documentation! development api_reference policy + test_dropdown/test1 .. test_dropdown/test1 diff --git a/docs/installation.rst b/docs/installation.rst index b4852be3..d1adfabb 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -27,13 +27,16 @@ USE_AZURE_TOKEN_PROVIDER ### ☁️ Azure Configuration - Install Azure CLI: - ```sh - curl -L https://aka.ms/InstallAzureCli | bash - ``` + + ```sh + curl -L https://aka.ms/InstallAzureCli | bash + ``` + - Log in to Azure: - ```sh - az login --use-device-code - ``` + + ```sh + az login --use-device-code + ``` - `exit` and re-login to your environment (this step may not be necessary). diff --git a/docs/scenarios_and_quickstart.rst b/docs/scenarios_and_quickstart.rst index e264cb24..5969d94b 100644 --- a/docs/scenarios_and_quickstart.rst +++ b/docs/scenarios_and_quickstart.rst @@ -23,7 +23,7 @@ Scenario lists - Scnarios' demo & quick start -========================= +============================ Scen1 ----- @@ -69,7 +69,7 @@ TODO: Show some examples: Scen2: ------ +------ 📄 Research Report-Based Factor Extraction Scen2 Intro diff --git a/pyproject.toml b/pyproject.toml index 40ced362..209ad958 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ requires = [ "setuptools", "setuptools-scm", ] -root= "rdagent" +root = "rdagent" [project] authors = [ @@ -27,14 +27,13 @@ dynamic = [ ] keywords = [ "Autonomous Agents", - "Research and Development", "Large Language Models", + "Research and Development", ] name = "rdagent" readme = "README.md" requires-python = ">=3.8" - [project.urls] homepage = "https://github.com/microsoft/RD-Agent/" issue = "https://github.com/microsoft/RD-Agent/issues" @@ -52,13 +51,13 @@ color_output = true profile = "black" [tool.mypy] -explicit_package_bases = true check_untyped_defs = true disallow_any_unimported = true disallow_untyped_defs = true enable_error_code = [ "ignore-without-code", ] +explicit_package_bases = true warn_return_any = true warn_unused_ignores = true diff --git a/rdagent/app/data_mining/conf.py b/rdagent/app/data_mining/conf.py index df2280ff..1e54faed 100644 --- a/rdagent/app/data_mining/conf.py +++ b/rdagent/app/data_mining/conf.py @@ -1,6 +1,5 @@ from pathlib import Path - from rdagent.components.workflow.conf import BasePropSetting @@ -19,11 +18,11 @@ class PropSetting(BasePropSetting): evolving_n: int = 10 - # 2) Extra config for the scenario + # 2) Extra config for the scenario # physionet account # NOTE: You should apply the account in https://physionet.org/ - username: str = '' - password: str = '' + username: str = "" + password: str = "" PROP_SETTING = PropSetting() diff --git a/rdagent/app/data_mining/model.py b/rdagent/app/data_mining/model.py index 83b83459..1fb56db3 100644 --- a/rdagent/app/data_mining/model.py +++ b/rdagent/app/data_mining/model.py @@ -1,8 +1,10 @@ import fire + from rdagent.app.data_mining.conf import PROP_SETTING from rdagent.components.workflow.rd_loop import RDLoop from rdagent.core.exception import ModelEmptyError + class ModelRDLoop(RDLoop): skip_loop_error = (ModelEmptyError,) diff --git a/rdagent/app/qlib_rd_loop/RDAgent.py b/rdagent/app/qlib_rd_loop/RDAgent.py index 3cc2f9bd..73571a50 100644 --- a/rdagent/app/qlib_rd_loop/RDAgent.py +++ b/rdagent/app/qlib_rd_loop/RDAgent.py @@ -1,4 +1,5 @@ import pickle + from rdagent.app.qlib_rd_loop.conf import PROP_SETTING from rdagent.core.developer import Developer from rdagent.core.exception import ModelEmptyError @@ -12,6 +13,7 @@ from rdagent.core.scenario import Scenario from rdagent.core.utils import import_class from rdagent.log import rdagent_logger as logger + # TODO: we can design a workflow that can automatically save session and traceback in the future class Model_RD_Agent: def __init__(self): @@ -20,50 +22,55 @@ class Model_RD_Agent: self.hypothesis2experiment: Hypothesis2Experiment = import_class(PROP_SETTING.model_hypothesis2experiment)() self.qlib_model_coder: Developer = import_class(PROP_SETTING.model_coder)(self.scen) self.qlib_model_runner: Developer = import_class(PROP_SETTING.model_runner)(self.scen) - self.qlib_model_summarizer: HypothesisExperiment2Feedback = import_class(PROP_SETTING.model_summarizer)(self.scen) + self.qlib_model_summarizer: HypothesisExperiment2Feedback = import_class(PROP_SETTING.model_summarizer)( + self.scen + ) self.trace = Trace(scen=self.scen) def generate_hypothesis(self): hypothesis = self.hypothesis_gen.gen(self.trace) - self.dump_objects(hypothesis=hypothesis, trace=self.trace, filename='step_hypothesis.pkl') + self.dump_objects(hypothesis=hypothesis, trace=self.trace, filename="step_hypothesis.pkl") return hypothesis def convert_hypothesis(self, hypothesis): exp = self.hypothesis2experiment.convert(hypothesis, self.trace) - self.dump_objects(exp=exp, hypothesis=hypothesis, trace=self.trace, filename='step_experiment.pkl') + self.dump_objects(exp=exp, hypothesis=hypothesis, trace=self.trace, filename="step_experiment.pkl") return exp def generate_code(self, exp): exp = self.qlib_model_coder.develop(exp) - self.dump_objects(exp=exp, trace=self.trace, filename='step_code.pkl') + self.dump_objects(exp=exp, trace=self.trace, filename="step_code.pkl") return exp def run_experiment(self, exp): exp = self.qlib_model_runner.develop(exp) - self.dump_objects(exp=exp, trace=self.trace, filename='step_run.pkl') + self.dump_objects(exp=exp, trace=self.trace, filename="step_run.pkl") return exp def generate_feedback(self, exp, hypothesis): feedback = self.qlib_model_summarizer.generate_feedback(exp, hypothesis, self.trace) - self.dump_objects(exp=exp, hypothesis=hypothesis, feedback=feedback, trace=self.trace, filename='step_feedback.pkl') + self.dump_objects( + exp=exp, hypothesis=hypothesis, feedback=feedback, trace=self.trace, filename="step_feedback.pkl" + ) return feedback def append_to_trace(self, hypothesis, exp, feedback): self.trace.hist.append((hypothesis, exp, feedback)) - self.dump_objects(trace=self.trace, filename='step_trace.pkl') + self.dump_objects(trace=self.trace, filename="step_trace.pkl") - def dump_objects(self, exp=None, hypothesis=None, feedback=None, trace=None, filename='dumped_objects.pkl'): - with open(filename, 'wb') as f: + def dump_objects(self, exp=None, hypothesis=None, feedback=None, trace=None, filename="dumped_objects.pkl"): + with open(filename, "wb") as f: pickle.dump((exp, hypothesis, feedback, trace or self.trace), f) def load_objects(self, filename): - with open(filename, 'rb') as f: + with open(filename, "rb") as f: return pickle.load(f) + def process_steps(agent): # Load trace if available try: - _, _, _, trace = agent.load_objects('step_trace.pkl') + _, _, _, trace = agent.load_objects("step_trace.pkl") agent.trace = trace print(trace.get_sota_hypothesis_and_experiment()) except FileNotFoundError: @@ -99,6 +106,7 @@ def process_steps(agent): # # Step 6: Append to trace # agent.append_to_trace(hypothesis, exp, feedback) + if __name__ == "__main__": agent = Model_RD_Agent() process_steps(agent) diff --git a/rdagent/app/qlib_rd_loop/factor_from_report.py b/rdagent/app/qlib_rd_loop/factor_from_report.py index 4ea9410a..281f9606 100644 --- a/rdagent/app/qlib_rd_loop/factor_from_report.py +++ b/rdagent/app/qlib_rd_loop/factor_from_report.py @@ -1,34 +1,38 @@ import json -from pathlib import Path import pickle +from pathlib import Path + +import pandas as pd from dotenv import load_dotenv from jinja2 import Environment, StrictUndefined -import pandas as pd from rdagent.app.qlib_rd_loop.conf import PROP_SETTING -from rdagent.components.document_reader.document_reader import load_and_process_pdfs_by_langchain +from rdagent.components.document_reader.document_reader import ( + load_and_process_pdfs_by_langchain, +) +from rdagent.core.developer import Developer from rdagent.core.prompts import Prompts +from rdagent.core.proposal import ( + Hypothesis, + Hypothesis2Experiment, + HypothesisExperiment2Feedback, + HypothesisGen, + Trace, +) from rdagent.core.scenario import Scenario from rdagent.core.utils import import_class from rdagent.log import rdagent_logger as logger from rdagent.oai.llm_utils import APIBackend from rdagent.scenarios.qlib.developer.factor_coder import QlibFactorCoSTEER -from rdagent.scenarios.qlib.experiment.factor_experiment import QlibFactorScenario, QlibFactorExperiment +from rdagent.scenarios.qlib.experiment.factor_experiment import ( + QlibFactorExperiment, + QlibFactorScenario, +) from rdagent.scenarios.qlib.factor_experiment_loader.pdf_loader import ( FactorExperimentLoaderFromPDFfiles, classify_report_from_dict, ) -from rdagent.core.proposal import ( - Hypothesis2Experiment, - HypothesisExperiment2Feedback, - HypothesisGen, - Hypothesis, - Trace, -) - -from rdagent.core.developer import Developer - assert load_dotenv() scen: Scenario = import_class(PROP_SETTING.factor_scen)() @@ -43,17 +47,21 @@ qlib_factor_runner: Developer = import_class(PROP_SETTING.factor_runner)(scen) qlib_factor_summarizer: HypothesisExperiment2Feedback = import_class(PROP_SETTING.factor_summarizer)(scen) -with open(PROP_SETTING.report_result_json_file_path, 'r') as f: +with open(PROP_SETTING.report_result_json_file_path, "r") as f: judge_pdf_data = json.load(f) prompts_path = Path(__file__).parent / "prompts.yaml" prompts = Prompts(file_path=prompts_path) + def generate_hypothesis(factor_result: dict, report_content: str) -> str: - system_prompt = Environment(undefined=StrictUndefined).from_string(prompts["hypothesis_generation"]["system"]).render() - user_prompt = Environment(undefined=StrictUndefined).from_string(prompts["hypothesis_generation"]["user"]).render( - factor_descriptions=json.dumps(factor_result), - report_content=report_content + system_prompt = ( + Environment(undefined=StrictUndefined).from_string(prompts["hypothesis_generation"]["system"]).render() + ) + user_prompt = ( + Environment(undefined=StrictUndefined) + .from_string(prompts["hypothesis_generation"]["user"]) + .render(factor_descriptions=json.dumps(factor_result), report_content=report_content) ) response = APIBackend().build_messages_and_create_chat_completion( @@ -68,16 +76,16 @@ def generate_hypothesis(factor_result: dict, report_content: str) -> str: return Hypothesis(hypothesis=hypothesis_text, reason=reason_text) + def extract_factors_and_implement(report_file_path: str) -> tuple: scenario = QlibFactorScenario() with logger.tag("extract_factors_and_implement"): with logger.tag("load_factor_tasks"): - exp = FactorExperimentLoaderFromPDFfiles().load(report_file_path) if exp is None or exp.sub_tasks == []: return None, None - + docs_dict = load_and_process_pdfs_by_langchain(Path(report_file_path)) factor_result = { @@ -85,7 +93,7 @@ def extract_factors_and_implement(report_file_path: str) -> tuple: "description": task.factor_description, "formulation": task.factor_formulation, "variables": task.variables, - "resources": task.factor_resources + "resources": task.factor_resources, } for task in exp.sub_tasks } @@ -95,6 +103,7 @@ def extract_factors_and_implement(report_file_path: str) -> tuple: return exp, hypothesis + trace = Trace(scen=scen) for file_path, attributes in judge_pdf_data.items(): diff --git a/rdagent/app/qlib_rd_loop/factor_from_report_sh.py b/rdagent/app/qlib_rd_loop/factor_from_report_sh.py index c8cc57e4..f0d0a961 100644 --- a/rdagent/app/qlib_rd_loop/factor_from_report_sh.py +++ b/rdagent/app/qlib_rd_loop/factor_from_report_sh.py @@ -1,35 +1,40 @@ # TODO: we should have more advanced mechanism to handle such requirements for saving sessions. import json -from pathlib import Path import pickle +from pathlib import Path + +import pandas as pd from dotenv import load_dotenv from jinja2 import Environment, StrictUndefined -import pandas as pd from rdagent.app.qlib_rd_loop.conf import PROP_SETTING -from rdagent.components.document_reader.document_reader import extract_first_page_screenshot_from_pdf, load_and_process_pdfs_by_langchain +from rdagent.components.document_reader.document_reader import ( + extract_first_page_screenshot_from_pdf, + load_and_process_pdfs_by_langchain, +) +from rdagent.core.developer import Developer from rdagent.core.prompts import Prompts +from rdagent.core.proposal import ( + Hypothesis, + Hypothesis2Experiment, + HypothesisExperiment2Feedback, + HypothesisGen, + Trace, +) from rdagent.core.scenario import Scenario from rdagent.core.utils import import_class from rdagent.log import rdagent_logger as logger from rdagent.oai.llm_utils import APIBackend from rdagent.scenarios.qlib.developer.factor_coder import QlibFactorCoSTEER -from rdagent.scenarios.qlib.experiment.factor_experiment import QlibFactorScenario, QlibFactorExperiment +from rdagent.scenarios.qlib.experiment.factor_experiment import ( + QlibFactorExperiment, + QlibFactorScenario, +) from rdagent.scenarios.qlib.factor_experiment_loader.pdf_loader import ( FactorExperimentLoaderFromPDFfiles, classify_report_from_dict, ) -from rdagent.core.proposal import ( - Hypothesis2Experiment, - HypothesisExperiment2Feedback, - HypothesisGen, - Hypothesis, - Trace, -) - -from rdagent.core.developer import Developer - assert load_dotenv() scen: Scenario = import_class(PROP_SETTING.factor_scen)() @@ -44,27 +49,33 @@ qlib_factor_runner: Developer = import_class(PROP_SETTING.factor_runner)(scen) qlib_factor_summarizer: HypothesisExperiment2Feedback = import_class(PROP_SETTING.factor_summarizer)(scen) -with open(PROP_SETTING.report_result_json_file_path, 'r') as f: +with open(PROP_SETTING.report_result_json_file_path, "r") as f: judge_pdf_data = json.load(f) prompts_path = Path(__file__).parent / "prompts.yaml" prompts = Prompts(file_path=prompts_path) + def save_progress(trace, current_index): with open(PROP_SETTING.progress_file_path, "wb") as f: pickle.dump((trace, current_index), f) + def load_progress(): if Path(PROP_SETTING.progress_file_path).exists(): with open(PROP_SETTING.progress_file_path, "rb") as f: return pickle.load(f) return Trace(scen=scen), 0 + def generate_hypothesis(factor_result: dict, report_content: str) -> str: - system_prompt = Environment(undefined=StrictUndefined).from_string(prompts["hypothesis_generation"]["system"]).render() - user_prompt = Environment(undefined=StrictUndefined).from_string(prompts["hypothesis_generation"]["user"]).render( - factor_descriptions=json.dumps(factor_result), - report_content=report_content + system_prompt = ( + Environment(undefined=StrictUndefined).from_string(prompts["hypothesis_generation"]["system"]).render() + ) + user_prompt = ( + Environment(undefined=StrictUndefined) + .from_string(prompts["hypothesis_generation"]["user"]) + .render(factor_descriptions=json.dumps(factor_result), report_content=report_content) ) response = APIBackend().build_messages_and_create_chat_completion( @@ -79,12 +90,12 @@ def generate_hypothesis(factor_result: dict, report_content: str) -> str: return Hypothesis(hypothesis=hypothesis_text, reason=reason_text) + def extract_factors_and_implement(report_file_path: str) -> tuple: scenario = QlibFactorScenario() with logger.tag("extract_factors_and_implement"): with logger.tag("load_factor_tasks"): - exp = FactorExperimentLoaderFromPDFfiles().load(report_file_path) if exp is None or exp.sub_tasks == []: return None, None @@ -100,7 +111,7 @@ def extract_factors_and_implement(report_file_path: str) -> tuple: "description": task.factor_description, "formulation": task.factor_formulation, "variables": task.variables, - "resources": task.factor_resources + "resources": task.factor_resources, } for task in exp.sub_tasks } @@ -110,6 +121,7 @@ def extract_factors_and_implement(report_file_path: str) -> tuple: return exp, hypothesis + trace, start_index = load_progress() try: @@ -122,7 +134,7 @@ try: report_file_path = Path(file_path.replace(PROP_SETTING.origin_report_path, PROP_SETTING.local_report_path)) if report_file_path.exists(): logger.info(f"Processing {report_file_path}") - + with logger.tag("r"): exp, hypothesis = extract_factors_and_implement(str(report_file_path)) if exp is None: @@ -132,7 +144,7 @@ try: exp.based_experiments.append(QlibFactorExperiment(sub_tasks=[])) logger.log_object(hypothesis, tag="hypothesis generation") logger.log_object(exp.sub_tasks, tag="experiment generation") - + with logger.tag("d"): exp = qlib_factor_coder.develop(exp) logger.log_object(exp.sub_workspace_list) @@ -145,10 +157,10 @@ try: logger.log_object(exp, tag="factor runner result") feedback = qlib_factor_summarizer.generate_feedback(exp, hypothesis, trace) logger.log_object(feedback, tag="feedback") - + trace.hist.append((hypothesis, exp, feedback)) logger.info(f"Processed {report_file_path}: Result: {exp}") - + # Save progress after processing each report save_progress(trace, index + 1) else: diff --git a/rdagent/app/qlib_rd_loop/factor_w_sc.py b/rdagent/app/qlib_rd_loop/factor_w_sc.py index 79087162..10c9d19e 100644 --- a/rdagent/app/qlib_rd_loop/factor_w_sc.py +++ b/rdagent/app/qlib_rd_loop/factor_w_sc.py @@ -30,4 +30,4 @@ def main(path=None, step_n=None): if __name__ == "__main__": - fire.Fire(main) \ No newline at end of file + fire.Fire(main) diff --git a/rdagent/app/quant_factor_benchmark/analysis.py b/rdagent/app/quant_factor_benchmark/analysis.py index 75aea00d..6167ab67 100644 --- a/rdagent/app/quant_factor_benchmark/analysis.py +++ b/rdagent/app/quant_factor_benchmark/analysis.py @@ -1,12 +1,14 @@ import json import pickle -import pandas as pd from pathlib import Path + import matplotlib.pyplot as plt +import pandas as pd import seaborn as sns -from rdagent.components.benchmark.eval_method import FactorImplementEval from rdagent.components.benchmark.conf import BenchmarkSettings +from rdagent.components.benchmark.eval_method import FactorImplementEval + class BenchmarkAnalyzer: def __init__(self, settings): @@ -25,10 +27,10 @@ class BenchmarkAnalyzer: file_path = Path(file_path) if not (file_path.is_file() and file_path.suffix == ".pkl"): raise ValueError("Invalid file path") - + with file_path.open("rb") as f: res = pickle.load(f) - + return res def process_results(self, results): @@ -39,7 +41,7 @@ class BenchmarkAnalyzer: processed_data = self.analyze_data(summarized_data) final_res[experiment] = processed_data.iloc[-1, :] return final_res - + def reformat_succ_rate(self, display_df): new_idx = [] display_df = display_df[display_df.index.isin(self.index_map.keys())] @@ -52,8 +54,10 @@ class BenchmarkAnalyzer: ) display_df = display_df.swaplevel(0, 2).swaplevel(0, 1).sort_index(axis=0) - return display_df.sort_index(key=lambda x: [{"Easy": 0, "Medium": 1, "Hard": 2, "New Discovery": 3}.get(i, i) for i in x]) - + return display_df.sort_index( + key=lambda x: [{"Easy": 0, "Medium": 1, "Hard": 2, "New Discovery": 3}.get(i, i) for i in x] + ) + def result_all_key_order(self, x): order_v = [] for i in x: @@ -92,9 +96,7 @@ class BenchmarkAnalyzer: sum_df_clean["FactorRowCountEvaluator"] - format_issue = ( - sum_df_clean["FactorRowCountEvaluator"] & sum_df_clean["FactorIndexEvaluator"] - ) + format_issue = sum_df_clean["FactorRowCountEvaluator"] & sum_df_clean["FactorIndexEvaluator"] eval_series = format_issue.unstack() succ_rate = eval_series.T.fillna(False).astype(bool) # false indicate failure format_succ_rate = succ_rate.mean(axis=0).to_frame("success rate") @@ -113,10 +115,7 @@ class BenchmarkAnalyzer: value_max_res = self.reformat_succ_rate(value_max) value_avg = ( - (sum_df_clean["FactorMissingValuesEvaluator"] * format_issue) - .unstack() - .T.mean(axis=0) - .to_frame("avg_value") + (sum_df_clean["FactorMissingValuesEvaluator"] * format_issue).unstack().T.mean(axis=0).to_frame("avg_value") ) value_avg_res = self.reformat_succ_rate(value_avg) @@ -148,7 +147,6 @@ class BenchmarkAnalyzer: return df_w_mean - class Plotter: @staticmethod def change_fs(font_size): @@ -169,6 +167,7 @@ class Plotter: plt.title("Comparison of Different Methods") plt.savefig(file_name) + if __name__ == "__main__": settings = BenchmarkSettings() benchmark = BenchmarkAnalyzer(settings) diff --git a/rdagent/app/quant_factor_benchmark/eval.py b/rdagent/app/quant_factor_benchmark/eval.py index a22f928f..2f3cf275 100644 --- a/rdagent/app/quant_factor_benchmark/eval.py +++ b/rdagent/app/quant_factor_benchmark/eval.py @@ -1,23 +1,20 @@ import os -from pathlib import Path import pickle import time +from pathlib import Path +from pprint import pprint + from rdagent.app.qlib_rd_loop.conf import PROP_SETTING +from rdagent.components.benchmark.conf import BenchmarkSettings +from rdagent.components.benchmark.eval_method import FactorImplementEval +from rdagent.core.scenario import Scenario +from rdagent.core.utils import import_class from rdagent.log import rdagent_logger as logger +from rdagent.scenarios.qlib.experiment.factor_experiment import QlibFactorScenario from rdagent.scenarios.qlib.factor_experiment_loader.json_loader import ( FactorTestCaseLoaderFromJsonFile, ) -from rdagent.components.benchmark.conf import BenchmarkSettings -from rdagent.components.benchmark.eval_method import FactorImplementEval -from rdagent.core.utils import import_class - -from rdagent.core.utils import import_class -from rdagent.core.scenario import Scenario -from rdagent.scenarios.qlib.experiment.factor_experiment import QlibFactorScenario - -from pprint import pprint - # 1.read the settings bs = BenchmarkSettings() @@ -28,7 +25,7 @@ test_cases = FactorTestCaseLoaderFromJsonFile().load(bs.bench_data_path) scen: Scenario = import_class(PROP_SETTING.factor_scen)() generate_method = import_class(bs.bench_method_cls)(scen=scen) - + # 4.declare the eval method and pass the arguments. eval_method = FactorImplementEval( method=generate_method, diff --git a/rdagent/components/benchmark/eval_method.py b/rdagent/components/benchmark/eval_method.py index 193d1ce2..b1b4632d 100644 --- a/rdagent/components/benchmark/eval_method.py +++ b/rdagent/components/benchmark/eval_method.py @@ -19,18 +19,17 @@ from rdagent.components.coder.factor_coder.CoSTEER.evaluators import ( from rdagent.components.coder.factor_coder.factor import FactorFBWorkspace from rdagent.core.conf import RD_AGENT_SETTINGS from rdagent.core.developer import Developer - from rdagent.core.exception import CoderException, RunnerException from rdagent.core.experiment import Task, Workspace from rdagent.core.scenario import Scenario from rdagent.core.utils import multiprocessing_wrapper - EVAL_RES = Dict[ str, List[Tuple[FactorEvaluator, Union[object, RunnerException]]], ] + class TestCase: def __init__( self, diff --git a/rdagent/components/coder/factor_coder/CoSTEER/evaluators.py b/rdagent/components/coder/factor_coder/CoSTEER/evaluators.py index a1ddf6f7..bd732bcb 100644 --- a/rdagent/components/coder/factor_coder/CoSTEER/evaluators.py +++ b/rdagent/components/coder/factor_coder/CoSTEER/evaluators.py @@ -356,7 +356,9 @@ class FactorValueEvaluator(FactorEvaluator): # Check if both dataframe have the same rows count if gt_implementation is not None: - feedback_str, single_column_result = FactorRowCountEvaluator(self.scen).evaluate(implementation, gt_implementation) + feedback_str, single_column_result = FactorRowCountEvaluator(self.scen).evaluate( + implementation, gt_implementation + ) conclusions.append(feedback_str) feedback_str, same_index_result = FactorIndexEvaluator(self.scen).evaluate( @@ -364,7 +366,9 @@ class FactorValueEvaluator(FactorEvaluator): ) conclusions.append(feedback_str) - feedback_str, output_format_result = FactorMissingValuesEvaluator(self.scen).evaluate(implementation, gt_implementation) + feedback_str, output_format_result = FactorMissingValuesEvaluator(self.scen).evaluate( + implementation, gt_implementation + ) conclusions.append(feedback_str) feedback_str, equal_value_ratio_result = FactorEqualValueCountEvaluator(self.scen).evaluate( @@ -464,8 +468,10 @@ class FactorFinalDecisionEvaluator(Evaluator): except KeyError as e: attempts += 1 if attempts >= max_attempts: - raise KeyError("Response from API is missing 'final_decision' or 'final_feedback' key after multiple attempts.") from e - + raise KeyError( + "Response from API is missing 'final_decision' or 'final_feedback' key after multiple attempts." + ) from e + return None, None diff --git a/rdagent/components/coder/factor_coder/CoSTEER/evolving_strategy.py b/rdagent/components/coder/factor_coder/CoSTEER/evolving_strategy.py index 1493cfac..b5a66e5f 100644 --- a/rdagent/components/coder/factor_coder/CoSTEER/evolving_strategy.py +++ b/rdagent/components/coder/factor_coder/CoSTEER/evolving_strategy.py @@ -68,8 +68,12 @@ class MultiProcessEvolvingStrategy(EvolvingStrategy): # if the number of factors to be implemented is larger than the limit, we need to select some of them if FACTOR_IMPLEMENT_SETTINGS.select_ratio < 1: # if the number of loops is equal to the select_loop, we need to select some of them - implementation_factors_per_round = round(FACTOR_IMPLEMENT_SETTINGS.select_ratio * len(to_be_finished_task_index) + 0.5) # ceilling - implementation_factors_per_round = min(implementation_factors_per_round, len(to_be_finished_task_index)) # but not exceed the total number of tasks + implementation_factors_per_round = round( + FACTOR_IMPLEMENT_SETTINGS.select_ratio * len(to_be_finished_task_index) + 0.5 + ) # ceilling + implementation_factors_per_round = min( + implementation_factors_per_round, len(to_be_finished_task_index) + ) # but not exceed the total number of tasks if FACTOR_IMPLEMENT_SETTINGS.select_method == "random": to_be_finished_task_index = RandomSelect( diff --git a/rdagent/components/coder/factor_coder/factor.py b/rdagent/components/coder/factor_coder/factor.py index 9dd51466..412f98a6 100644 --- a/rdagent/components/coder/factor_coder/factor.py +++ b/rdagent/components/coder/factor_coder/factor.py @@ -10,11 +10,7 @@ import pandas as pd from filelock import FileLock from rdagent.components.coder.factor_coder.config import FACTOR_IMPLEMENT_SETTINGS -from rdagent.core.exception import ( - CodeFormatError, - NoOutputError, - CustomRuntimeError, -) +from rdagent.core.exception import CodeFormatError, CustomRuntimeError, NoOutputError from rdagent.core.experiment import Experiment, FBWorkspace, Task from rdagent.log import rdagent_logger as logger from rdagent.oai.llm_utils import md5_hash diff --git a/rdagent/components/coder/model_coder/model.py b/rdagent/components/coder/model_coder/model.py index eeff3d7c..7d680185 100644 --- a/rdagent/components/coder/model_coder/model.py +++ b/rdagent/components/coder/model_coder/model.py @@ -16,7 +16,14 @@ from rdagent.utils import get_module_by_module_path class ModelTask(Task): def __init__( - self, name: str, description: str, formulation: str, architecture: str, variables: Dict[str, str], hyperparameters: Dict[str, str], model_type: Optional[str] = None + self, + name: str, + description: str, + formulation: str, + architecture: str, + variables: Dict[str, str], + hyperparameters: Dict[str, str], + model_type: Optional[str] = None, ) -> None: self.name: str = name self.description: str = description @@ -24,7 +31,9 @@ class ModelTask(Task): self.architecture: str = architecture self.variables: str = variables self.hyperparameters: str = hyperparameters - self.model_type: str = model_type # Tabular for tabular model, TimesSeries for time series model, Graph for graph model + self.model_type: str = ( + model_type # Tabular for tabular model, TimesSeries for time series model, Graph for graph model + ) def get_task_information(self): return f"""name: {self.name} @@ -107,21 +116,21 @@ class ModelFBWorkspace(FBWorkspace): # Initialize all parameters of `m` to `param_init_value` for _, param in m.named_parameters(): param.data.fill_(param_init_value) - + # Execute the model if self.target_task.model_type == "Graph": out = m(*data) else: out = m(data) - + execution_model_output = out.cpu().detach() execution_feedback_str = f"Execution successful, output tensor shape: {execution_model_output.shape}" - + if MODEL_IMPL_SETTINGS.enable_execution_cache: pickle.dump((execution_feedback_str, execution_model_output), open(cache_file_path, "wb")) - + return execution_feedback_str, execution_model_output - + except Exception as e: return f"Execution error: {e}", None diff --git a/rdagent/components/document_reader/document_reader.py b/rdagent/components/document_reader/document_reader.py index 9341066f..d6147691 100644 --- a/rdagent/components/document_reader/document_reader.py +++ b/rdagent/components/document_reader/document_reader.py @@ -1,16 +1,17 @@ from __future__ import annotations from pathlib import Path -import fitz -from PIL import Image from typing import TYPE_CHECKING +import fitz from azure.ai.formrecognizer import DocumentAnalysisClient from azure.core.credentials import AzureKeyCredential from langchain.document_loaders import PyPDFDirectoryLoader, PyPDFLoader +from PIL import Image if TYPE_CHECKING: from langchain_core.documents import Document + from rdagent.core.conf import RD_AGENT_SETTINGS @@ -99,10 +100,11 @@ def load_and_process_pdfs_by_azure_document_intelligence(path: Path) -> dict[str ) return content_dict + def extract_first_page_screenshot_from_pdf(pdf_path: Path) -> Image: doc = fitz.open(pdf_path) page = doc.load_page(0) pix = page.get_pixmap() image = Image.frombytes("RGB", [pix.width, pix.height], pix.samples) - - return image \ No newline at end of file + + return image diff --git a/rdagent/components/proposal/model_proposal.py b/rdagent/components/proposal/model_proposal.py index c6b59a03..a5d28006 100644 --- a/rdagent/components/proposal/model_proposal.py +++ b/rdagent/components/proposal/model_proposal.py @@ -15,11 +15,11 @@ from rdagent.core.proposal import ( ) from rdagent.oai.llm_utils import APIBackend - ModelHypothesis = Hypothesis prompt_dict = Prompts(file_path=Path(__file__).parent / "prompts.yaml") + class ModelHypothesisGen(HypothesisGen): prompts: Prompts = prompt_dict diff --git a/rdagent/components/workflow/conf.py b/rdagent/components/workflow/conf.py index fc8f84ad..054c63ce 100644 --- a/rdagent/components/workflow/conf.py +++ b/rdagent/components/workflow/conf.py @@ -1,5 +1,6 @@ from pydantic_settings import BaseSettings + class BasePropSetting(BaseSettings): """ The common part of the config for RD Loop to propose and developement @@ -11,6 +12,7 @@ class BasePropSetting(BaseSettings): env_prefix = "DM_MODEL_" # Use MODEL_CODER_ as prefix for environment variables protected_namespaces = () # Add 'model_' to the protected namespaces """ + scen: str = "" hypothesis_gen: str = "" hypothesis2experiment: str = "" diff --git a/rdagent/components/workflow/rd_loop.py b/rdagent/components/workflow/rd_loop.py index d1a40017..dcb1cbdc 100644 --- a/rdagent/components/workflow/rd_loop.py +++ b/rdagent/components/workflow/rd_loop.py @@ -4,6 +4,7 @@ It is from `rdagent/app/qlib_rd_loop/model.py` and try to replace `rdagent/app/q """ from typing import Any + from rdagent.components.workflow.conf import BasePropSetting from rdagent.core.developer import Developer from rdagent.core.proposal import ( @@ -15,11 +16,10 @@ from rdagent.core.proposal import ( from rdagent.core.scenario import Scenario from rdagent.core.utils import import_class from rdagent.log import rdagent_logger as logger +from rdagent.utils.workflow import LoopBase, LoopMeta -from rdagent.utils.workflow import LoopMeta, LoopBase class RDLoop(LoopBase, metaclass=LoopMeta): - def __init__(self, PROP_SETTING: BasePropSetting): scen: Scenario = import_class(PROP_SETTING.scen)() @@ -62,4 +62,4 @@ class RDLoop(LoopBase, metaclass=LoopMeta): feedback = self.summarizer.generate_feedback(prev_out["running"], prev_out["propose"], self.trace) with logger.tag("ef"): # evaluate and feedback logger.log_object(feedback, tag="feedback") - self.trace.hist.append((prev_out["propose"],prev_out["running"] , feedback)) + self.trace.hist.append((prev_out["propose"], prev_out["running"], feedback)) diff --git a/rdagent/core/evolving_agent.py b/rdagent/core/evolving_agent.py index 25506dcb..c797d75f 100644 --- a/rdagent/core/evolving_agent.py +++ b/rdagent/core/evolving_agent.py @@ -25,14 +25,16 @@ class EvoAgent(ABC): evo: EvolvableSubjects, eva: Evaluator | Feedback, filter_final_evo: bool = False, - ) -> EvolvableSubjects: ... + ) -> EvolvableSubjects: + ... @abstractmethod def filter_evolvable_subjects_by_feedback( self, evo: EvolvableSubjects, feedback: Feedback | None, - ) -> EvolvableSubjects: ... + ) -> EvolvableSubjects: + ... class RAGEvoAgent(EvoAgent): @@ -52,7 +54,6 @@ class RAGEvoAgent(EvoAgent): self.with_feedback = with_feedback self.knowledge_self_gen = knowledge_self_gen - def multistep_evolve( self, evo: EvolvableSubjects, @@ -86,7 +87,9 @@ class RAGEvoAgent(EvoAgent): es.feedback = ( # TODO: Due to the irregular design of rdagent.core.evaluation.Evaluator, # it fails mypy's test here, so we'll ignore this error for now. - eva if isinstance(eva, Feedback) else eva.evaluate(evo, queried_knowledge=queried_knowledge) # type: ignore[arg-type, call-arg] + eva + if isinstance(eva, Feedback) + else eva.evaluate(evo, queried_knowledge=queried_knowledge) # type: ignore[arg-type, call-arg] ) logger.log_object(es.feedback, tag="evolving feedback") diff --git a/rdagent/core/proposal.py b/rdagent/core/proposal.py index 1da6bd55..c1e598ed 100644 --- a/rdagent/core/proposal.py +++ b/rdagent/core/proposal.py @@ -99,12 +99,12 @@ class HypothesisGen(ABC): # def gen(self, scenario_desc: str, ) -> Hypothesis: """ Motivation of the variable `scenario_desc`: - - Mocking a data-scientist is observing the scenario. + - Mocking a data-scientist is observing the scenario. - scenario_desc may conclude: - - data observation: - - Original or derivative - - Task information: + scenario_desc may include: + - data observation: + - Original or derivative + - Task information: """ diff --git a/rdagent/core/utils.py b/rdagent/core/utils.py index 949d4a37..9a19b6a8 100644 --- a/rdagent/core/utils.py +++ b/rdagent/core/utils.py @@ -18,6 +18,7 @@ class SingletonBaseClass: Because we try to support defining Singleton with `class A(SingletonBaseClass)` instead of `A(metaclass=SingletonMeta)` this class becomes necessary. """ + _instance_dict: ClassVar[dict] = {} def __new__(cls, *args: Any, **kwargs: Any) -> Any: @@ -34,7 +35,6 @@ class SingletonBaseClass: return cls._instance_dict[kwargs_hash] - def parse_json(response: str) -> Any: try: return json.loads(response) diff --git a/rdagent/log/base.py b/rdagent/log/base.py index bdcf3c04..305fb7d3 100644 --- a/rdagent/log/base.py +++ b/rdagent/log/base.py @@ -2,19 +2,22 @@ from __future__ import annotations from abc import abstractmethod from collections.abc import Generator +from dataclasses import dataclass from datetime import datetime from pathlib import Path -from typing import Literal, Optional, Union, Literal -from dataclasses import dataclass +from typing import Literal, Optional, Union @dataclass class Message: """The info unit of the storage""" + tag: str # namespace like like a.b.c level: Literal["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"] # The level of the logging timestamp: datetime # The time when the message is generated - caller: Optional[str] # The caller of the logging like `rdagent.oai.llm_utils:_create_chat_completion_inner_function:55`(file:func:line) + caller: Optional[ + str + ] # The caller of the logging like `rdagent.oai.llm_utils:_create_chat_completion_inner_function:55`(file:func:line) pid_trace: Optional[str] # The process id trace; A-B-C represents A create B, B create C content: object # The content @@ -39,7 +42,8 @@ class Storage: @abstractmethod def log( self, - obj: object, name: str = "", + obj: object, + name: str = "", save_type: Literal["json", "text", "pkl"] = "text", timestamp: datetime | None = None, **kwargs: dict, diff --git a/rdagent/log/logger.py b/rdagent/log/logger.py index fc987fea..87caf32c 100644 --- a/rdagent/log/logger.py +++ b/rdagent/log/logger.py @@ -3,15 +3,17 @@ import sys from contextlib import contextmanager from datetime import datetime, timezone from functools import partial +from logging import LogRecord from multiprocessing import Pipe from multiprocessing.connection import Connection from pathlib import Path -from typing import TYPE_CHECKING, Union, Generator, Dict, Any -from logging import LogRecord +from typing import TYPE_CHECKING, Any, Dict, Generator, Union from loguru import logger + if TYPE_CHECKING: from loguru import Record + from psutil import Process from rdagent.core.conf import RD_AGENT_SETTINGS diff --git a/rdagent/log/storage.py b/rdagent/log/storage.py index ff514892..7ed014c5 100644 --- a/rdagent/log/storage.py +++ b/rdagent/log/storage.py @@ -1,9 +1,9 @@ -import re import json import pickle +import re from datetime import datetime, timezone from pathlib import Path -from typing import Literal, Generator, Union, Any, cast +from typing import Any, Generator, Literal, Union, cast from .base import Message, Storage @@ -17,7 +17,6 @@ class FileStorage(Storage): TODO: describe the storage format """ - def __init__(self, path: str | Path = "./log/") -> None: self.path = Path(path) self.path.mkdir(parents=True, exist_ok=True) @@ -69,7 +68,7 @@ class FileStorage(Storage): def iter_msg(self, watch: bool = False) -> Generator[Message, None, None]: msg_l = [] for file in self.path.glob("**/*.log"): - tag = '.'.join(str(file.relative_to(self.path)).replace("/", ".").split(".")[:-3]) + tag = ".".join(str(file.relative_to(self.path)).replace("/", ".").split(".")[:-3]) pid = file.parent.name with file.open("r") as f: @@ -92,12 +91,7 @@ class FileStorage(Storage): message_content = content[message_start:message_end].strip() m = Message( - tag=tag, - level=level, - timestamp=timestamp, - caller=caller, - pid_trace=pid, - content=message_content + tag=tag, level=level, timestamp=timestamp, caller=caller, pid_trace=pid, content=message_content ) if isinstance(m.content, str) and "Logging object in" in m.content: @@ -119,7 +113,6 @@ class FileStorage(Storage): def truncate(self, time: datetime) -> None: # any message later than `time` will be removed for file in self.path.glob("**/*.log"): - with file.open("r") as f: content = f.read() @@ -135,7 +128,7 @@ class FileStorage(Storage): log_start = match.start() log_end = next_match.start() if next_match else len(content) - msg = content[match.end():log_end].strip() + msg = content[match.end() : log_end].strip() if timestamp > time: if "Logging object in" in msg: diff --git a/rdagent/log/ui/app.py b/rdagent/log/ui/app.py index 038dc8bf..52060c68 100644 --- a/rdagent/log/ui/app.py +++ b/rdagent/log/ui/app.py @@ -1,9 +1,15 @@ -from rdagent.log.ui.web import WebView, SimpleTraceWindow, TraceObjWindow, mock_msg, TraceWindow -from rdagent.log.storage import FileStorage, Message -from rdagent.core.proposal import Trace -from pathlib import Path import pickle +from pathlib import Path +from rdagent.core.proposal import Trace +from rdagent.log.storage import FileStorage, Message +from rdagent.log.ui.web import ( + SimpleTraceWindow, + TraceObjWindow, + TraceWindow, + WebView, + mock_msg, +) # show logs folder WebView(TraceWindow()).display(FileStorage("/data/home/bowen/workspace/RD-Agent/log/yuante/2024-07-24_04-03-33-691119")) diff --git a/rdagent/log/ui/st_fixed_container.py b/rdagent/log/ui/st_fixed_container.py index ae3d8cc2..262f0388 100644 --- a/rdagent/log/ui/st_fixed_container.py +++ b/rdagent/log/ui/st_fixed_container.py @@ -123,4 +123,4 @@ if __name__ == "__main__": st.container(border=True).write("This is a regular container.") for i in range(30): - st.write(f"Line {i}") \ No newline at end of file + st.write(f"Line {i}") diff --git a/rdagent/log/ui/web.py b/rdagent/log/ui/web.py index 16f68f8e..83c66bff 100644 --- a/rdagent/log/ui/web.py +++ b/rdagent/log/ui/web.py @@ -1,51 +1,45 @@ -import pandas as pd -import streamlit as st -import plotly.express as px import time - -from rdagent.log.base import Storage, View -from rdagent.log.base import Message -from datetime import timezone, datetime from collections import defaultdict from copy import deepcopy -from rdagent.core.proposal import Trace - +from datetime import datetime, timezone from typing import Callable, Type -from streamlit.delta_generator import DeltaGenerator -from rdagent.core.proposal import Hypothesis, HypothesisFeedback +import pandas as pd +import plotly.express as px +import streamlit as st +from streamlit.delta_generator import DeltaGenerator + +from rdagent.components.coder.factor_coder.CoSTEER.evaluators import ( + FactorSingleFeedback, +) +from rdagent.components.coder.factor_coder.factor import FactorFBWorkspace, FactorTask +from rdagent.components.coder.model_coder.CoSTEER.evaluators import ModelCoderFeedback +from rdagent.components.coder.model_coder.model import ModelFBWorkspace, ModelTask +from rdagent.core.proposal import Hypothesis, HypothesisFeedback, Trace +from rdagent.log.base import Message, Storage, View from rdagent.scenarios.qlib.experiment.factor_experiment import QlibFactorExperiment from rdagent.scenarios.qlib.experiment.model_experiment import QlibModelExperiment -from rdagent.components.coder.factor_coder.factor import FactorTask, FactorFBWorkspace -from rdagent.components.coder.factor_coder.CoSTEER.evaluators import FactorSingleFeedback -from rdagent.components.coder.model_coder.CoSTEER.evaluators import ModelCoderFeedback -from rdagent.components.coder.model_coder.model import ModelTask, ModelFBWorkspace - - st.set_page_config(layout="wide") TIME_DELAY = 0.001 -class WebView(View): - def __init__(self, ui: 'StWindow'): +class WebView(View): + def __init__(self, ui: "StWindow"): self.ui = ui # Save logs to your desired data structure # ... def display(self, s: Storage, watch: bool = False): - for msg in s.iter_msg(): # iterate overtime # NOTE: iter_msg will correctly seperate the information. # TODO: msg may support streaming mode. self.ui.consume_msg(msg) - class StWindow: - - def __init__(self, container: 'DeltaGenerator'): + def __init__(self, container: "DeltaGenerator"): self.container = container def consume_msg(self, msg: Message): @@ -54,31 +48,32 @@ class StWindow: class LLMWindow(StWindow): - def __init__(self, container: 'DeltaGenerator', session_name: str="common"): + def __init__(self, container: "DeltaGenerator", session_name: str = "common"): self.session_name = session_name self.container = container.expander(f"{self.session_name} message") def consume_msg(self, msg: Message): - self.container.chat_message('user').markdown(f"{msg.content}") + self.container.chat_message("user").markdown(f"{msg.content}") class ProgressTabsWindow(StWindow): - ''' + """ For windows with stream messages, will refresh when a new tab is created. - ''' - def __init__(self, - container: 'DeltaGenerator', - inner_class: Type[StWindow] = StWindow, - mapper: Callable[[Message], str] = lambda x: x.pid_trace): - + """ + + def __init__( + self, + container: "DeltaGenerator", + inner_class: Type[StWindow] = StWindow, + mapper: Callable[[Message], str] = lambda x: x.pid_trace, + ): self.inner_class = inner_class self.mapper = mapper self.container = container.empty() self.tab_windows: dict[str, StWindow] = defaultdict(None) self.tab_caches: dict[str, list[Message]] = defaultdict(list) - - + def consume_msg(self, msg: Message): name = self.mapper(msg) @@ -93,61 +88,67 @@ class ProgressTabsWindow(StWindow): for id, name in enumerate(names): self.tab_windows[name] = self.inner_class(tabs[id]) - + # consume the cache for name in self.tab_caches: for msg in self.tab_caches[name]: self.tab_windows[name].consume_msg(msg) - + self.tab_caches[name].append(msg) self.tab_windows[name].consume_msg(msg) class ObjectsTabsWindow(StWindow): - def __init__(self, - container: 'DeltaGenerator', - inner_class: Type[StWindow] = StWindow, - mapper: Callable[[object], str] = lambda x: str(x), - tab_names: list[str] | None = None): + def __init__( + self, + container: "DeltaGenerator", + inner_class: Type[StWindow] = StWindow, + mapper: Callable[[object], str] = lambda x: str(x), + tab_names: list[str] | None = None, + ): self.inner_class = inner_class self.mapper = mapper self.container = container self.tab_names = tab_names - + def consume_msg(self, msg: Message): if isinstance(msg.content, list): if self.tab_names: - assert len(self.tab_names) == len(msg.content), "List of objects should have the same length as provided tab names." + assert len(self.tab_names) == len( + msg.content + ), "List of objects should have the same length as provided tab names." objs_dict = {self.tab_names[id]: obj for id, obj in enumerate(msg.content)} else: objs_dict = {self.mapper(obj): obj for obj in msg.content} elif not isinstance(msg.content, dict): raise ValueError("Message content should be a list or a dict of objects.") - + # two many tabs may cause display problem tab_names = list(objs_dict.keys()) tabs = [] for i in range(0, len(tab_names), 10): - tabs.extend(self.container.tabs(tab_names[i:i+10])) - + tabs.extend(self.container.tabs(tab_names[i : i + 10])) + for id, obj in enumerate(objs_dict.values()): - splited_msg = Message(tag=msg.tag, - level=msg.level, - timestamp=msg.timestamp, - caller=msg.caller, - pid_trace=msg.pid_trace, - content=obj) + splited_msg = Message( + tag=msg.tag, + level=msg.level, + timestamp=msg.timestamp, + caller=msg.caller, + pid_trace=msg.pid_trace, + content=obj, + ) self.inner_class(tabs[id]).consume_msg(splited_msg) class RoundTabsWindow(StWindow): - - def __init__(self, - container: 'DeltaGenerator', - new_tab_func: Callable[[Message], bool], - inner_class: Type[StWindow] = StWindow, - title: str = 'Round tabs'): - + def __init__( + self, + container: "DeltaGenerator", + new_tab_func: Callable[[Message], bool], + inner_class: Type[StWindow] = StWindow, + title: str = "Round tabs", + ): container.markdown(f"### **{title}**") self.inner_class = inner_class self.new_tab_func = new_tab_func @@ -156,42 +157,42 @@ class RoundTabsWindow(StWindow): self.current_win = StWindow(container) self.tabs_c = container.empty() - def consume_msg(self, msg: Message): if self.new_tab_func(msg): self.round += 1 - self.current_win = self.inner_class(self.tabs_c.tabs([str(i) for i in range(1, self.round+1)])[-1]) + self.current_win = self.inner_class(self.tabs_c.tabs([str(i) for i in range(1, self.round + 1)])[-1]) self.current_win.consume_msg(msg) class HypothesisWindow(StWindow): - def consume_msg(self, msg: Message | Hypothesis): h: Hypothesis = msg.content if isinstance(msg, Message) else msg - self.container.markdown('#### **Hypothesis💡**') - self.container.markdown(f""" + self.container.markdown("#### **Hypothesis💡**") + self.container.markdown( + f""" - **Hypothesis**: {h.hypothesis} -- **Reason**: {h.reason}""") +- **Reason**: {h.reason}""" + ) class HypothesisFeedbackWindow(StWindow): - def consume_msg(self, msg: Message | HypothesisFeedback): h: HypothesisFeedback = msg.content if isinstance(msg, Message) else msg - self.container.markdown('#### **Hypothesis Feedback🔍**') - self.container.markdown(f""" + self.container.markdown("#### **Hypothesis Feedback🔍**") + self.container.markdown( + f""" - **Observations**: {h.observations} - **Hypothesis Evaluation**: {h.hypothesis_evaluation} - **New Hypothesis**: {h.new_hypothesis} - **Decision**: {h.decision} -- **Reason**: {h.reason}""") +- **Reason**: {h.reason}""" + ) class FactorTaskWindow(StWindow): - def consume_msg(self, msg: Message | FactorTask): ft: FactorTask = msg.content if isinstance(msg, Message) else msg @@ -199,14 +200,13 @@ class FactorTaskWindow(StWindow): self.container.markdown(f"**Description**: {ft.factor_description}") self.container.latex(f"Formulation: {ft.factor_formulation}") - variables_df = pd.DataFrame(ft.variables, index=['Description']).T - variables_df.index.name = 'Variable' + variables_df = pd.DataFrame(ft.variables, index=["Description"]).T + variables_df.index.name = "Variable" self.container.table(variables_df) self.container.text(f"Factor resources: {ft.factor_resources}") class ModelTaskWindow(StWindow): - def consume_msg(self, msg: Message | ModelTask): mt: ModelTask = msg.content if isinstance(msg, Message) else msg @@ -214,18 +214,18 @@ class ModelTaskWindow(StWindow): self.container.markdown(f"**Model Type**: {mt.model_type}") self.container.markdown(f"**Description**: {mt.description}") self.container.markdown(f"**Formulation**: {mt.formulation}") - - variables_df = pd.DataFrame(mt.variables, index=['Value']).T - variables_df.index.name = 'Variable' + + variables_df = pd.DataFrame(mt.variables, index=["Value"]).T + variables_df.index.name = "Variable" self.container.table(variables_df) class FactorFeedbackWindow(StWindow): - def consume_msg(self, msg: Message | FactorSingleFeedback): fb: FactorSingleFeedback = msg.content if isinstance(msg, Message) else msg - self.container.markdown(f"""### :blue[Factor Execution Feedback] + self.container.markdown( + f"""### :blue[Factor Execution Feedback] {fb.execution_feedback} ### :blue[Factor Code Feedback] {fb.code_feedback} @@ -235,15 +235,16 @@ class FactorFeedbackWindow(StWindow): {fb.final_feedback} ### :blue[Factor Final Decision] This implementation is {'SUCCESS' if fb.final_decision else 'FAIL'}. -""") +""" + ) class ModelFeedbackWindow(StWindow): - def consume_msg(self, msg: Message | ModelCoderFeedback): mb: ModelCoderFeedback = msg.content if isinstance(msg, Message) else msg - self.container.markdown(f"""### :blue[Model Execution Feedback] + self.container.markdown( + f"""### :blue[Model Execution Feedback] {mb.execution_feedback} ### :blue[Model Shape Feedback] {mb.shape_feedback} @@ -255,11 +256,12 @@ class ModelFeedbackWindow(StWindow): {mb.final_feedback} ### :blue[Model Final Decision] This implementation is {'SUCCESS' if mb.final_decision else 'FAIL'}. -""") +""" + ) class WorkspaceWindow(StWindow): - def __init__(self, container: 'DeltaGenerator', show_task_info: bool = False): + def __init__(self, container: "DeltaGenerator", show_task_info: bool = False): self.container = container self.show_task_info = show_task_info @@ -267,21 +269,22 @@ class WorkspaceWindow(StWindow): ws: FactorFBWorkspace | ModelFBWorkspace = msg.content if isinstance(msg, Message) else msg # no workspace - if ws is None: return + if ws is None: + return # task info if self.show_task_info: task_msg = deepcopy(msg) task_msg.content = ws.target_task if isinstance(ws, FactorFBWorkspace): - self.container.subheader('Factor Info') + self.container.subheader("Factor Info") FactorTaskWindow(self.container.container()).consume_msg(task_msg) else: - self.container.subheader('Model Info') + self.container.subheader("Model Info") ModelTaskWindow(self.container.container()).consume_msg(task_msg) # task codes - for k,v in ws.code_dict.items(): + for k, v in ws.code_dict.items(): self.container.markdown(f"`{k}`") self.container.code(v, language="python") @@ -302,24 +305,25 @@ class QlibFactorExpWindow(StWindow): if self.show_task_info: ftm_msg = deepcopy(msg) ftm_msg.content = [ws for ws in exp.sub_workspace_list if ws] - self.container.markdown('**Factor Tasks**') - ObjectsTabsWindow(self.container.container(), - inner_class=WorkspaceWindow, - mapper=lambda x: x.target_task.factor_name, - ).consume_msg(ftm_msg) + self.container.markdown("**Factor Tasks**") + ObjectsTabsWindow( + self.container.container(), + inner_class=WorkspaceWindow, + mapper=lambda x: x.target_task.factor_name, + ).consume_msg(ftm_msg) # result - self.container.markdown('**Results**') - results = pd.DataFrame({f'base_exp_{id}':e.result for id, e in enumerate(exp.based_experiments)}) - results['now'] = exp.result + self.container.markdown("**Results**") + results = pd.DataFrame({f"base_exp_{id}": e.result for id, e in enumerate(exp.based_experiments)}) + results["now"] = exp.result - self.container.expander('results table').table(results) + self.container.expander("results table").table(results) try: - bar_chart = px.bar(results, orientation='h', barmode='group') - self.container.expander('results chart').plotly_chart(bar_chart) + bar_chart = px.bar(results, orientation="h", barmode="group") + self.container.expander("results chart").plotly_chart(bar_chart) except: - self.container.text('Results are incomplete.') + self.container.text("Results are incomplete.") class QlibModelExpWindow(StWindow): @@ -334,44 +338,45 @@ class QlibModelExpWindow(StWindow): if self.show_task_info: _msg = deepcopy(msg) _msg.content = [ws for ws in exp.sub_workspace_list if ws] - self.container.markdown('**Model Tasks**') - ObjectsTabsWindow(self.container.container(), - inner_class=WorkspaceWindow, - mapper=lambda x: x.target_task.name, - ).consume_msg(_msg) + self.container.markdown("**Model Tasks**") + ObjectsTabsWindow( + self.container.container(), + inner_class=WorkspaceWindow, + mapper=lambda x: x.target_task.name, + ).consume_msg(_msg) # result - self.container.subheader('Results', divider=True) - results = pd.DataFrame({f'base_exp_{id}':e.result for id, e in enumerate(exp.based_experiments)}) - results['now'] = exp.result + self.container.subheader("Results", divider=True) + results = pd.DataFrame({f"base_exp_{id}": e.result for id, e in enumerate(exp.based_experiments)}) + results["now"] = exp.result - self.container.expander('results table').table(results) + self.container.expander("results table").table(results) class SimpleTraceWindow(StWindow): - - def __init__(self, container: 'DeltaGenerator' = st.container(), show_llm: bool = False, show_common_logs: bool = False): + def __init__( + self, container: "DeltaGenerator" = st.container(), show_llm: bool = False, show_common_logs: bool = False + ): super().__init__(container) self.show_llm = show_llm self.show_common_logs = show_common_logs - self.pid_trace = '' - self.current_tag = '' + self.pid_trace = "" + self.current_tag = "" self.current_win = StWindow(self.container) self.evolving_tasks: list[str] = [] def consume_msg(self, msg: Message): - # divide tag levels if len(msg.tag) > len(self.current_tag): # write a header about current task, if it is llm message, not write. - if not msg.tag.endswith('llm_messages'): - self.container.header(msg.tag.replace('.', ' ➡ '), divider=True) - + if not msg.tag.endswith("llm_messages"): + self.container.header(msg.tag.replace(".", " ➡ "), divider=True) + self.current_tag = msg.tag # set log writer (window) according to msg - if msg.tag.endswith('llm_messages'): + if msg.tag.endswith("llm_messages"): # llm messages logs if not self.show_llm: return @@ -392,29 +397,41 @@ class SimpleTraceWindow(StWindow): if len(msg.content) == 0: return if isinstance(msg.content[0], FactorTask): - self.current_win = ObjectsTabsWindow(self.container.expander('Factor Tasks'), FactorTaskWindow, lambda x: x.factor_name) + self.current_win = ObjectsTabsWindow( + self.container.expander("Factor Tasks"), FactorTaskWindow, lambda x: x.factor_name + ) elif isinstance(msg.content[0], ModelTask): - self.current_win = ObjectsTabsWindow(self.container.expander('Model Tasks'), ModelTaskWindow, lambda x: x.name) - + self.current_win = ObjectsTabsWindow( + self.container.expander("Model Tasks"), ModelTaskWindow, lambda x: x.name + ) + elif isinstance(msg.content[0], FactorFBWorkspace): - self.current_win = ObjectsTabsWindow(self.container.expander('Factor Workspaces'), - inner_class=WorkspaceWindow, - mapper=lambda x: x.target_task.factor_name) + self.current_win = ObjectsTabsWindow( + self.container.expander("Factor Workspaces"), + inner_class=WorkspaceWindow, + mapper=lambda x: x.target_task.factor_name, + ) self.evolving_tasks = [m.target_task.factor_name for m in msg.content] elif isinstance(msg.content[0], ModelFBWorkspace): - self.current_win = ObjectsTabsWindow(self.container.expander('Model Workspaces'), - inner_class=WorkspaceWindow, - mapper=lambda x: x.target_task.name) + self.current_win = ObjectsTabsWindow( + self.container.expander("Model Workspaces"), + inner_class=WorkspaceWindow, + mapper=lambda x: x.target_task.name, + ) self.evolving_tasks = [m.target_task.name for m in msg.content] elif isinstance(msg.content[0], FactorSingleFeedback): - self.current_win = ObjectsTabsWindow(self.container.expander('Factor Feedbacks'), - inner_class=FactorFeedbackWindow, - tab_names=self.evolving_tasks) + self.current_win = ObjectsTabsWindow( + self.container.expander("Factor Feedbacks"), + inner_class=FactorFeedbackWindow, + tab_names=self.evolving_tasks, + ) elif isinstance(msg.content[0], ModelCoderFeedback): - self.current_win = ObjectsTabsWindow(self.container.expander('Model Feedbacks'), - inner_class=ModelFeedbackWindow, - tab_names=self.evolving_tasks) + self.current_win = ObjectsTabsWindow( + self.container.expander("Model Feedbacks"), + inner_class=ModelFeedbackWindow, + tab_names=self.evolving_tasks, + ) else: # common logs if not self.show_common_logs: @@ -425,12 +442,11 @@ class SimpleTraceWindow(StWindow): def mock_msg(obj) -> Message: - return Message(tag='mock', level='INFO', timestamp=datetime.now(), pid_trace='000', caller='mock',content=obj) + return Message(tag="mock", level="INFO", timestamp=datetime.now(), pid_trace="000", caller="mock", content=obj) class TraceObjWindow(StWindow): - - def __init__(self, container: 'DeltaGenerator' = st.container()): + def __init__(self, container: "DeltaGenerator" = st.container()): self.container = container def consume_msg(self, msg: Message | Trace): @@ -440,7 +456,7 @@ class TraceObjWindow(StWindow): trace = msg for id, (h, e, hf) in enumerate(trace.hist): - self.container.header(f'Trace History {id}', divider=True) + self.container.header(f"Trace History {id}", divider=True) HypothesisWindow(self.container).consume_msg(mock_msg(h)) if isinstance(e, QlibFactorExperiment): QlibFactorExpWindow(self.container).consume_msg(mock_msg(e)) @@ -450,70 +466,74 @@ class TraceObjWindow(StWindow): class ResearchWindow(StWindow): - def consume_msg(self, msg: Message): - if msg.tag.endswith('hypothesis generation'): + if msg.tag.endswith("hypothesis generation"): HypothesisWindow(self.container.container()).consume_msg(msg) - elif msg.tag.endswith('experiment generation'): + elif msg.tag.endswith("experiment generation"): if isinstance(msg.content, list): if isinstance(msg.content[0], FactorTask): - self.container.markdown('**Factor Tasks**') - ObjectsTabsWindow(self.container.container(), FactorTaskWindow, lambda x: x.factor_name).consume_msg(msg) + self.container.markdown("**Factor Tasks**") + ObjectsTabsWindow( + self.container.container(), FactorTaskWindow, lambda x: x.factor_name + ).consume_msg(msg) elif isinstance(msg.content[0], ModelTask): - self.container.markdown('**Model Tasks**') + self.container.markdown("**Model Tasks**") ObjectsTabsWindow(self.container.container(), ModelTaskWindow, lambda x: x.name).consume_msg(msg) class EvolvingWindow(StWindow): - def __init__(self, container: 'DeltaGenerator'): + def __init__(self, container: "DeltaGenerator"): self.container = container self.evolving_tasks: list[str] = [] def consume_msg(self, msg: Message): - if msg.tag.endswith('evolving code'): + if msg.tag.endswith("evolving code"): if isinstance(msg.content, list): msg.content = [m for m in msg.content if m] if len(msg.content) == 0: return if isinstance(msg.content[0], FactorFBWorkspace): - self.container.markdown('**Factor Codes**') - ObjectsTabsWindow(self.container.container(), - inner_class=WorkspaceWindow, - mapper=lambda x: x.target_task.factor_name).consume_msg(msg) + self.container.markdown("**Factor Codes**") + ObjectsTabsWindow( + self.container.container(), + inner_class=WorkspaceWindow, + mapper=lambda x: x.target_task.factor_name, + ).consume_msg(msg) self.evolving_tasks = [m.target_task.factor_name for m in msg.content] elif isinstance(msg.content[0], ModelFBWorkspace): - self.container.markdown('**Model Codes**') - ObjectsTabsWindow(self.container.container(), - inner_class=WorkspaceWindow, - mapper=lambda x: x.target_task.name).consume_msg(msg) + self.container.markdown("**Model Codes**") + ObjectsTabsWindow( + self.container.container(), inner_class=WorkspaceWindow, mapper=lambda x: x.target_task.name + ).consume_msg(msg) self.evolving_tasks = [m.target_task.name for m in msg.content] - elif msg.tag.endswith('evolving feedback'): + elif msg.tag.endswith("evolving feedback"): if isinstance(msg.content, list): msg.content = [m for m in msg.content if m] if len(msg.content) == 0: return if isinstance(msg.content[0], FactorSingleFeedback): - self.container.markdown('**Factor Feedbacks🔍**') - ObjectsTabsWindow(self.container.container(), - inner_class=FactorFeedbackWindow, - tab_names=self.evolving_tasks).consume_msg(msg) + self.container.markdown("**Factor Feedbacks🔍**") + ObjectsTabsWindow( + self.container.container(), inner_class=FactorFeedbackWindow, tab_names=self.evolving_tasks + ).consume_msg(msg) elif isinstance(msg.content[0], ModelCoderFeedback): - self.container.markdown('**Model Feedbacks🔍**') - ObjectsTabsWindow(self.container.container(), - inner_class=ModelFeedbackWindow, - tab_names=self.evolving_tasks).consume_msg(msg) + self.container.markdown("**Model Feedbacks🔍**") + ObjectsTabsWindow( + self.container.container(), inner_class=ModelFeedbackWindow, tab_names=self.evolving_tasks + ).consume_msg(msg) class DevelopmentWindow(StWindow): - - def __init__(self, container: 'DeltaGenerator'): - self.E_win = RoundTabsWindow(container.container(), - new_tab_func=lambda x: x.tag.endswith('evolving code'), - inner_class=EvolvingWindow, - title='Evolving Loops🔧') + def __init__(self, container: "DeltaGenerator"): + self.E_win = RoundTabsWindow( + container.container(), + new_tab_func=lambda x: x.tag.endswith("evolving code"), + inner_class=EvolvingWindow, + title="Evolving Loops🔧", + ) def consume_msg(self, msg: Message): - if 'evolving' in msg.tag: + if "evolving" in msg.tag: self.E_win.consume_msg(msg) # elif msg.tag.endswith('result'): # self.container.subheader('Results') @@ -528,8 +548,7 @@ class DevelopmentWindow(StWindow): class FeedbackWindow(StWindow): - - def __init__(self, container: 'DeltaGenerator'): + def __init__(self, container: "DeltaGenerator"): self.container = container def consume_msg(self, msg: Message): @@ -542,8 +561,7 @@ class FeedbackWindow(StWindow): class SingleRDLoopWindow(StWindow): - - def __init__(self, container: 'DeltaGenerator'): + def __init__(self, container: "DeltaGenerator"): self.container = container col1, col2 = self.container.columns([2, 3]) self.R_win = ResearchWindow(col1.container(border=True)) @@ -551,58 +569,66 @@ class SingleRDLoopWindow(StWindow): self.D_win = DevelopmentWindow(col2.container(border=True)) def consume_msg(self, msg: Message): - tags = msg.tag.split('.') - if 'r' in tags: + tags = msg.tag.split(".") + if "r" in tags: self.R_win.consume_msg(msg) - elif 'd' in tags: + elif "d" in tags: self.D_win.consume_msg(msg) - elif 'ef' in tags: + elif "ef" in tags: self.F_win.consume_msg(msg) class TraceWindow(StWindow): - - def __init__(self, container: 'DeltaGenerator' = st.container(), show_llm: bool = False, show_common_logs: bool = False): + def __init__( + self, container: "DeltaGenerator" = st.container(), show_llm: bool = False, show_common_logs: bool = False + ): self.show_llm = show_llm self.show_common_logs = show_common_logs top_container = container.container() - col1, col2 = top_container.columns([2,3]) + col1, col2 = top_container.columns([2, 3]) chart_c = col2.container(border=True, height=300) - chart_c.markdown('**Metrics📈**') + chart_c.markdown("**Metrics📈**") self.chart_c = chart_c.empty() hypothesis_status_c = col1.container(border=True, height=300) - hypothesis_status_c.markdown('**Hypotheses🏅**') + hypothesis_status_c.markdown("**Hypotheses🏅**") self.summary_c = hypothesis_status_c.empty() - self.RDL_win = RoundTabsWindow(container.container(), - new_tab_func=lambda x: x.tag.endswith('hypothesis generation'), - inner_class=SingleRDLoopWindow, - title='R&D Loops♾️') - + self.RDL_win = RoundTabsWindow( + container.container(), + new_tab_func=lambda x: x.tag.endswith("hypothesis generation"), + inner_class=SingleRDLoopWindow, + title="R&D Loops♾️", + ) + self.hypothesis_decisions = defaultdict(bool) self.current_hypothesis = None self.results = [] def consume_msg(self, msg: Message): - if not self.show_llm and 'llm_messages' in msg.tag: + if not self.show_llm and "llm_messages" in msg.tag: return if not self.show_common_logs and isinstance(msg.content, str): return if isinstance(msg.content, dict): return - if msg.tag.endswith('hypothesis generation'): + if msg.tag.endswith("hypothesis generation"): self.current_hypothesis = msg.content.hypothesis - elif msg.tag.endswith('ef.feedback'): + elif msg.tag.endswith("ef.feedback"): self.hypothesis_decisions[self.current_hypothesis] = msg.content.decision - self.summary_c.markdown('\n'.join(f"{id+1}. :green[{h}]\n" if d else f"{id+1}. {h}\n" for id,(h,d) in enumerate(self.hypothesis_decisions.items()))) - elif msg.tag.endswith('ef.model runner result') or msg.tag.endswith('ef.factor runner result'): + self.summary_c.markdown( + "\n".join( + f"{id+1}. :green[{h}]\n" if d else f"{id+1}. {h}\n" + for id, (h, d) in enumerate(self.hypothesis_decisions.items()) + ) + ) + elif msg.tag.endswith("ef.model runner result") or msg.tag.endswith("ef.factor runner result"): self.results.append(msg.content.result) if len(self.results) == 1: self.chart_c.table(self.results[0]) else: - df = pd.DataFrame(self.results, index=range(1, len(self.results)+1)) + df = pd.DataFrame(self.results, index=range(1, len(self.results) + 1)) fig = px.line(df, x=df.index, y=df.columns, markers=True) self.chart_c.plotly_chart(fig) diff --git a/rdagent/log/utils.py b/rdagent/log/utils.py index 746db520..b87682bc 100644 --- a/rdagent/log/utils.py +++ b/rdagent/log/utils.py @@ -1,7 +1,6 @@ import inspect import re - -from typing import Union, Dict, TypedDict, Optional +from typing import Dict, Optional, TypedDict, Union class LogColors: diff --git a/rdagent/scenarios/data_mining/developer/feedback.py b/rdagent/scenarios/data_mining/developer/feedback.py index 3f30c0fc..af18f0c4 100644 --- a/rdagent/scenarios/data_mining/developer/feedback.py +++ b/rdagent/scenarios/data_mining/developer/feedback.py @@ -18,7 +18,7 @@ from rdagent.log import rdagent_logger as logger from rdagent.oai.llm_utils import APIBackend from rdagent.utils import convert2bool -feedback_prompts = Prompts(file_path=Path(__file__).parent.parent.parent/ "qlib" / "prompts.yaml") +feedback_prompts = Prompts(file_path=Path(__file__).parent.parent.parent / "qlib" / "prompts.yaml") DIRNAME = Path(__file__).absolute().resolve().parent diff --git a/rdagent/scenarios/data_mining/developer/model_runner.py b/rdagent/scenarios/data_mining/developer/model_runner.py index 8b643bf9..a3ef7e3d 100644 --- a/rdagent/scenarios/data_mining/developer/model_runner.py +++ b/rdagent/scenarios/data_mining/developer/model_runner.py @@ -15,7 +15,6 @@ from rdagent.utils.env import DMDockerEnv class DMModelRunner(CachedRunner[DMModelExperiment]): - def develop(self, exp: DMModelExperiment) -> DMModelExperiment: if RUNNER_SETTINGS.cache_result: cache_hit, result = self.get_cache_result(exp) diff --git a/rdagent/scenarios/data_mining/experiment/model_experiment.py b/rdagent/scenarios/data_mining/experiment/model_experiment.py index f737f83d..216524cf 100644 --- a/rdagent/scenarios/data_mining/experiment/model_experiment.py +++ b/rdagent/scenarios/data_mining/experiment/model_experiment.py @@ -22,11 +22,11 @@ class DMModelScenario(Scenario): @property def background(self) -> str: return prompt_dict["dm_model_background"] - + @property def source_data(self) -> str: raise NotImplementedError("source_data is not implemented") - + @property def output_format(self) -> str: return prompt_dict["dm_model_output_format"] @@ -38,9 +38,9 @@ class DMModelScenario(Scenario): @property def simulator(self) -> str: return prompt_dict["dm_model_simulator"] - + @property - def rich_style_description(self)->str: + def rich_style_description(self) -> str: return "Below is MIMIC Model Evolving Automatic R&D Demo." def get_scenario_all_desc(self) -> str: diff --git a/rdagent/scenarios/data_mining/experiment/model_template/train.py b/rdagent/scenarios/data_mining/experiment/model_template/train.py index 74440e1c..05a670c9 100644 --- a/rdagent/scenarios/data_mining/experiment/model_template/train.py +++ b/rdagent/scenarios/data_mining/experiment/model_template/train.py @@ -1,21 +1,23 @@ -from pathlib import Path -import pandas as pd -import torch -import torch.nn.functional as F -from torchvision import transforms, datasets -from torch.utils.data import DataLoader, Dataset -import torch.nn as nn -import sparse -import random import os +import random +from pathlib import Path + +import numpy as np +import pandas as pd +import sparse +import torch +import torch.nn as nn +import torch.nn.functional as F from model import model_cls from sklearn.metrics import accuracy_score, roc_auc_score -import numpy as np +from torch.utils.data import DataLoader, Dataset +from torchvision import datasets, transforms # Set device for training device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") # device = torch.device("cpu") + class MyDataset(Dataset): def __init__(self, x, label, device): self.x1 = x @@ -28,9 +30,11 @@ class MyDataset(Dataset): def __getitem__(self, idx): if torch.is_tensor(idx): idx = idx.tolist() - return torch.FloatTensor(self.x1[idx]).to(self.device), \ - torch.tensor(self.label[idx], dtype=torch.float).to(self.device) - + return torch.FloatTensor(self.x1[idx]).to(self.device), torch.tensor(self.label[idx], dtype=torch.float).to( + self.device + ) + + def collate_fn(batch): x, label = [], [] for data in batch: @@ -39,12 +43,12 @@ def collate_fn(batch): return torch.stack(x, 0), torch.stack(label, 0) -datapath = '/root/.data' +datapath = "/root/.data" # datapath = '/home/v-suhancui/RD-Agent/physionet.org/files/mimic-eicu-fiddle-feature/1.0.0/FIDDLE_mimic3' -X = sparse.load_npz(datapath+'/features/ARF_12h/X.npz').todense() -df_pop = pd.read_csv(datapath+'/population/ARF_12h.csv')['ARF_LABEL'] +X = sparse.load_npz(datapath + "/features/ARF_12h/X.npz").todense() +df_pop = pd.read_csv(datapath + "/population/ARF_12h.csv")["ARF_LABEL"] X = X.transpose(0, 2, 1) @@ -56,8 +60,12 @@ X_train, y_train = X[indices[:split_point]], np.array(df_pop[indices[:split_poin X_test, y_test = X[indices[split_point:]], np.array(df_pop[indices[split_point:]]) -train_dataloader = DataLoader(MyDataset(X_train, y_train, device), collate_fn=collate_fn, shuffle=True, drop_last=True, batch_size=64) -test_dataloader = DataLoader(MyDataset(X_test, y_test, device), collate_fn=collate_fn, shuffle=False, drop_last=False, batch_size=64) +train_dataloader = DataLoader( + MyDataset(X_train, y_train, device), collate_fn=collate_fn, shuffle=True, drop_last=True, batch_size=64 +) +test_dataloader = DataLoader( + MyDataset(X_test, y_test, device), collate_fn=collate_fn, shuffle=False, drop_last=False, batch_size=64 +) num_features = 4816 num_timesteps = 12 @@ -88,5 +96,5 @@ acc = roc_auc_score(y_test, np.concatenate(y_pred)) print(acc) # Save the predictions to submission.csv -with open('./submission.txt', 'w') as f: - f.write(str(acc)) \ No newline at end of file +with open("./submission.txt", "w") as f: + f.write(str(acc)) diff --git a/rdagent/scenarios/data_mining/experiment/workspace.py b/rdagent/scenarios/data_mining/experiment/workspace.py index f70fd090..e2e9d3a1 100644 --- a/rdagent/scenarios/data_mining/experiment/workspace.py +++ b/rdagent/scenarios/data_mining/experiment/workspace.py @@ -2,10 +2,11 @@ from pathlib import Path import pandas as pd +from rdagent.app.data_mining.conf import PROP_SETTING from rdagent.core.experiment import FBWorkspace from rdagent.log import rdagent_logger as logger from rdagent.utils.env import DMDockerEnv -from rdagent.app.data_mining.conf import PROP_SETTING + class DMFBWorkspace(FBWorkspace): def __init__(self, template_folder_path: Path, *args, **kwargs) -> None: @@ -27,5 +28,5 @@ class DMFBWorkspace(FBWorkspace): if not csv_path.exists(): logger.error(f"File {csv_path} does not exist.") return None - with open(self.workspace_path / "submission.txt", 'r') as f: + with open(self.workspace_path / "submission.txt", "r") as f: return f.read() diff --git a/rdagent/scenarios/data_mining/proposal/model_proposal.py b/rdagent/scenarios/data_mining/proposal/model_proposal.py index 32ec8896..064a1a9d 100644 --- a/rdagent/scenarios/data_mining/proposal/model_proposal.py +++ b/rdagent/scenarios/data_mining/proposal/model_proposal.py @@ -30,6 +30,7 @@ class DMModelHypothesisGen(ModelHypothesisGen): class XXXDMModelHypothesisGen(DMModelHypothesisGen): prompts: Prompts = a_specifc_prompt_dict """ + def __init__(self, scen: Scenario) -> Tuple[dict, bool]: super().__init__(scen) @@ -43,7 +44,7 @@ class DMModelHypothesisGen(ModelHypothesisGen): "hypothesis_and_feedback": hypothesis_feedback, "RAG": "", "hypothesis_output_format": prompt_dict["hypothesis_output_format"], - "hypothesis_specification": prompt_dict["model_hypothesis_specification"] + "hypothesis_specification": prompt_dict["model_hypothesis_specification"], } return context_dict, True @@ -89,7 +90,9 @@ class DMModelHypothesis2Experiment(ModelHypothesis2Experiment): variables = response_dict[model_name]["variables"] hyperparameters = response_dict[model_name]["hyperparameters"] model_type = response_dict[model_name]["model_type"] - tasks.append(ModelTask(model_name, description, formulation, architecture, variables, hyperparameters, model_type)) + tasks.append( + ModelTask(model_name, description, formulation, architecture, variables, hyperparameters, model_type) + ) exp = DMModelExperiment(tasks) exp.based_experiments = [t[1] for t in trace.hist if t[2]] return exp diff --git a/rdagent/scenarios/qlib/developer/feedback.py b/rdagent/scenarios/qlib/developer/feedback.py index 612e09f1..7e62071f 100644 --- a/rdagent/scenarios/qlib/developer/feedback.py +++ b/rdagent/scenarios/qlib/developer/feedback.py @@ -1,8 +1,8 @@ import json from pathlib import Path -from jinja2 import Environment, StrictUndefined import pandas as pd +from jinja2 import Environment, StrictUndefined from rdagent.core.experiment import Experiment from rdagent.core.prompts import Prompts @@ -19,37 +19,38 @@ from rdagent.utils import convert2bool feedback_prompts = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml") DIRNAME = Path(__file__).absolute().resolve().parent + def process_results(current_result, sota_result): # Convert the results to dataframes current_df = pd.DataFrame(current_result) sota_df = pd.DataFrame(sota_result) - + # Set the metric as the index - current_df.index.name = 'metric' - sota_df.index.name = 'metric' - + current_df.index.name = "metric" + sota_df.index.name = "metric" + # Rename the value column to reflect the result type - current_df.rename(columns={'0': 'Current Result'}, inplace=True) - sota_df.rename(columns={'0': 'SOTA Result'}, inplace=True) - + current_df.rename(columns={"0": "Current Result"}, inplace=True) + sota_df.rename(columns={"0": "SOTA Result"}, inplace=True) + # Combine the dataframes on the Metric index combined_df = pd.concat([current_df, sota_df], axis=1) - + # Select important metrics for comparison important_metrics = [ - 'Rank ICIR', - '1day.excess_return_without_cost.max_drawdown', - '1day.excess_return_without_cost.information_ratio', - '1day.excess_return_without_cost.annualized_return', - '1day.excess_return_with_cost.max_drawdown', - '1day.excess_return_with_cost.information_ratio', - '1day.excess_return_with_cost.annualized_return', - 'IC', + "Rank ICIR", + "1day.excess_return_without_cost.max_drawdown", + "1day.excess_return_without_cost.information_ratio", + "1day.excess_return_without_cost.annualized_return", + "1day.excess_return_with_cost.max_drawdown", + "1day.excess_return_with_cost.information_ratio", + "1day.excess_return_with_cost.annualized_return", + "IC", ] - + # Filter the combined DataFrame to retain only the important metrics filtered_combined_df = combined_df.loc[important_metrics] - + return filtered_combined_df.to_dict() diff --git a/rdagent/scenarios/qlib/experiment/factor_experiment.py b/rdagent/scenarios/qlib/experiment/factor_experiment.py index 09e166d5..f245cffe 100644 --- a/rdagent/scenarios/qlib/experiment/factor_experiment.py +++ b/rdagent/scenarios/qlib/experiment/factor_experiment.py @@ -39,9 +39,9 @@ class QlibFactorScenario(Scenario): @property def simulator(self) -> str: return prompt_dict["qlib_factor_simulator"] - + @property - def rich_style_description(self)->str: + def rich_style_description(self) -> str: return "Below is QlibFactor Evolving Automatic R&D Demo." def get_scenario_all_desc(self) -> str: diff --git a/rdagent/scenarios/qlib/experiment/factor_template/read_exp_res.py b/rdagent/scenarios/qlib/experiment/factor_template/read_exp_res.py index d08e3b4c..928cc645 100644 --- a/rdagent/scenarios/qlib/experiment/factor_template/read_exp_res.py +++ b/rdagent/scenarios/qlib/experiment/factor_template/read_exp_res.py @@ -45,4 +45,4 @@ else: print(f"Output has been saved to {output_path}") ret_data_frame = latest_recorder.load_object("portfolio_analysis/report_normal_1day.pkl") - ret_data_frame.to_pickle("ret.pkl") \ No newline at end of file + ret_data_frame.to_pickle("ret.pkl") diff --git a/rdagent/scenarios/qlib/experiment/model_experiment.py b/rdagent/scenarios/qlib/experiment/model_experiment.py index 3167c4f0..f1c99c8d 100644 --- a/rdagent/scenarios/qlib/experiment/model_experiment.py +++ b/rdagent/scenarios/qlib/experiment/model_experiment.py @@ -38,10 +38,10 @@ class QlibModelScenario(Scenario): @property def simulator(self) -> str: return prompt_dict["qlib_model_simulator"] - + @property - def rich_style_description(self)->str: - return ''' + def rich_style_description(self) -> str: + return """ ### Qlib Model Evolving Automatic R&D Demo #### Overview @@ -75,7 +75,7 @@ The demo showcases the iterative process of hypothesis generation, knowledge con To demonstrate the dynamic evolution of models through the Qlib platform, emphasizing how each iteration enhances the accuracy and reliability of the resulting models. - ''' + """ def get_scenario_all_desc(self) -> str: return f"""Background of the scenario: diff --git a/rdagent/scenarios/qlib/experiment/model_template/read_exp_res.py b/rdagent/scenarios/qlib/experiment/model_template/read_exp_res.py index 01976cb9..928cc645 100644 --- a/rdagent/scenarios/qlib/experiment/model_template/read_exp_res.py +++ b/rdagent/scenarios/qlib/experiment/model_template/read_exp_res.py @@ -39,7 +39,6 @@ else: # Load the specified file from the latest recorder metrics = pd.Series(latest_recorder.list_metrics()) - output_path = Path(__file__).resolve().parent / "qlib_res.csv" metrics.to_csv(output_path) diff --git a/rdagent/scenarios/qlib/factor_experiment_loader/json_loader.py b/rdagent/scenarios/qlib/factor_experiment_loader/json_loader.py index 28c37b10..5f080f80 100644 --- a/rdagent/scenarios/qlib/factor_experiment_loader/json_loader.py +++ b/rdagent/scenarios/qlib/factor_experiment_loader/json_loader.py @@ -56,9 +56,7 @@ class FactorTestCaseLoaderFromJsonFile: variables=factor_data["variables"], ) gt = FactorFBWorkspace(task) - code = { - "factor.py": factor_data["gt_code"] - } + code = {"factor.py": factor_data["gt_code"]} gt.inject_code(**code) gt.execute() TestData.target_task.sub_tasks.append(task) diff --git a/rdagent/scenarios/qlib/factor_experiment_loader/pdf_loader.py b/rdagent/scenarios/qlib/factor_experiment_loader/pdf_loader.py index d3d4f471..b0c5e9ee 100644 --- a/rdagent/scenarios/qlib/factor_experiment_loader/pdf_loader.py +++ b/rdagent/scenarios/qlib/factor_experiment_loader/pdf_loader.py @@ -196,7 +196,11 @@ def __extract_factor_and_formulation_from_one_report( factor_dict, ) for factor_name in factor_dict: - if factor_name not in factor_to_formulation or "formulation" not in factor_to_formulation[factor_name] or "variables" not in factor_to_formulation[factor_name]: + if ( + factor_name not in factor_to_formulation + or "formulation" not in factor_to_formulation[factor_name] + or "variables" not in factor_to_formulation[factor_name] + ): continue final_factor_dict_to_one_report.setdefault(factor_name, {}) @@ -272,7 +276,6 @@ def merge_file_to_factor_dict_to_factor_dict( def __check_factor_dict_viability_simulate_json_mode( factor_df_string: str, ) -> dict[str, dict[str, str]]: - extract_result_resp = APIBackend().build_messages_and_create_chat_completion( system_prompt=document_process_prompts["factor_viability_system"], user_prompt=factor_df_string, @@ -512,11 +515,11 @@ class FactorExperimentLoaderFromPDFfiles(FactorExperimentLoader): logger.log_object(docs_dict) selected_report_dict = classify_report_from_dict(report_dict=docs_dict, vote_time=1) - + with logger.tag("file_to_factor_result"): file_to_factor_result = extract_factors_from_report_dict(docs_dict, selected_report_dict) logger.log_object(file_to_factor_result) - + with logger.tag("factor_dict"): factor_dict = merge_file_to_factor_dict_to_factor_dict(file_to_factor_result) logger.log_object(factor_dict) @@ -526,5 +529,5 @@ class FactorExperimentLoaderFromPDFfiles(FactorExperimentLoader): logger.log_object(filtered_factor_dict) # factor_dict, duplication_names_list = deduplicate_factors_by_llm(factor_dict, factor_viability) - + return FactorExperimentLoaderFromDict().load(filtered_factor_dict) diff --git a/rdagent/scenarios/qlib/proposal/factor_proposal.py b/rdagent/scenarios/qlib/proposal/factor_proposal.py index eea5fa5e..871c3354 100644 --- a/rdagent/scenarios/qlib/proposal/factor_proposal.py +++ b/rdagent/scenarios/qlib/proposal/factor_proposal.py @@ -39,7 +39,11 @@ class QlibFactorHypothesisGen(FactorHypothesisGen): def convert_response(self, response: str) -> FactorHypothesis: response_dict = json.loads(response) - hypothesis = QlibFactorHypothesis(hypothesis=response_dict["hypothesis"], reason=response_dict["reason"], concise_reason=response_dict["concise_reason"]) + hypothesis = QlibFactorHypothesis( + hypothesis=response_dict["hypothesis"], + reason=response_dict["reason"], + concise_reason=response_dict["concise_reason"], + ) return hypothesis @@ -78,15 +82,15 @@ class QlibFactorHypothesis2Experiment(FactorHypothesis2Experiment): formulation = response_dict[factor_name]["formulation"] variables = response_dict[factor_name]["variables"] tasks.append(FactorTask(factor_name, description, formulation, variables)) - + exp = QlibFactorExperiment(tasks) exp.based_experiments = [t[1] for t in trace.hist if t[2]] - + if len(exp.based_experiments) == 0: exp.based_experiments.append(QlibFactorExperiment(sub_tasks=[])) unique_tasks = [] - + for task in tasks: duplicate = False for based_exp in exp.based_experiments: diff --git a/rdagent/scenarios/qlib/proposal/model_proposal.py b/rdagent/scenarios/qlib/proposal/model_proposal.py index fb2bcefa..796e0a1c 100644 --- a/rdagent/scenarios/qlib/proposal/model_proposal.py +++ b/rdagent/scenarios/qlib/proposal/model_proposal.py @@ -33,13 +33,17 @@ class QlibModelHypothesisGen(ModelHypothesisGen): "hypothesis_and_feedback": hypothesis_feedback, "RAG": "In Quantitative Finance, market data could be time-series, and GRU model/LSTM model are suitable for them. Do not generate GNN model as for now.", "hypothesis_output_format": prompt_dict["hypothesis_output_format"], - "hypothesis_specification": prompt_dict["model_hypothesis_specification"] + "hypothesis_specification": prompt_dict["model_hypothesis_specification"], } return context_dict, True def convert_response(self, response: str) -> ModelHypothesis: response_dict = json.loads(response) - hypothesis = QlibModelHypothesis(hypothesis=response_dict["hypothesis"], reason=response_dict["reason"], concise_reason=response_dict["concise_reason"]) + hypothesis = QlibModelHypothesis( + hypothesis=response_dict["hypothesis"], + reason=response_dict["reason"], + concise_reason=response_dict["concise_reason"], + ) return hypothesis @@ -79,7 +83,9 @@ class QlibModelHypothesis2Experiment(ModelHypothesis2Experiment): variables = response_dict[model_name]["variables"] hyperparameters = response_dict[model_name]["hyperparameters"] model_type = response_dict[model_name]["model_type"] - tasks.append(ModelTask(model_name, description, formulation, architecture, variables, hyperparameters, model_type)) + tasks.append( + ModelTask(model_name, description, formulation, architecture, variables, hyperparameters, model_type) + ) exp = QlibModelExperiment(tasks) exp.based_experiments = [t[1] for t in trace.hist if t[2]] return exp diff --git a/rdagent/utils/agent/ret.py b/rdagent/utils/agent/ret.py index 8c8c29f4..6aad3433 100644 --- a/rdagent/utils/agent/ret.py +++ b/rdagent/utils/agent/ret.py @@ -3,8 +3,8 @@ The output of a agent is very important. We think this part can be shared. """ -from abc import abstractclassmethod import re +from abc import abstractclassmethod from typing import Any from rdagent.utils.agent.tpl import T diff --git a/rdagent/utils/agent/tpl.py b/rdagent/utils/agent/tpl.py index e7ec2c56..a3047b9f 100644 --- a/rdagent/utils/agent/tpl.py +++ b/rdagent/utils/agent/tpl.py @@ -4,13 +4,12 @@ Here are some infrastruture to build a agent The motivation of tempalte and AgentOutput Design """ -from typing import Any -from jinja2 import Environment, StrictUndefined - +import inspect from pathlib import Path +from typing import Any import yaml -import inspect +from jinja2 import Environment, StrictUndefined from rdagent.core.utils import SingletonBaseClass @@ -21,9 +20,10 @@ PROJ_PATH = DIRNAME.parent.parent # class T(SingletonBaseClass): TODO: singleton does not support args now. class T: """Use the simplest way to (C)reate a Template and (r)ender it!!""" + def __init__(self, uri: str): """ - here are some uri usages + here are some uri usages case 1) "a.b.c:x.y.z" It will load DIRNAME/a/b/c.yaml as `yaml` and load yaml[x][y][z] case 2) ".c:x.y.z" @@ -38,16 +38,16 @@ class T: caller_dir = Path(caller_module.__file__).parent # Parse the URI - path_part, yaml_path = uri.split(':') - yaml_keys = yaml_path.split('.') + path_part, yaml_path = uri.split(":") + yaml_keys = yaml_path.split(".") - if path_part.startswith('.'): + if path_part.startswith("."): yaml_file_path = caller_dir / f"{path_part[1:].replace('.', '/')}.yaml" else: - yaml_file_path = (PROJ_PATH / path_part.replace('.', '/')).with_suffix('.yaml') + yaml_file_path = (PROJ_PATH / path_part.replace(".", "/")).with_suffix(".yaml") # Load the YAML file - with open(yaml_file_path, 'r') as file: + with open(yaml_file_path, "r") as file: yaml_content = yaml.safe_load(file) # Traverse the YAML content to get the desired template diff --git a/rdagent/utils/env.py b/rdagent/utils/env.py index 1836566e..c062aaea 100644 --- a/rdagent/utils/env.py +++ b/rdagent/utils/env.py @@ -144,16 +144,21 @@ class QlibDockerConf(DockerConf): class DMDockerConf(DockerConf): class Config: - env_prefix = "DM_DOCKER_" + env_prefix = "DM_DOCKER_" build_from_dockerfile: bool = True dockerfile_folder_path: Path = Path(__file__).parent.parent / "scenarios" / "data_mining" / "docker" image: str = "local_dm:latest" mount_path: str = "/workspace/dm_workspace/" default_entry: str = "python train.py" - extra_volumes: dict = {Path("~/.rdagent/.data/physionet.org/files/mimic-eicu-fiddle-feature/1.0.0/FIDDLE_mimic3/").expanduser().resolve(): "/root/.data/"} + extra_volumes: dict = { + Path("~/.rdagent/.data/physionet.org/files/mimic-eicu-fiddle-feature/1.0.0/FIDDLE_mimic3/") + .expanduser() + .resolve(): "/root/.data/" + } shm_size: str | None = "16g" + # physionet.org/files/mimic-eicu-fiddle-feature/1.0.0/FIDDLE_mimic3 class DockerEnv(Env[DockerConf]): # TODO: Save the output into a specific file @@ -181,9 +186,9 @@ class DockerEnv(Env[DockerConf]): if not self.conf.enable_gpu: return {} gpu_kwargs = { - "device_requests": [ - docker.types.DeviceRequest(count=-1, capabilities=[['gpu']]) - ] if self.conf.enable_gpu else None, + "device_requests": [docker.types.DeviceRequest(count=-1, capabilities=[["gpu"]])] + if self.conf.enable_gpu + else None, } try: client.containers.run(self.conf.image, "nvidia-smi", **gpu_kwargs) @@ -220,7 +225,7 @@ class DockerEnv(Env[DockerConf]): # auto_remove=True, # remove too fast might cause the logs not to be get network=self.conf.network, shm_size=self.conf.shm_size, - **self._gpu_kwargs(client) + **self._gpu_kwargs(client), ) logs = container.logs(stream=True) for log in logs: @@ -273,7 +278,9 @@ class DMDockerEnv(DockerEnv): data_path = next(iter(self.conf.extra_volumes.keys())) if not (Path(data_path)).exists(): logger.info("We are downloading!") - cmd = 'wget -r -N -c -np --user={} --password={} -P ~/.rdagent/.data/ https://physionet.org/files/mimic-eicu-fiddle-feature/1.0.0/'.format(username, password) + cmd = "wget -r -N -c -np --user={} --password={} -P ~/.rdagent/.data/ https://physionet.org/files/mimic-eicu-fiddle-feature/1.0.0/".format( + username, password + ) os.system(cmd) else: logger.info("Data already exists. Download skipped.") diff --git a/rdagent/utils/workflow.py b/rdagent/utils/workflow.py index 605bfcfa..63da2dbf 100644 --- a/rdagent/utils/workflow.py +++ b/rdagent/utils/workflow.py @@ -7,20 +7,19 @@ Postscripts: However, Python generator is not picklable (dill does not support pickle as well) """ -from pathlib import Path +import datetime import pickle -from tqdm.auto import tqdm - - from collections import defaultdict from dataclasses import dataclass, field -import datetime +from pathlib import Path from typing import Callable + +from tqdm.auto import tqdm + from rdagent.log import rdagent_logger as logger class LoopMeta(type): - @staticmethod def _get_steps(bases): """ @@ -28,7 +27,7 @@ class LoopMeta(type): """ steps = [] for base in bases: - steps.extend(LoopMeta._get_steps(base.__bases__) + getattr(base,"steps", [])) + steps.extend(LoopMeta._get_steps(base.__bases__) + getattr(base, "steps", [])) return steps def __new__(cls, clsname, bases, attrs): @@ -52,12 +51,14 @@ class LoopBase: steps: list[Callable] # a list of steps to work on loop_trace: dict[int, list[LoopTrace]] - skip_loop_error: tuple[Exception] = field(default_factory=tuple) # you can define a list of error that will skip current loop + skip_loop_error: tuple[Exception] = field( + default_factory=tuple + ) # you can define a list of error that will skip current loop def __init__(self): - self.loop_idx = 0 # current loop index - self.step_idx = 0 # the index of next step to be run - self.loop_prev_out = {} # the step results of current loop + self.loop_idx = 0 # current loop index + self.step_idx = 0 # the index of next step to be run + self.loop_prev_out = {} # the step results of current loop self.loop_trace = defaultdict(list[LoopTrace]) # the key is the number of loop self.session_folder = logger.log_trace_path / "__session__" @@ -121,7 +122,7 @@ class LoopBase: with path.open("rb") as f: session = pickle.load(f) logger.set_trace_path(session.session_folder.parent) - + max_loop = max(session.loop_trace.keys()) logger.storage.truncate(time=session.loop_trace[max_loop][-1].end) return session diff --git a/test/utils/test_agent_infra.py b/test/utils/test_agent_infra.py index ea1ce25d..4c41293f 100644 --- a/test/utils/test_agent_infra.py +++ b/test/utils/test_agent_infra.py @@ -5,12 +5,8 @@ from rdagent.utils.agent.ret import PythonAgentOut from rdagent.utils.agent.tpl import T - - class TestAgentInfra(unittest.TestCase): - def test_agent_infra(self): - # NOTE: It is not serious. It is just for testing sys_prompt = T("components.proposal.prompts:hypothesis_gen.system_prompt").r( targets="targets", diff --git a/test/utils/test_misc.py b/test/utils/test_misc.py index 968b4af3..5d54770d 100644 --- a/test/utils/test_misc.py +++ b/test/utils/test_misc.py @@ -9,7 +9,6 @@ class A(SingletonBaseClass): class MiscTest(unittest.TestCase): - def test_singleton(self): a1 = A() a2 = A()