diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d243bfb1..f47df5cc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,12 +29,10 @@ jobs: EMBEDDING_AZURE_API_VERSION: ${{ secrets.CHAT_AZURE_API_VERSION }} EMBEDDING_MODEL: ${{ secrets.EMBEDDING_MODEL }} name: lint test docs and build - run: make lint test docs build sphinx + run: make lint # test docs build sphinx strategy: matrix: python-version: - - '3.8' - - '3.9' - '3.10' - '3.11' dependabot: @@ -56,16 +54,12 @@ jobs: with: cache: pip python-version: | - 3.8 - 3.9 3.10 3.11 - name: Install pipenv using pipx run: pipx install pipenv - name: Generate constraints for all supported Python versions run: | - CI= PYTHON_VERSION=3.8 make constraints - CI= PYTHON_VERSION=3.9 make constraints CI= PYTHON_VERSION=3.10 make constraints CI= PYTHON_VERSION=3.11 make constraints - name: Push changes if applicable diff --git a/Makefile b/Makefile index a2dadb58..2f1ce9d3 100644 --- a/Makefile +++ b/Makefile @@ -84,28 +84,31 @@ black: $(PIPRUN) python -m black --check . --extend-exclude test/scripts --extend-exclude git_ignore_folder -l 120 sphinx: - cd docs - $(PIPRUN) sphinx-build -W --keep-going -b html . _build - cd .. + $(PIPRUN) sphinx-build -W --keep-going -b html ./docs _build # Check lint with isort. isort: $(PIPRUN) python -m isort --check . -s git_ignore_folder -s test/scripts # Check lint with mypy. +# First deal with the core folder, and then gradually increase the scope of detection, +# and eventually realize the detection of the complete project. mypy: - $(PIPRUN) python -m mypy . --exclude rdagent/scripts --exclude git_ignore_folder + $(PIPRUN) python -m mypy rdagent/core # --exclude rdagent/scripts,git_ignore_folder # Check lint with ruff. +# First deal with the core folder, and then gradually increase the scope of detection, +# and eventually realize the detection of the complete project. ruff: - $(PIPRUN) ruff check . --exclude rdagent/scripts,git_ignore_folder + $(PIPRUN) ruff check rdagent/core --ignore FBT001,FBT002 # --exclude rdagent/scripts,git_ignore_folder # Check lint with toml-sort. toml-sort: $(PIPRUN) toml-sort --check pyproject.toml # Check lint with all linters. -lint: black isort mypy ruff toml-sort +# lint: black isort mypy ruff toml-sort +lint: mypy ruff # Run pre-commit with autofix against all files. pre-commit: diff --git a/constraints/3.8.txt b/constraints/3.8.txt deleted file mode 100644 index 7b9a455d..00000000 --- a/constraints/3.8.txt +++ /dev/null @@ -1,154 +0,0 @@ -aiohttp==3.9.1 -aiosignal==1.3.1 -alabaster==0.7.13 -annotated-types==0.6.0 -anyio==4.2.0 -appdirs==1.4.4 -async-timeout==4.0.3 -attrs==23.2.0 -autodoc-pydantic==2.0.1 -azure-ai-formrecognizer==3.3.2 -azure-common==1.1.28 -azure-core==1.29.6 -Babel==2.14.0 -beautifulsoup4==4.12.2 -black==23.12.1 -build==1.0.3 -certifi==2023.11.17 -cffi==1.16.0 -charset-normalizer==3.3.2 -click==8.1.7 -colorama==0.4.6 -coverage==7.4.0 -cryptography==41.0.7 -Cython==3.0.7 -dataclasses-json==0.6.3 -distro==1.9.0 -docutils==0.20.1 -exceptiongroup==1.2.0 -filelock==3.13.1 -fire==0.5.0 -frozenlist==1.4.1 -fsspec==2023.12.2 -furo==2023.9.10 -fuzzywuzzy==0.18.0 -git-changelog==2.4.0 -greenlet==3.0.3 -h11==0.14.0 -httpcore==1.0.2 -httpx==0.26.0 -idna==3.6 -imagesize==1.4.1 -importlib-metadata==7.0.1 -importlib-resources==6.1.1 -iniconfig==2.0.0 -isodate==0.6.1 -isort==5.13.2 -jaraco.classes==3.3.0 -jeepney==0.8.0 -Jinja2==3.1.2 -jsonpatch==1.33 -jsonpointer==2.4 -keyring==24.3.0 -langchain==0.0.353 -langchain-community==0.0.7 -langchain-core==0.1.4 -langsmith==0.0.75 -livereload==2.6.3 -loguru==0.7.2 -lxml==5.0.0 -markdown-it-py==3.0.0 -MarkupSafe==2.1.3 -marshmallow==3.20.1 -mdit-py-plugins==0.4.0 -mdurl==0.1.2 -more-itertools==10.1.0 -mpmath==1.3.0 -msrest==0.7.1 -multidict==6.0.4 -mypy==1.10.0 -mypy-extensions==1.0.0 -myst-parser==2.0.0 -networkx==3.1 -nh3==0.2.15 -numpy==1.24.4 -nvidia-cublas-cu12==12.1.3.1 -nvidia-cuda-cupti-cu12==12.1.105 -nvidia-cuda-nvrtc-cu12==12.1.105 -nvidia-cuda-runtime-cu12==12.1.105 -nvidia-cudnn-cu12==8.9.2.26 -nvidia-cufft-cu12==11.0.2.54 -nvidia-curand-cu12==10.3.2.106 -nvidia-cusolver-cu12==11.4.5.107 -nvidia-cusparse-cu12==12.1.0.106 -nvidia-nccl-cu12==2.18.1 -nvidia-nvjitlink-cu12==12.3.101 -nvidia-nvtx-cu12==12.1.105 -oauthlib==3.2.2 -openai==1.6.1 -packaging==23.2 -pandas==2.0.3 -pathspec==0.12.1 -pkginfo==1.9.6 -platformdirs==4.1.0 -pluggy==1.3.0 -pycparser==2.21 -pydantic==2.5.3 -pydantic-settings==2.1.0 -pydantic_core==2.14.6 -Pygments==2.17.2 -pypdf==3.17.4 -pyproject_hooks==1.0.0 -pytest==7.4.4 -python-dateutil==2.8.2 -python-dotenv==1.0.0 -pytz==2023.3.post1 -PyYAML==6.0.1 -readme-renderer==42.0 -requests==2.31.0 -requests-oauthlib==1.3.1 -requests-toolbelt==1.0.0 -rfc3986==2.0.0 -rich==13.7.0 -ruamel.yaml==0.18.5 -ruamel.yaml.clib==0.2.8 -ruff==0.4.5 -scipy==1.10.1 -SecretStorage==3.3.3 -semver==3.0.2 -setuptools-scm==8.0.4 -shellingham==1.5.4 -six==1.16.0 -sniffio==1.3.0 -snowballstemmer==2.2.0 -soupsieve==2.5 -Sphinx==7.1.2 -sphinx-autobuild==2021.3.14 -sphinx-basic-ng==1.0.0b2 -sphinx-click==5.1.0 -sphinx-togglebutton==0.3.2 -sphinxcontrib-applehelp==1.0.4 -sphinxcontrib-devhelp==1.0.2 -sphinxcontrib-htmlhelp==2.0.1 -sphinxcontrib-jsmath==1.0.1 -sphinxcontrib-qthelp==1.0.3 -sphinxcontrib-serializinghtml==1.1.5 -SQLAlchemy==2.0.24 -sympy==1.12 -tenacity==8.2.3 -termcolor==2.4.0 -toml-sort==0.23.1 -tomli==2.0.1 -tomlkit==0.12.3 -torch==2.1.2 -tornado==6.4 -tqdm==4.66.1 -triton==2.1.0 -twine==4.0.2 -typer==0.9.0 -typing-inspect==0.9.0 -typing_extensions==4.9.0 -tzdata==2023.4 -urllib3==2.1.0 -yarl==1.9.4 -zipp==3.17.0 \ No newline at end of file diff --git a/constraints/3.9.txt b/constraints/3.9.txt deleted file mode 100644 index 0eb3965a..00000000 --- a/constraints/3.9.txt +++ /dev/null @@ -1,153 +0,0 @@ -aiohttp==3.9.1 -aiosignal==1.3.1 -alabaster==0.7.13 -annotated-types==0.6.0 -anyio==4.2.0 -appdirs==1.4.4 -async-timeout==4.0.3 -attrs==23.2.0 -autodoc-pydantic==2.0.1 -azure-ai-formrecognizer==3.3.2 -azure-common==1.1.28 -azure-core==1.29.6 -Babel==2.14.0 -beautifulsoup4==4.12.2 -black==23.12.1 -build==1.0.3 -certifi==2023.11.17 -cffi==1.16.0 -charset-normalizer==3.3.2 -click==8.1.7 -colorama==0.4.6 -coverage==7.4.0 -cryptography==41.0.7 -Cython==3.0.7 -dataclasses-json==0.6.3 -distro==1.9.0 -docutils==0.20.1 -exceptiongroup==1.2.0 -filelock==3.13.1 -fire==0.5.0 -frozenlist==1.4.1 -fsspec==2023.12.2 -furo==2023.9.10 -fuzzywuzzy==0.18.0 -git-changelog==2.4.0 -greenlet==3.0.3 -h11==0.14.0 -httpcore==1.0.2 -httpx==0.26.0 -idna==3.6 -imagesize==1.4.1 -importlib-metadata==7.0.1 -iniconfig==2.0.0 -isodate==0.6.1 -isort==5.13.2 -jaraco.classes==3.3.0 -jeepney==0.8.0 -Jinja2==3.1.2 -jsonpatch==1.33 -jsonpointer==2.4 -keyring==24.3.0 -langchain==0.0.353 -langchain-community==0.0.7 -langchain-core==0.1.4 -langsmith==0.0.75 -livereload==2.6.3 -loguru==0.7.2 -lxml==5.0.0 -markdown-it-py==3.0.0 -MarkupSafe==2.1.3 -marshmallow==3.20.1 -mdit-py-plugins==0.4.0 -mdurl==0.1.2 -more-itertools==10.1.0 -mpmath==1.3.0 -msrest==0.7.1 -multidict==6.0.4 -mypy==1.10.0 -mypy-extensions==1.0.0 -myst-parser==2.0.0 -networkx==3.2.1 -nh3==0.2.15 -numpy==1.26.2 -nvidia-cublas-cu12==12.1.3.1 -nvidia-cuda-cupti-cu12==12.1.105 -nvidia-cuda-nvrtc-cu12==12.1.105 -nvidia-cuda-runtime-cu12==12.1.105 -nvidia-cudnn-cu12==8.9.2.26 -nvidia-cufft-cu12==11.0.2.54 -nvidia-curand-cu12==10.3.2.106 -nvidia-cusolver-cu12==11.4.5.107 -nvidia-cusparse-cu12==12.1.0.106 -nvidia-nccl-cu12==2.18.1 -nvidia-nvjitlink-cu12==12.3.101 -nvidia-nvtx-cu12==12.1.105 -oauthlib==3.2.2 -openai==1.6.1 -packaging==23.2 -pandas==2.1.4 -pathspec==0.12.1 -pkginfo==1.9.6 -platformdirs==4.1.0 -pluggy==1.3.0 -pycparser==2.21 -pydantic==2.5.3 -pydantic-settings==2.1.0 -pydantic_core==2.14.6 -Pygments==2.17.2 -pypdf==3.17.4 -pyproject_hooks==1.0.0 -pytest==7.4.4 -python-dateutil==2.8.2 -python-dotenv==1.0.0 -pytz==2023.3.post1 -PyYAML==6.0.1 -readme-renderer==42.0 -requests==2.31.0 -requests-oauthlib==1.3.1 -requests-toolbelt==1.0.0 -rfc3986==2.0.0 -rich==13.7.0 -ruamel.yaml==0.18.5 -ruamel.yaml.clib==0.2.8 -ruff==0.4.5 -scipy==1.11.4 -SecretStorage==3.3.3 -semver==3.0.2 -setuptools-scm==8.0.4 -shellingham==1.5.4 -six==1.16.0 -sniffio==1.3.0 -snowballstemmer==2.2.0 -soupsieve==2.5 -Sphinx==7.2.6 -sphinx-autobuild==2021.3.14 -sphinx-basic-ng==1.0.0b2 -sphinx-click==5.1.0 -sphinx-togglebutton==0.3.2 -sphinxcontrib-applehelp==1.0.7 -sphinxcontrib-devhelp==1.0.5 -sphinxcontrib-htmlhelp==2.0.4 -sphinxcontrib-jsmath==1.0.1 -sphinxcontrib-qthelp==1.0.6 -sphinxcontrib-serializinghtml==1.1.9 -SQLAlchemy==2.0.24 -sympy==1.12 -tenacity==8.2.3 -termcolor==2.4.0 -toml-sort==0.23.1 -tomli==2.0.1 -tomlkit==0.12.3 -torch==2.1.2 -tornado==6.4 -tqdm==4.66.1 -triton==2.1.0 -twine==4.0.2 -typer==0.9.0 -typing-inspect==0.9.0 -typing_extensions==4.9.0 -tzdata==2023.4 -urllib3==2.1.0 -yarl==1.9.4 -zipp==3.17.0 diff --git a/rdagent/app/CI/run.py b/rdagent/app/CI/run.py index aa2ae6fa..ea02f5be 100644 --- a/rdagent/app/CI/run.py +++ b/rdagent/app/CI/run.py @@ -698,7 +698,7 @@ class CIEvoAgent(EvoAgent): super().__init__(max_loop=1, evolving_strategy=evolving_strategy) self.evolving_trace = [] - def multistep_evolve(self, evo: Repo, eva: Evaluator, **kwargs: Any) -> Repo: + def multistep_evolve(self, evo: Repo, eva: Evaluator) -> Repo: evo = self.evolving_strategy.evolve( evo=evo, evolving_trace=self.evolving_trace, diff --git a/rdagent/components/coder/factor_coder/CoSTEER/__init__.py b/rdagent/components/coder/factor_coder/CoSTEER/__init__.py index 55870c7e..2155be49 100644 --- a/rdagent/components/coder/factor_coder/CoSTEER/__init__.py +++ b/rdagent/components/coder/factor_coder/CoSTEER/__init__.py @@ -90,15 +90,17 @@ class FactorCoSTEER(Developer[FactorExperiment]): factor_experiment = FactorEvolvingItem(sub_tasks=exp.sub_tasks) self.evolve_agent = FactorRAGEvoAgent( - max_loop=self.max_loop, evolving_strategy=self.evolving_strategy, rag=self.rag + max_loop=self.max_loop, + evolving_strategy=self.evolving_strategy, + rag=self.rag, + with_knowledge=self.with_knowledge, + with_feedback=self.with_feedback, + knowledge_self_gen=self.knowledge_self_gen, ) factor_experiment = self.evolve_agent.multistep_evolve( factor_experiment, self.factor_evaluator, - with_knowledge=self.with_knowledge, - with_feedback=self.with_feedback, - knowledge_self_gen=self.knowledge_self_gen, filter_final_evo=self.filter_final_evo, ) diff --git a/rdagent/components/coder/model_coder/CoSTEER/__init__.py b/rdagent/components/coder/model_coder/CoSTEER/__init__.py index 6c44e96d..91d8b7a2 100644 --- a/rdagent/components/coder/model_coder/CoSTEER/__init__.py +++ b/rdagent/components/coder/model_coder/CoSTEER/__init__.py @@ -73,15 +73,17 @@ class ModelCoSTEER(Developer[ModelExperiment]): model_experiment = ModelEvolvingItem(sub_tasks=exp.sub_tasks) self.evolve_agent = ModelRAGEvoAgent( - max_loop=self.max_loop, evolving_strategy=self.evolving_strategy, rag=self.rag + max_loop=self.max_loop, + evolving_strategy=self.evolving_strategy, + rag=self.rag, + with_knowledge=self.with_knowledge, + with_feedback=self.with_feedback, + knowledge_self_gen=self.knowledge_self_gen, ) model_experiment = self.evolve_agent.multistep_evolve( model_experiment, self.model_evaluator, - with_knowledge=self.with_knowledge, - with_feedback=self.with_feedback, - knowledge_self_gen=self.knowledge_self_gen, filter_final_evo=self.filter_final_evo, ) diff --git a/rdagent/core/evolving_agent.py b/rdagent/core/evolving_agent.py index d9a16b73..25506dcb 100644 --- a/rdagent/core/evolving_agent.py +++ b/rdagent/core/evolving_agent.py @@ -1,7 +1,7 @@ from __future__ import annotations from abc import ABC, abstractmethod -from typing import TYPE_CHECKING, Any, Type +from typing import TYPE_CHECKING, Any from tqdm import tqdm @@ -10,7 +10,7 @@ if TYPE_CHECKING: from rdagent.core.evolving_framework import EvolvableSubjects from rdagent.core.evaluation import Feedback -from rdagent.core.evolving_framework import EvoStep, EvolvingStrategy +from rdagent.core.evolving_framework import EvolvingStrategy, EvoStep from rdagent.log import rdagent_logger as logger @@ -24,40 +24,48 @@ class EvoAgent(ABC): self, evo: EvolvableSubjects, eva: Evaluator | Feedback, - **kwargs: Any, + filter_final_evo: bool = False, ) -> EvolvableSubjects: ... @abstractmethod def filter_evolvable_subjects_by_feedback( self, evo: EvolvableSubjects, - feedback: Feedback, + feedback: Feedback | None, ) -> EvolvableSubjects: ... class RAGEvoAgent(EvoAgent): - def __init__(self, max_loop: int, evolving_strategy: EvolvingStrategy, rag: Any) -> None: + def __init__( + self, + max_loop: int, + evolving_strategy: EvolvingStrategy, + rag: Any, + with_knowledge: bool = False, + with_feedback: bool = True, + knowledge_self_gen: bool = False, + ) -> None: super().__init__(max_loop, evolving_strategy) self.rag = rag self.evolving_trace: list[EvoStep] = [] + self.with_knowledge = with_knowledge + self.with_feedback = with_feedback + self.knowledge_self_gen = knowledge_self_gen + def multistep_evolve( self, evo: EvolvableSubjects, eva: Evaluator | Feedback, - *, - with_knowledge: bool = False, - with_feedback: bool = True, - knowledge_self_gen: bool = False, filter_final_evo: bool = False, ) -> EvolvableSubjects: for _ in tqdm(range(self.max_loop), "Implementing"): # 1. knowledge self-evolving - if knowledge_self_gen and self.rag is not None: + if self.knowledge_self_gen and self.rag is not None: self.rag.generate_knowledge(self.evolving_trace) # 2. RAG queried_knowledge = None - if with_knowledge and self.rag is not None: + if self.with_knowledge and self.rag is not None: # TODO: Putting the evolving trace in here doesn't actually work queried_knowledge = self.rag.query(evo, self.evolving_trace) @@ -67,20 +75,23 @@ class RAGEvoAgent(EvoAgent): evolving_trace=self.evolving_trace, queried_knowledge=queried_knowledge, ) - logger.log_object(evo.sub_workspace_list, tag="evolving code") + # TODO: Due to design issues, we have chosen to ignore this mypy error. + logger.log_object(evo.sub_workspace_list, tag="evolving code") # type: ignore[attr-defined] # 4. Pack evolve results es = EvoStep(evo, queried_knowledge) # 5. Evaluation - if with_feedback: + if self.with_feedback: es.feedback = ( - eva if isinstance(eva, Feedback) else eva.evaluate(evo, queried_knowledge=queried_knowledge) + # TODO: Due to the irregular design of rdagent.core.evaluation.Evaluator, + # it fails mypy's test here, so we'll ignore this error for now. + eva if isinstance(eva, Feedback) else eva.evaluate(evo, queried_knowledge=queried_knowledge) # type: ignore[arg-type, call-arg] ) logger.log_object(es.feedback, tag="evolving feedback") # 6. update trace self.evolving_trace.append(es) - if with_feedback and filter_final_evo: + if self.with_feedback and filter_final_evo: evo = self.filter_evolvable_subjects_by_feedback(evo, self.evolving_trace[-1].feedback) return evo diff --git a/rdagent/core/experiment.py b/rdagent/core/experiment.py index b99999b2..7b1f00ad 100644 --- a/rdagent/core/experiment.py +++ b/rdagent/core/experiment.py @@ -31,11 +31,11 @@ class Workspace(ABC, Generic[ASpecificTask]): To get a snapshot of the workspace, make sure call `copy` to get a copy of the workspace. """ - def __init__(self, target_task: ASpecificTask = None) -> None: - self.target_task: ASpecificTask = target_task + def __init__(self, target_task: ASpecificTask | None = None) -> None: + self.target_task: ASpecificTask | None = target_task @abstractmethod - def execute(self, *args: Any, **kwargs: Any) -> object: + def execute(self, *args: Any, **kwargs: Any) -> object | None: error_message = "execute method is not implemented." raise NotImplementedError(error_message) @@ -80,6 +80,7 @@ class FBWorkspace(Workspace): def __init__(self, *args: Any, **kwargs: Any) -> None: super().__init__(*args, **kwargs) + self.code_dict: dict[str, Any] = {} self.code_dict = ( {} ) # The code injected into the folder, store them in the variable to reproduce the former result @@ -145,13 +146,13 @@ class FBWorkspace(Workspace): shutil.rmtree(self.workspace_path) self.code_dict = {} - @abstractmethod - def execute(self, *args: Any, **kwargs: Any) -> object: + def execute(self) -> object | None: """ Before each execution, make sure to prepare and inject code """ self.prepare() self.inject_code(**self.code_dict) + return None ASpecificWSForExperiment = TypeVar("ASpecificWSForExperiment", bound=Workspace) @@ -165,10 +166,10 @@ class Experiment(ABC, Generic[ASpecificTask, ASpecificWSForExperiment, ASpecific def __init__(self, sub_tasks: Sequence[ASpecificTask]) -> None: self.sub_tasks = sub_tasks - self.sub_workspace_list: Sequence[ASpecificWSForSubTasks] = [None for _ in self.sub_tasks] + self.sub_workspace_list: list[ASpecificWSForSubTasks | None] = [None] * len(self.sub_tasks) self.based_experiments: Sequence[ASpecificWSForExperiment] = [] self.result: object = None # The result of the experiment, can be different types in different scenarios. - self.experiment_workspace: ASpecificWSForExperiment = None + self.experiment_workspace: ASpecificWSForExperiment | None = None ASpecificExp = TypeVar("ASpecificExp", bound=Experiment) diff --git a/rdagent/core/prompts.py b/rdagent/core/prompts.py index 32ab9046..c999d98e 100644 --- a/rdagent/core/prompts.py +++ b/rdagent/core/prompts.py @@ -1,12 +1,14 @@ -from pathlib import Path +from pathlib import Path # noqa: I001 from typing import Dict import yaml + from rdagent.core.utils import SingletonBaseClass class Prompts(SingletonBaseClass, Dict[str, str]): def __init__(self, file_path: Path) -> None: + super().__init__() with file_path.open(encoding="utf8") as file: prompt_yaml_dict = yaml.safe_load(file) diff --git a/rdagent/core/proposal.py b/rdagent/core/proposal.py index f2745b58..1da6bd55 100644 --- a/rdagent/core/proposal.py +++ b/rdagent/core/proposal.py @@ -5,13 +5,15 @@ from __future__ import annotations from abc import ABC, abstractmethod -from typing import Generic, TypeVar +from typing import TYPE_CHECKING, Generic, TypeVar from rdagent.core.evaluation import Feedback from rdagent.core.experiment import ASpecificExp, Experiment -from rdagent.core.prompts import Prompts from rdagent.core.scenario import Scenario +if TYPE_CHECKING: + from rdagent.core.prompts import Prompts + # class data_ana: XXX @@ -23,7 +25,7 @@ class Hypothesis: - Belief """ - def __init__(self, hypothesis: str, reason: str, concise_reason = str) -> None: + def __init__(self, hypothesis: str, reason: str, concise_reason: str) -> None: self.hypothesis: str = hypothesis self.reason: str = reason self.concise_reason: str = concise_reason @@ -45,7 +47,7 @@ class HypothesisFeedback(Feedback): hypothesis_evaluation: str, new_hypothesis: str, reason: str, - decision: bool, # noqa: FBT001 + decision: bool, ) -> None: self.observations = observations self.hypothesis_evaluation = hypothesis_evaluation @@ -72,7 +74,7 @@ class Trace(Generic[ASpecificScen]): self.scen: ASpecificScen = scen self.hist: list[tuple[Hypothesis, Experiment, HypothesisFeedback]] = [] - def get_sota_hypothesis_and_experiment(self) -> tuple[Hypothesis, Experiment]: + def get_sota_hypothesis_and_experiment(self) -> tuple[Hypothesis | None, Experiment | None]: """Access the last experiment result, sub-task, and the corresponding hypothesis.""" # TODO: The return value does not align with the signature. for hypothesis, experiment, feedback in self.hist[::-1]: diff --git a/rdagent/core/scenario.py b/rdagent/core/scenario.py index 18ffcd3f..4ce48621 100644 --- a/rdagent/core/scenario.py +++ b/rdagent/core/scenario.py @@ -35,7 +35,3 @@ class Scenario(ABC): @abstractmethod def get_scenario_all_desc(self) -> str: """Combine all the description together""" - - - - diff --git a/rdagent/core/utils.py b/rdagent/core/utils.py index 9c18bba6..a01f2a64 100644 --- a/rdagent/core/utils.py +++ b/rdagent/core/utils.py @@ -4,22 +4,23 @@ import importlib import json import multiprocessing as mp from collections.abc import Callable -from typing import Any +from typing import Any, ClassVar, cast -from fuzzywuzzy import fuzz +from fuzzywuzzy import fuzz # type: ignore[import-untyped] class RDAgentException(Exception): # noqa: N818 pass -class SingletonMeta(type): - def __init__(cls, *args: Any, **kwargs: Any) -> None: - cls._instance_dict: dict = {} - # This must be the class variable instead of sharing one in all classes to avoid confliction like `A()`, `B()` - super().__init__(*args, **kwargs) +class SingletonBaseClass: + """ + Because we try to support defining Singleton with `class A(SingletonBaseClass)` + instead of `A(metaclass=SingletonMeta)` this class becomes necessary. + """ + _instance_dict: ClassVar[dict] = {} - def __call__(cls, *args: Any, **kwargs: Any) -> Any: + def __new__(cls, *args: Any, **kwargs: Any) -> Any: # Since it's hard to align the difference call using args and kwargs, we strictly ask to use kwargs in Singleton if args: # TODO: this restriction can be solved. @@ -27,18 +28,11 @@ class SingletonMeta(type): raise RDAgentException(exception_message) kwargs_hash = hash(tuple(sorted(kwargs.items()))) if kwargs_hash not in cls._instance_dict: - cls._instance_dict[kwargs_hash] = super().__call__(**kwargs) + cls._instance_dict[kwargs_hash] = super().__new__(cls) # Corrected call + cls._instance_dict[kwargs_hash].__init__(**kwargs) # Ensure __init__ is called return cls._instance_dict[kwargs_hash] -class SingletonBaseClass(metaclass=SingletonMeta): - """ - Because we try to support defining Singleton with `class A(SingletonBaseClass)` - instead of `A(metaclass=SingletonMeta)` this class becomes necessary. - """ - - # TODO: Add move this class to Qlib's general utils. - def parse_json(response: str) -> Any: try: @@ -54,7 +48,7 @@ def similarity(text1: str, text2: str) -> int: text2 = text2 if isinstance(text2, str) else "" # Maybe we can use other similarity algorithm such as tfidf - return fuzz.ratio(text1, text2) + return cast(int, fuzz.ratio(text1, text2)) # mypy does not reguard it as int def import_class(class_path: str) -> Any: diff --git a/rdagent/log/base.py b/rdagent/log/base.py index fc62ed39..bdcf3c04 100644 --- a/rdagent/log/base.py +++ b/rdagent/log/base.py @@ -4,7 +4,7 @@ from abc import abstractmethod from collections.abc import Generator from datetime import datetime from pathlib import Path -from typing import Literal, Optional +from typing import Literal, Optional, Union, Literal from dataclasses import dataclass @@ -37,7 +37,13 @@ class Storage: """ @abstractmethod - def log(self, obj: object, name: str = "", **kwargs: dict) -> str | Path: + def log( + self, + obj: object, name: str = "", + save_type: Literal["json", "text", "pkl"] = "text", + timestamp: datetime | None = None, + **kwargs: dict, + ) -> str | Path: """ Parameters @@ -75,7 +81,7 @@ class View: # TODO: pleas fix me @abstractmethod - def display(s: Storage, watch: bool = False): + def display(self, s: Storage, watch: bool = False) -> None: """ Parameters diff --git a/rdagent/log/logger.py b/rdagent/log/logger.py index e69ed8b4..5ac566e4 100644 --- a/rdagent/log/logger.py +++ b/rdagent/log/logger.py @@ -6,8 +6,10 @@ from functools import partial from multiprocessing import Pipe from multiprocessing.connection import Connection from pathlib import Path +from typing import Union, Generator, Dict, Any +from logging import LogRecord -from loguru import logger +from loguru import logger, Record from psutil import Process from rdagent.core.conf import RD_AGENT_SETTINGS @@ -49,24 +51,25 @@ class RDAgentLog(SingletonBaseClass): # feedback = logger.get_reps() _tag: str = "" - def __init__(self, log_trace_path: str | None = RD_AGENT_SETTINGS.log_trace_path) -> None: + def __init__(self, log_trace_path: Union[str, None] = RD_AGENT_SETTINGS.log_trace_path) -> None: if log_trace_path is None: timestamp = datetime.now(timezone.utc).strftime("%Y-%m-%d_%H-%M-%S-%f") - log_trace_path: Path = Path.cwd() / "log" / timestamp + self.log_trace_path = Path.cwd() / "log" / timestamp + else: + self.log_trace_path = Path(log_trace_path) - self.log_trace_path = Path(log_trace_path) self.log_trace_path.mkdir(parents=True, exist_ok=True) - self.storage = FileStorage(log_trace_path) + self.storage = FileStorage(self.log_trace_path) self.main_pid = os.getpid() - def set_trace_path(self, log_trace_path): + def set_trace_path(self, log_trace_path: str | Path) -> None: self.log_trace_path = Path(log_trace_path) self.storage = FileStorage(log_trace_path) @contextmanager - def tag(self, tag: str): + def tag(self, tag: str) -> Generator[None, None, None]: if tag.strip() == "": raise ValueError("Tag cannot be empty.") if self._tag != "": @@ -94,7 +97,7 @@ class RDAgentLog(SingletonBaseClass): process = parent_process return pid_chain - def file_format(self, record, raw: bool = False): + def file_format(self, record: Record, raw: bool = False) -> str: # FIXME: the formmat is tightly coupled with the message reading in storage. record["message"] = LogColors.remove_ansi_codes(record["message"]) if raw: @@ -111,7 +114,7 @@ class RDAgentLog(SingletonBaseClass): file_handler_id = logger.add( self.log_trace_path / tag.replace(".", "/") / "common_logs.log", format=self.file_format ) - logger.patch(lambda r: r.update(caller_info)).info(f"Logging object in {logp.absolute()}") + logger.patch(lambda r: r.update(caller_info)).info(f"Logging object in {Path(logp).absolute()}") logger.remove(file_handler_id) def info(self, msg: str, *, tag: str = "", raw: bool = False) -> None: diff --git a/rdagent/log/storage.py b/rdagent/log/storage.py index ab0b4104..ff514892 100644 --- a/rdagent/log/storage.py +++ b/rdagent/log/storage.py @@ -3,10 +3,12 @@ import json import pickle from datetime import datetime, timezone from pathlib import Path -from typing import Literal, Generator +from typing import Literal, Generator, Union, Any, cast from .base import Message, Storage +LOG_LEVEL = Literal["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"] + class FileStorage(Storage): """ @@ -15,7 +17,8 @@ class FileStorage(Storage): TODO: describe the storage format """ - def __init__(self, path: str = "./log/") -> None: + + def __init__(self, path: str | Path = "./log/") -> None: self.path = Path(path) self.path.mkdir(parents=True, exist_ok=True) @@ -25,7 +28,8 @@ class FileStorage(Storage): name: str = "", save_type: Literal["json", "text", "pkl"] = "text", timestamp: datetime | None = None, - ) -> Path: + **kwargs: Any, + ) -> Union[str, Path]: # TODO: We can remove the timestamp after we implement PipeLog if timestamp is None: timestamp = datetime.now(timezone.utc) @@ -79,7 +83,7 @@ class FileStorage(Storage): timestamp_str = match.group("timestamp") timestamp = datetime.strptime(timestamp_str, "%Y-%m-%d %H:%M:%S.%f").replace(tzinfo=timezone.utc) - level = match.group("level") + level: LOG_LEVEL = cast(LOG_LEVEL, match.group("level")) caller = match.group("caller") # Extract the message content @@ -96,7 +100,7 @@ class FileStorage(Storage): content=message_content ) - if "Logging object in" in m.content: + if isinstance(m.content, str) and "Logging object in" in m.content: absolute_p = m.content.split("Logging object in ")[1] relative_p = "." + absolute_p.split(self.path.name)[1] pkl_path = self.path / relative_p diff --git a/rdagent/log/utils.py b/rdagent/log/utils.py index 0d237bc6..746db520 100644 --- a/rdagent/log/utils.py +++ b/rdagent/log/utils.py @@ -1,6 +1,8 @@ import inspect import re +from typing import Union, Dict, TypedDict, Optional + class LogColors: """ @@ -56,13 +58,19 @@ class LogColors: return ansi_escape.sub("", s) -def get_caller_info(): +class CallerInfo(TypedDict): + function: str + line: int + name: Optional[str] + + +def get_caller_info() -> CallerInfo: # Get the current stack information stack = inspect.stack() # The second element is usually the caller's information caller_info = stack[2] frame = caller_info[0] - info = { + info: CallerInfo = { "line": caller_info.lineno, "name": frame.f_globals["__name__"], # Get the module name from the frame's globals "function": frame.f_code.co_name, # Get the caller's function name diff --git a/requirements/lint.txt b/requirements/lint.txt index 8b8f70fe..03ed7089 100644 --- a/requirements/lint.txt +++ b/requirements/lint.txt @@ -4,3 +4,6 @@ isort mypy ruff toml-sort +types-PyYAML +types-psutil +types-tqdm diff --git a/test/utils/test_misc.py b/test/utils/test_misc.py new file mode 100644 index 00000000..968b4af3 --- /dev/null +++ b/test/utils/test_misc.py @@ -0,0 +1,43 @@ +import unittest + +from rdagent.core.utils import SingletonBaseClass + + +class A(SingletonBaseClass): + def __init__(self, **kwargs): + self.kwargs = kwargs + + +class MiscTest(unittest.TestCase): + + def test_singleton(self): + a1 = A() + a2 = A() + a3 = A(x=3) + a4 = A(x=2) + a5 = A(b=3) + a6 = A(x=3) + + # Check that a1 and a2 are the same instance + self.assertIs(a1, a2) + + # Check that a3 and a6 are the same instance + self.assertIs(a3, a6) + + # Check that a1 and a3 are different instances + self.assertIsNot(a1, a3) + + # Check that a3 and a4 are different instances + self.assertIsNot(a3, a4) + + # Check that a4 and a5 are different instances + self.assertIsNot(a4, a5) + + # Check that a5 and a6 are different instances + self.assertIsNot(a5, a6) + + print(id(a1), id(a2), id(a3), id(a4), id(a5), id(a6)) + + +if __name__ == "__main__": + unittest.main()