Fix ruff error1 (#81)

* fix_ruff_error1

* fix_ruff_error

* fix ruff error

* fix ruff error

* pass model.py

* rename exception class

* rename exception class

* rename func name generate_feedback

* remove prepare args

* optimize code

* optimize code

* fix code error
This commit is contained in:
Linlang
2024-07-18 22:36:04 +08:00
committed by GitHub
parent 74bc047615
commit ae2aa6e9b4
20 changed files with 121 additions and 94 deletions
+3 -3
View File
@@ -4,7 +4,7 @@ TODO: Factor Structure RD-Loop
from dotenv import load_dotenv
from rdagent.core.exception import FactorEmptyException
from rdagent.core.exception import FactorEmptyError
from rdagent.core.scenario import Scenario
from rdagent.log import rdagent_logger as logger
@@ -50,10 +50,10 @@ for _ in range(PROP_SETTING.evolving_n):
with logger.tag("ef"):
exp = qlib_factor_runner.develop(exp)
logger.log_object(exp, tag="factor runner result")
feedback = qlib_factor_summarizer.generateFeedback(exp, hypothesis, trace)
feedback = qlib_factor_summarizer.generate_feedback(exp, hypothesis, trace)
logger.log_object(feedback, tag="feedback")
trace.hist.append((hypothesis, exp, feedback))
except FactorEmptyException as e:
except FactorEmptyError as e:
logger.warning(e)
continue
+3 -3
View File
@@ -7,7 +7,7 @@ TODO: move the following code to a new class: Model_RD_Agent
from rdagent.app.qlib_rd_loop.conf import PROP_SETTING
from rdagent.core.developer import Developer
from rdagent.core.exception import ModelEmptyException
from rdagent.core.exception import ModelEmptyError
from rdagent.core.proposal import (
Hypothesis2Experiment,
HypothesisExperiment2Feedback,
@@ -45,9 +45,9 @@ with logger.tag("model.loop"):
with logger.tag("ef"): # evaluate and feedback
exp = qlib_model_runner.develop(exp)
logger.log_object(exp, tag="model runner result")
feedback = qlib_model_summarizer.generateFeedback(exp, hypothesis, trace)
feedback = qlib_model_summarizer.generate_feedback(exp, hypothesis, trace)
logger.log_object(feedback, tag="feedback")
trace.hist.append((hypothesis, exp, feedback))
except ModelEmptyException as e:
except ModelEmptyError as e:
logger.warning(e)
continue
+2 -2
View File
@@ -18,7 +18,7 @@ from rdagent.components.coder.factor_coder.CoSTEER.evaluators import (
from rdagent.components.coder.factor_coder.factor import FactorFBWorkspace
from rdagent.core.conf import RD_AGENT_SETTINGS
from rdagent.core.developer import Developer
from rdagent.core.exception import CoderException
from rdagent.core.exception import CoderError
from rdagent.core.experiment import Task, Workspace
from rdagent.core.utils import multiprocessing_wrapper
@@ -97,7 +97,7 @@ class BaseEval:
try:
eval_res.append((ev, ev.evaluate(implementation=case_gen, gt_implementation=case_gt)))
# if the corr ev is successfully evaluated and achieve the best performance, then break
except CoderException as e:
except CoderError as e:
return e
except Exception as e:
# exception when evaluation
@@ -11,9 +11,9 @@ from filelock import FileLock
from rdagent.components.coder.factor_coder.config import FACTOR_IMPLEMENT_SETTINGS
from rdagent.core.exception import (
CodeFormatException,
NoOutputException,
RuntimeErrorException,
CodeFormatError,
NoOutputError,
CustomRuntimeError,
)
from rdagent.core.experiment import Experiment, FBWorkspace, Task
from rdagent.log import rdagent_logger as logger
@@ -106,7 +106,7 @@ class FactorFBWorkspace(FBWorkspace):
super().execute()
if self.code_dict is None or "factor.py" not in self.code_dict:
if self.raise_exception:
raise CodeFormatException(self.FB_CODE_NOT_SET)
raise CodeFormatError(self.FB_CODE_NOT_SET)
else:
return self.FB_CODE_NOT_SET, None
with FileLock(self.workspace_path / "execution.lock"):
@@ -163,11 +163,11 @@ class FactorFBWorkspace(FBWorkspace):
execution_feedback[:1000] + "....hidden long error message...." + execution_feedback[-1000:]
)
if self.raise_exception:
raise RuntimeErrorException(execution_feedback)
raise CustomRuntimeError(execution_feedback)
except subprocess.TimeoutExpired:
execution_feedback += f"Execution timeout error and the timeout is set to {FACTOR_IMPLEMENT_SETTINGS.file_based_execution_timeout} seconds."
if self.raise_exception:
raise RuntimeErrorException(execution_feedback)
raise CustomRuntimeError(execution_feedback)
workspace_output_file_path = self.workspace_path / "result.h5"
if workspace_output_file_path.exists() and execution_success:
@@ -3,12 +3,12 @@ import pickle
import site
import uuid
from pathlib import Path
from typing import Dict, Optional
from typing import Any, Dict, Optional
import torch
from rdagent.components.coder.model_coder.conf import MODEL_IMPL_SETTINGS
from rdagent.core.exception import CodeFormatException
from rdagent.core.exception import CodeFormatError
from rdagent.core.experiment import Experiment, FBWorkspace, Task
from rdagent.oai.llm_utils import md5_hash
from rdagent.utils import get_module_by_module_path
-2
View File
@@ -11,8 +11,6 @@ from pydantic_settings import BaseSettings
# make sure that env variable is loaded while calling Config()
load_dotenv(verbose=True, override=True)
from pydantic_settings import BaseSettings
class RDAgentSettings(BaseSettings):
# TODO: (xiao) I think LLMSetting may be a better name.
+8 -3
View File
@@ -1,8 +1,12 @@
from __future__ import annotations
from abc import ABC, abstractmethod
from typing import Generic, List
from typing import TYPE_CHECKING, Generic
from rdagent.core.experiment import ASpecificExp
from rdagent.core.scenario import Scenario
if TYPE_CHECKING:
from rdagent.core.scenario import Scenario
class Developer(ABC, Generic[ASpecificExp]):
@@ -18,4 +22,5 @@ class Developer(ABC, Generic[ASpecificExp]):
due to it affects the learning process.
"""
raise NotImplementedError("generate method is not implemented.")
error_message = "generate method is not implemented."
raise NotImplementedError(error_message)
+2 -2
View File
@@ -21,6 +21,6 @@ class Evaluator(ABC):
target_task: Task,
implementation: Workspace,
gt_implementation: Workspace,
**kwargs,
):
**kwargs: object,
) -> None:
raise NotImplementedError
+21 -10
View File
@@ -1,34 +1,45 @@
from __future__ import annotations
from abc import ABC, abstractmethod
from typing import Any, List
from typing import TYPE_CHECKING, Any, Type
from tqdm import tqdm
from rdagent.core.evaluation import Evaluator
from rdagent.core.evolving_framework import EvolvableSubjects, EvoStep, Feedback
if TYPE_CHECKING:
from rdagent.core.evaluation import Evaluator
from rdagent.core.evolving_framework import EvolvableSubjects
from rdagent.core.evaluation import Feedback
from rdagent.core.evolving_framework import EvoStep, EvolvingStrategy
from rdagent.log import rdagent_logger as logger
class EvoAgent(ABC):
def __init__(self, max_loop, evolving_strategy) -> None:
def __init__(self, max_loop: int, evolving_strategy: EvolvingStrategy) -> None:
self.max_loop = max_loop
self.evolving_strategy = evolving_strategy
@abstractmethod
def multistep_evolve(
self, evo: EvolvableSubjects, eva: Evaluator | Feedback, **kwargs: Any
self,
evo: EvolvableSubjects,
eva: Evaluator | Feedback,
**kwargs: Any,
) -> EvolvableSubjects: ...
@abstractmethod
def filter_evolvable_subjects_by_feedback(
self, evo: EvolvableSubjects, feedback: Feedback
self,
evo: EvolvableSubjects,
feedback: Feedback,
) -> EvolvableSubjects: ...
class RAGEvoAgent(EvoAgent):
def __init__(self, max_loop, evolving_strategy, rag) -> None:
def __init__(self, max_loop: int, evolving_strategy: EvolvingStrategy, rag: Any) -> None:
super().__init__(max_loop, evolving_strategy)
self.rag = rag
self.evolving_trace: List[EvoStep] = []
self.evolving_trace: list[EvoStep] = []
def multistep_evolve(
self,
@@ -56,7 +67,7 @@ class RAGEvoAgent(EvoAgent):
evolving_trace=self.evolving_trace,
queried_knowledge=queried_knowledge,
)
logger.log_object(evo.sub_workspace_list, tag=f"evolving code")
logger.log_object(evo.sub_workspace_list, tag="evolving code")
# 4. Pack evolve results
es = EvoStep(evo, queried_knowledge)
@@ -66,7 +77,7 @@ class RAGEvoAgent(EvoAgent):
es.feedback = (
eva if isinstance(eva, Feedback) else eva.evaluate(evo, queried_knowledge=queried_knowledge)
)
logger.log_object(es.feedback, tag=f"evolving feedback")
logger.log_object(es.feedback, tag="evolving feedback")
# 6. update trace
self.evolving_trace.append(es)
+4 -3
View File
@@ -3,10 +3,11 @@ from __future__ import annotations
import copy
from abc import ABC, abstractmethod
from dataclasses import dataclass
from typing import Any
from typing import TYPE_CHECKING, Any
from rdagent.core.evaluation import Feedback
from rdagent.core.scenario import Scenario
if TYPE_CHECKING:
from rdagent.core.evaluation import Feedback
from rdagent.core.scenario import Scenario
class Knowledge:
+7 -7
View File
@@ -1,4 +1,4 @@
class CoderException(Exception):
class CoderError(Exception):
"""
Exceptions raised when Implementing and running code.
- start: FactorTask => FactorGenerator
@@ -8,37 +8,37 @@ class CoderException(Exception):
"""
class CodeFormatException(CoderException):
class CodeFormatError(CoderError):
"""
The generated code is not found due format error.
"""
class RuntimeErrorException(CoderException):
class CustomRuntimeError(CoderError):
"""
The generated code fail to execute the script.
"""
class NoOutputException(CoderException):
class NoOutputError(CoderError):
"""
The code fail to generate output file.
"""
class RunnerException(Exception):
class CustomRunnerError(Exception):
"""
Exceptions raised when running the code output.
"""
class FactorEmptyException(Exception):
class FactorEmptyError(Exception):
"""
Exceptions raised when no factor is generated correctly
"""
class ModelEmptyException(Exception):
class ModelEmptyError(Exception):
"""
Exceptions raised when no model is generated correctly
"""
+22 -18
View File
@@ -5,7 +5,7 @@ import uuid
from abc import ABC, abstractmethod
from copy import deepcopy
from pathlib import Path
from typing import Any, Dict, Generic, Optional, Sequence, TypeVar
from typing import Any, Generic, Sequence, TypeVar
from rdagent.core.conf import RD_AGENT_SETTINGS
@@ -16,11 +16,10 @@ This file contains the all the class about organizing the task in RD-Agent.
class Task(ABC):
@abstractmethod
def get_task_information(self):
def get_task_information(self) -> str:
"""
Get the task information string to build the unique key
"""
pass
ASpecificTask = TypeVar("ASpecificTask", bound=Task)
@@ -36,12 +35,14 @@ class Workspace(ABC, Generic[ASpecificTask]):
self.target_task: ASpecificTask = target_task
@abstractmethod
def execute(self, *args, **kwargs) -> object:
raise NotImplementedError("execute method is not implemented.")
def execute(self, *args: Any, **kwargs: Any) -> object:
error_message = "execute method is not implemented."
raise NotImplementedError(error_message)
@abstractmethod
def copy(self) -> Workspace:
raise NotImplementedError("copy method is not implemented.")
error_message = "copy method is not implemented."
raise NotImplementedError(error_message)
ASpecificWS = TypeVar("ASpecificWS", bound=Workspace)
@@ -50,7 +51,8 @@ ASpecificWS = TypeVar("ASpecificWS", bound=Workspace)
class WsLoader(ABC, Generic[ASpecificTask, ASpecificWS]):
@abstractmethod
def load(self, task: ASpecificTask) -> ASpecificWS:
raise NotImplementedError("load method is not implemented.")
error_message = "load method is not implemented."
raise NotImplementedError(error_message)
class FBWorkspace(Workspace):
@@ -63,8 +65,9 @@ class FBWorkspace(Workspace):
- Output
- After execution, it will generate the final output as file.
A typical way to run the pipeline of FBWorkspace will be
(We didn't add it as a method due to that we may pass arguments into `prepare` or `execute` based on our requirements.)
A typical way to run the pipeline of FBWorkspace will be:
(We didn't add it as a method due to that we may pass arguments into
`prepare` or `execute` based on our requirements.)
.. code-block:: python
@@ -75,7 +78,7 @@ class FBWorkspace(Workspace):
"""
def __init__(self, *args, **kwargs) -> None:
def __init__(self, *args: Any, **kwargs: Any) -> None:
super().__init__(*args, **kwargs)
self.code_dict = (
{}
@@ -89,7 +92,7 @@ class FBWorkspace(Workspace):
code_string += f"File: {file_name}\n{code}\n"
return code_string
def prepare(self, *args, **kwargs):
def prepare(self) -> None:
"""
Prepare the workspace except the injected code
- Data
@@ -99,7 +102,7 @@ class FBWorkspace(Workspace):
"""
self.workspace_path.mkdir(parents=True, exist_ok=True)
def inject_code(self, **files: str):
def inject_code(self, **files: str) -> None:
"""
Inject the code into the folder.
{
@@ -109,7 +112,7 @@ class FBWorkspace(Workspace):
self.prepare()
for k, v in files.items():
self.code_dict[k] = v
with open(self.workspace_path / k, "w") as f:
with Path.open(self.workspace_path / k, "w") as f:
f.write(v)
def get_files(self) -> list[Path]:
@@ -121,12 +124,12 @@ class FBWorkspace(Workspace):
"""
return list(self.workspace_path.iterdir())
def inject_code_from_folder(self, folder_path: Path):
def inject_code_from_folder(self, folder_path: Path) -> None:
"""
Load the workspace from the folder
"""
for file_path in folder_path.iterdir():
if file_path.suffix == ".py" or file_path.suffix == ".yaml":
if file_path.suffix in {".py", ".yaml"}:
self.inject_code(**{file_path.name: file_path.read_text()})
def copy(self) -> FBWorkspace:
@@ -143,7 +146,7 @@ class FBWorkspace(Workspace):
self.code_dict = {}
@abstractmethod
def execute(self, *args, **kwargs) -> object:
def execute(self, *args: Any, **kwargs: Any) -> object:
"""
Before each execution, make sure to prepare and inject code
"""
@@ -175,5 +178,6 @@ TaskOrExperiment = TypeVar("TaskOrExperiment", Task, Experiment)
class Loader(ABC, Generic[TaskOrExperiment]):
@abstractmethod
def load(self, *args, **kwargs) -> TaskOrExperiment:
raise NotImplementedError("load method is not implemented.")
def load(self, *args: Any, **kwargs: Any) -> TaskOrExperiment:
err_msg = "load method is not implemented."
raise NotImplementedError(err_msg)
-1
View File
@@ -2,7 +2,6 @@ from pathlib import Path
from typing import Dict
import yaml
from rdagent.core.utils import SingletonBaseClass
+26 -13
View File
@@ -2,11 +2,13 @@
"""
from __future__ import annotations
from abc import ABC, abstractmethod
from typing import Any, Dict, Generic, List, Tuple, TypeVar
from typing import Generic, TypeVar
from rdagent.core.evaluation import Feedback
from rdagent.core.experiment import ASpecificExp, ASpecificTask, Experiment
from rdagent.core.experiment import ASpecificExp, Experiment
from rdagent.core.scenario import Scenario
# class data_ana: XXX
@@ -35,14 +37,21 @@ Reason: {self.reason}"""
class HypothesisFeedback(Feedback):
def __init__(self, observations: str, hypothesis_evaluation: str, new_hypothesis: str, reason: str, decision: bool):
def __init__(
self,
observations: str,
hypothesis_evaluation: str,
new_hypothesis: str,
reason: str,
decision: bool, # noqa: FBT001
) -> None:
self.observations = observations
self.hypothesis_evaluation = hypothesis_evaluation
self.new_hypothesis = new_hypothesis
self.reason = reason
self.decision = decision
def __bool__(self):
def __bool__(self) -> bool:
return self.decision
def __str__(self) -> str:
@@ -59,9 +68,9 @@ ASpecificScen = TypeVar("ASpecificScen", bound=Scenario)
class Trace(Generic[ASpecificScen]):
def __init__(self, scen: ASpecificScen) -> None:
self.scen: ASpecificScen = scen
self.hist: list[Tuple[Hypothesis, Experiment, HypothesisFeedback]] = []
self.hist: list[tuple[Hypothesis, Experiment, HypothesisFeedback]] = []
def get_SOTA_hypothesis_and_experiment(self) -> Tuple[Hypothesis, Experiment]:
def get_sota_hypothesis_and_experiment(self) -> tuple[Hypothesis, Experiment]:
"""Access the last experiment result, sub-task, and the corresponding hypothesis."""
# TODO: The return value does not align with the signature.
for hypothesis, experiment, feedback in self.hist[::-1]:
@@ -72,7 +81,7 @@ class Trace(Generic[ASpecificScen]):
class HypothesisGen(ABC):
def __init__(self, scen: Scenario):
def __init__(self, scen: Scenario) -> None:
self.scen = scen
@abstractmethod
@@ -103,15 +112,19 @@ class Hypothesis2Experiment(ABC, Generic[ASpecificExp]):
# Boolean, Reason, Confidence, etc.
class HypothesisExperiment2Feedback:
""" "Generated feedbacks on the hypothesis from **Executed** Implementations of different tasks & their comparisons with previous performances"""
class HypothesisExperiment2Feedback(ABC):
""" "Generated feedbacks on the hypothesis from **Executed** Implementations of different tasks
& their comparisons with previous performances"""
def __init__(self, scen: Scenario):
def __init__(self, scen: Scenario) -> None:
self.scen = scen
def generateFeedback(self, exp: Experiment, hypothesis: Hypothesis, trace: Trace) -> HypothesisFeedback:
@abstractmethod
def generate_feedback(self, exp: Experiment, hypothesis: Hypothesis, trace: Trace) -> HypothesisFeedback:
"""
The `exp` should be executed and the results should be included, as well as the comparison between previous results (done by LLM).
The `exp` should be executed and the results should be included, as well as the comparison
between previous results (done by LLM).
For example: `mlflow` of Qlib will be included.
"""
raise NotImplementedError("generateFeedback method is not implemented.")
error_message = "generate_feedback method is not implemented."
raise NotImplementedError(error_message)
+5 -5
View File
@@ -4,27 +4,27 @@ from abc import ABC, abstractmethod
class Scenario(ABC):
@property
@abstractmethod
def background(self):
def background(self) -> str:
"""Background information"""
@property
@abstractmethod
def source_data(self):
def source_data(self) -> str:
"""Source data description"""
@property
@abstractmethod
def interface(self):
def interface(self) -> str:
"""Interface description about how to run the code"""
@property
@abstractmethod
def output_format(self):
def output_format(self) -> str:
"""Output format description"""
@property
@abstractmethod
def simulator(self):
def simulator(self) -> str:
"""Simulator description"""
@abstractmethod
+2 -7
View File
@@ -3,14 +3,9 @@ from __future__ import annotations
import importlib
import json
import multiprocessing as mp
import os
import random
import string
from collections.abc import Callable
from pathlib import Path
from typing import Any, ClassVar
from typing import Any
import yaml
from fuzzywuzzy import fuzz
@@ -19,7 +14,7 @@ class RDAgentException(Exception): # noqa: N818
class SingletonMeta(type):
def __init__(cls, *args, **kwargs):
def __init__(cls, *args: Any, **kwargs: Any) -> None:
cls._instance_dict: dict = {}
# This must be the class variable instead of sharing one in all classes to avoid confliction like `A()`, `B()`
super().__init__(*args, **kwargs)
@@ -6,7 +6,7 @@ import pandas as pd
from rdagent.components.runner import CachedRunner
from rdagent.components.runner.conf import RUNNER_SETTINGS
from rdagent.core.exception import FactorEmptyException
from rdagent.core.exception import FactorEmptyError
from rdagent.log import rdagent_logger as logger
from rdagent.scenarios.qlib.experiment.factor_experiment import QlibFactorExperiment
@@ -60,7 +60,7 @@ class QlibFactorRunner(CachedRunner[QlibFactorExperiment]):
new_factors = self.process_factor_data(exp)
if new_factors.empty:
raise FactorEmptyException("No valid factor data found to merge.")
raise FactorEmptyError("No valid factor data found to merge.")
# Combine the SOTA factor and new factors if SOTA factor exists
if SOTA_factor is not None and not SOTA_factor.empty:
+3 -3
View File
@@ -23,7 +23,7 @@ DIRNAME = Path(__file__).absolute().resolve().parent
class QlibFactorHypothesisExperiment2Feedback(HypothesisExperiment2Feedback):
def generateFeedback(self, exp: Experiment, hypothesis: Hypothesis, trace: Trace) -> HypothesisFeedback:
def generate_feedback(self, exp: Experiment, hypothesis: Hypothesis, trace: Trace) -> HypothesisFeedback:
"""
Generate feedback for the given experiment and hypothesis.
@@ -89,7 +89,7 @@ class QlibFactorHypothesisExperiment2Feedback(HypothesisExperiment2Feedback):
class QlibModelHypothesisExperiment2Feedback(HypothesisExperiment2Feedback):
"""Generated feedbacks on the hypothesis from **Executed** Implementations of different tasks & their comparisons with previous performances"""
def generateFeedback(self, exp: Experiment, hypothesis: Hypothesis, trace: Trace) -> HypothesisFeedback:
def generate_feedback(self, exp: Experiment, hypothesis: Hypothesis, trace: Trace) -> HypothesisFeedback:
"""
The `ti` should be executed and the results should be included, as well as the comparison between previous results (done by LLM).
For example: `mlflow` of Qlib will be included.
@@ -101,7 +101,7 @@ class QlibModelHypothesisExperiment2Feedback(HypothesisExperiment2Feedback):
# Define the user prompt for hypothesis feedback
context = trace.scen
SOTA_hypothesis, SOTA_experiment = trace.get_SOTA_hypothesis_and_experiment()
SOTA_hypothesis, SOTA_experiment = trace.get_sota_hypothesis_and_experiment()
user_prompt = (
Environment(undefined=StrictUndefined)
@@ -8,7 +8,7 @@ from rdagent.components.coder.model_coder.model import ModelExperiment, ModelFBW
from rdagent.components.runner import CachedRunner
from rdagent.components.runner.conf import RUNNER_SETTINGS
from rdagent.core.developer import Developer
from rdagent.core.exception import ModelEmptyException
from rdagent.core.exception import ModelEmptyError
from rdagent.log import rdagent_logger as logger
from rdagent.scenarios.qlib.experiment.model_experiment import QlibModelExperiment
from rdagent.utils.env import QTDockerEnv
@@ -35,7 +35,7 @@ class QlibModelRunner(CachedRunner[QlibModelExperiment]):
return exp
if exp.sub_workspace_list[0].code_dict.get("model.py") is None:
raise ModelEmptyException("model.py is empty")
raise ModelEmptyError("model.py is empty")
# to replace & inject code
exp.experiment_workspace.inject_code(**{"model.py": exp.sub_workspace_list[0].code_dict["model.py"]})
@@ -1,4 +1,5 @@
from pathlib import Path
from typing import Any
import pandas as pd