CI checks that can be automatically repaired (#119)

* fix isort & black & toml-sort & sphinx error

* fix ci error

* fix ci error

* add comments

* Update Makefile

* change sphinx build command

* add auto-lint

* add black args

* format with black

* Auto Linting document

* fix ci error

---------

Co-authored-by: you-n-g <you-n-g@users.noreply.github.com>
Co-authored-by: Young <afe.young@gmail.com>

rdagent/utils/agent/ret.py

@@ -3,8 +3,8 @@ The output of a agent is very important.
 
 We think this part can be shared.
 """
-from abc import abstractclassmethod
 import re
+from abc import abstractclassmethod
 from typing import Any
 
 from rdagent.utils.agent.tpl import T

rdagent/utils/agent/tpl.py

@@ -4,13 +4,12 @@ Here are some infrastruture to build a agent
 The motivation of tempalte and AgentOutput Design
 """
 
-from typing import Any
-from jinja2 import Environment, StrictUndefined
-
+import inspect
 from pathlib import Path
+from typing import Any
 
 import yaml
-import inspect
+from jinja2 import Environment, StrictUndefined
 
 from rdagent.core.utils import SingletonBaseClass
 
@@ -21,9 +20,10 @@ PROJ_PATH = DIRNAME.parent.parent
 # class T(SingletonBaseClass): TODO: singleton does not support args now.
 class T:
     """Use the simplest way to (C)reate a Template and (r)ender it!!"""
+
     def __init__(self, uri: str):
         """
-        here are some uri usages 
+        here are some uri usages
             case 1) "a.b.c:x.y.z"
                 It will load DIRNAME/a/b/c.yaml as `yaml` and load yaml[x][y][z]
             case 2) ".c:x.y.z"
@@ -38,16 +38,16 @@ class T:
         caller_dir = Path(caller_module.__file__).parent
 
         # Parse the URI
-        path_part, yaml_path = uri.split(':')
-        yaml_keys = yaml_path.split('.')
+        path_part, yaml_path = uri.split(":")
+        yaml_keys = yaml_path.split(".")
 
-        if path_part.startswith('.'):
+        if path_part.startswith("."):
             yaml_file_path = caller_dir / f"{path_part[1:].replace('.', '/')}.yaml"
         else:
-            yaml_file_path = (PROJ_PATH / path_part.replace('.', '/')).with_suffix('.yaml')
+            yaml_file_path = (PROJ_PATH / path_part.replace(".", "/")).with_suffix(".yaml")
 
         # Load the YAML file
-        with open(yaml_file_path, 'r') as file:
+        with open(yaml_file_path, "r") as file:
             yaml_content = yaml.safe_load(file)
 
         # Traverse the YAML content to get the desired template

rdagent/utils/env.py

@@ -144,16 +144,21 @@ class QlibDockerConf(DockerConf):
 
 class DMDockerConf(DockerConf):
     class Config:
-        env_prefix = "DM_DOCKER_"  
+        env_prefix = "DM_DOCKER_"
 
     build_from_dockerfile: bool = True
     dockerfile_folder_path: Path = Path(__file__).parent.parent / "scenarios" / "data_mining" / "docker"
     image: str = "local_dm:latest"
     mount_path: str = "/workspace/dm_workspace/"
     default_entry: str = "python train.py"
-    extra_volumes: dict = {Path("~/.rdagent/.data/physionet.org/files/mimic-eicu-fiddle-feature/1.0.0/FIDDLE_mimic3/").expanduser().resolve(): "/root/.data/"}
+    extra_volumes: dict = {
+        Path("~/.rdagent/.data/physionet.org/files/mimic-eicu-fiddle-feature/1.0.0/FIDDLE_mimic3/")
+        .expanduser()
+        .resolve(): "/root/.data/"
+    }
     shm_size: str | None = "16g"
 
+
 # physionet.org/files/mimic-eicu-fiddle-feature/1.0.0/FIDDLE_mimic3
 class DockerEnv(Env[DockerConf]):
     # TODO: Save the output into a specific file
@@ -181,9 +186,9 @@ class DockerEnv(Env[DockerConf]):
         if not self.conf.enable_gpu:
             return {}
         gpu_kwargs = {
-            "device_requests": [
-                docker.types.DeviceRequest(count=-1, capabilities=[['gpu']])
-            ] if self.conf.enable_gpu else None,
+            "device_requests": [docker.types.DeviceRequest(count=-1, capabilities=[["gpu"]])]
+            if self.conf.enable_gpu
+            else None,
         }
         try:
             client.containers.run(self.conf.image, "nvidia-smi", **gpu_kwargs)
@@ -220,7 +225,7 @@ class DockerEnv(Env[DockerConf]):
                 # auto_remove=True, # remove too fast might cause the logs not to be get
                 network=self.conf.network,
                 shm_size=self.conf.shm_size,
-                **self._gpu_kwargs(client)
+                **self._gpu_kwargs(client),
             )
             logs = container.logs(stream=True)
             for log in logs:
@@ -273,7 +278,9 @@ class DMDockerEnv(DockerEnv):
         data_path = next(iter(self.conf.extra_volumes.keys()))
         if not (Path(data_path)).exists():
             logger.info("We are downloading!")
-            cmd = 'wget -r -N -c -np --user={} --password={} -P ~/.rdagent/.data/ https://physionet.org/files/mimic-eicu-fiddle-feature/1.0.0/'.format(username, password)
+            cmd = "wget -r -N -c -np --user={} --password={} -P ~/.rdagent/.data/ https://physionet.org/files/mimic-eicu-fiddle-feature/1.0.0/".format(
+                username, password
+            )
             os.system(cmd)
         else:
             logger.info("Data already exists. Download skipped.")

rdagent/utils/workflow.py

@@ -7,20 +7,19 @@ Postscripts:
   However, Python generator is not picklable (dill does not support pickle as well)
 
 """
-from pathlib import Path
+import datetime
 import pickle
-from tqdm.auto import tqdm
-
-
 from collections import defaultdict
 from dataclasses import dataclass, field
-import datetime
+from pathlib import Path
 from typing import Callable
+
+from tqdm.auto import tqdm
+
 from rdagent.log import rdagent_logger as logger
 
 
 class LoopMeta(type):
-
     @staticmethod
     def _get_steps(bases):
         """
@@ -28,7 +27,7 @@ class LoopMeta(type):
         """
         steps = []
         for base in bases:
-            steps.extend(LoopMeta._get_steps(base.__bases__) + getattr(base,"steps", []))
+            steps.extend(LoopMeta._get_steps(base.__bases__) + getattr(base, "steps", []))
         return steps
 
     def __new__(cls, clsname, bases, attrs):
@@ -52,12 +51,14 @@ class LoopBase:
     steps: list[Callable]  # a list of steps to work on
     loop_trace: dict[int, list[LoopTrace]]
 
-    skip_loop_error: tuple[Exception]  = field(default_factory=tuple)  # you can define a list of error that will skip current loop
+    skip_loop_error: tuple[Exception] = field(
+        default_factory=tuple
+    )  # you can define a list of error that will skip current loop
 
     def __init__(self):
-        self.loop_idx = 0 # current loop index
-        self.step_idx = 0 # the index of next step to be run
-        self.loop_prev_out = {} # the step results of current loop
+        self.loop_idx = 0  # current loop index
+        self.step_idx = 0  # the index of next step to be run
+        self.loop_prev_out = {}  # the step results of current loop
         self.loop_trace = defaultdict(list[LoopTrace])  # the key is the number of loop
         self.session_folder = logger.log_trace_path / "__session__"
 
@@ -121,7 +122,7 @@ class LoopBase:
         with path.open("rb") as f:
             session = pickle.load(f)
         logger.set_trace_path(session.session_folder.parent)
-        
+
         max_loop = max(session.loop_trace.keys())
         logger.storage.truncate(time=session.loop_trace[max_loop][-1].end)
         return session