Refine all the implementation code to higher quality for release (#29)

* refine CI script

* refine all the code to higher quality

* refine the script to factor extraction and implementation

* add task loader interface

* add a task loader interface && move pdf analysis to pdf task loader

* change the name to global variables

---------

Co-authored-by: xuyang1 <xuyang1@microsoft.com>

rdagent/factor_implementation/share_modules/factor_implementation_config.py

@@ -9,13 +9,13 @@ SELECT_METHOD = Literal["random", "scheduler"]
 
 class FactorImplementSettings(BaseSettings):
     file_based_execution_data_folder: str = str(
-        (Path().cwd() / "git_ignore_folder" / "factor_implementation_source_data").absolute(),
+        (Path().cwd() / "factor_implementation_source_data").absolute(),
     )
     file_based_execution_workspace: str = str(
-        (Path().cwd() / "git_ignore_folder" / "factor_implementation_workspace").absolute(),
+        (Path().cwd() / "factor_implementation_workspace").absolute(),
     )
     implementation_execution_cache_location: str = str(
-        (Path().cwd() / "git_ignore_folder" / "factor_implementation_execution_cache.pkl").absolute(),
+        (Path().cwd() / "factor_implementation_execution_cache").absolute(),
     )
     enable_execution_cache: bool = True  # whether to enable the execution cache
 
@@ -45,9 +45,5 @@ class FactorImplementSettings(BaseSettings):
     knowledge_base_path: Union[str, None] = None
     new_knowledge_base_path: Union[str, None] = None
 
-    chat_token_limit: int = (
-        100000  # 100000 is the maximum limit of gpt4, which might increase in the future version of gpt
-    )
 
-
-FIS = FactorImplementSettings()
+FACTOR_IMPLEMENT_SETTINGS = FactorImplementSettings()

rdagent/factor_implementation/share_modules/factor_implementation_utils.py

@@ -4,7 +4,7 @@ import pandas as pd
 
 # render it with jinja
 from jinja2 import Template
-from rdagent.factor_implementation.share_modules.factor_implementation_config import FIS
+from rdagent.factor_implementation.share_modules.factor_implementation_config import FACTOR_IMPLEMENT_SETTINGS
 from rdagent.factor_implementation.evolving.factor import FactorImplementTask
 
 
@@ -17,12 +17,13 @@ TPL = """
 # Create a Jinja template from the string
 JJ_TPL = Template(TPL)
 
+
 def get_data_folder_intro():
     """Direclty get the info of the data folder.
     It is for preparing prompting message.
     """
     content_l = []
-    for p in Path(FIS.file_based_execution_data_folder).iterdir():
+    for p in Path(FACTOR_IMPLEMENT_SETTINGS.file_based_execution_data_folder).iterdir():
         if p.name.endswith(".h5"):
             df = pd.read_hdf(p)
             # get  df.head() as string with full width
@@ -49,17 +50,3 @@ def get_data_folder_intro():
                 f"file type {p.name} is not supported. Please implement its description function.",
             )
     return "\n ----------------- file spliter -------------\n".join(content_l)
-
-def load_data_from_dict(factor_dict:dict) -> list[FactorImplementTask]:
-    """Load data from a dict.
-    """
-    task_l = []
-    for factor_name, factor_data in factor_dict.items():
-        task = FactorImplementTask(
-            factor_name=factor_name,
-            factor_description=factor_data["description"],
-            factor_formulation=factor_data["formulation"],
-            variables=factor_data["variables"],
-        )
-        task_l.append(task)
-    return task_l