gavindiaz/NexQuant
1
0
Fork 0
mirror of https://github.com/NicolasBohn/NexQuant.git synced 2026-07-27 15:37:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

Several update on the repo (see desc) (#76)

Browse Source 
* ignore result csv file

* fix app scripts

* rename taskgenerator to developer and generate to develop

* fix a config bug in coder

* fix a small bug in factor coder evaluators

* remove a single logger in factor coder evaluators

* fix a small bug in model coder main.py

* rename Implementation to Workspace

* move the prepare the inject_code into FBWorkspace to align all the behavior

* fix a small bug in model feedback

* remove debug lines for multi processing and simplify evaluators multi proc

* add a copy function to workspace to freeze the workspace && add config prefix to speed up debugging

* make hypothesisgen a abc class

* use Qlib***Experiment

* fix a small bug

* rename Imp to Ws

* rename sub_implementations to sub_workspace_list

* fix a bug in feedback not presented as content in prompts

* move proposal pys to proposal folder

* reformat the folder

* align factor and model qlib workspace and use template to handle the workspace

* add a filter to evoagent to filter out false evo

* align multi_proc_n into RDAGENT seeting

* handle when runner gets empty experiment

* fix logger merge remaining problems

* fix black and isort automatically
This commit is contained in:
Xu Yang
2024-07-17 15:00:13 +08:00
committed by GitHub
parent eee2b3c56a
commit e0a24fb46f
76 changed files with 804 additions and 702 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+6 -2
View File
@@ -155,6 +155,10 @@ git_ignore_folder/
*.db
# Docker
env_factor/mlruns/
factor_template/mlruns/
env_tpl
mlruns/
mlruns/
# possible output from coder or runner
*.pth
*qlib_res.csv
docs/conf.py
+11 -11
View File
@@ -8,9 +8,9 @@
import importlib.metadata
project = 'RDAgent'
copyright = '2024, Microsoft'
author = 'Microsoft'
project = "RDAgent"
copyright = "2024, Microsoft"
author = "Microsoft"
# -- General configuration ---------------------------------------------------
# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration
@@ -20,13 +20,13 @@ extensions = []
autodoc_member_order = "bysource"
# The suffix of source filenames.
source_suffix = '.rst'
source_suffix = ".rst"
# The encoding of source files.
source_encoding = 'utf-8'
source_encoding = "utf-8"
# The main toctree document.
master_doc = 'index'
master_doc = "index"
# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
@@ -42,21 +42,21 @@ language = "en"
# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = ['build']
exclude_patterns = ["build"]
# -- Options for HTML output -------------------------------------------------
# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output
# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'
pygments_style = "sphinx"
try:
import furo
html_theme = 'furo'
html_theme = "furo"
html_theme_options = {
"navigation_with_keys": True,
}
except ImportError:
html_theme = 'default'
html_theme = "default"
html_static_path = ['_static']
html_static_path = ["_static"]
rdagent/app/factor_extraction_and_code/factor_extract_and_implement.py
+28
View File
@@ -0,0 +1,28 @@
# %%
from dotenv import load_dotenv
from rdagent.log import rdagent_logger as logger
from rdagent.scenarios.qlib.developer.factor_coder import QlibFactorCoSTEER
from rdagent.scenarios.qlib.experiment.factor_experiment import QlibFactorScenario
from rdagent.scenarios.qlib.factor_experiment_loader.pdf_loader import (
FactorExperimentLoaderFromPDFfiles,
)
assert load_dotenv()
def extract_factors_and_implement(report_file_path: str) -> None:
scenario = QlibFactorScenario()
with logger.tag("extract_factors_and_implement"):
with logger.tag("load_factor_tasks"):
exp = FactorExperimentLoaderFromPDFfiles().load(report_file_path)
with logger.tag("implement_factors"):
exp = QlibFactorCoSTEER(scenario).develop(exp)
# Qlib to run the implementation in rd loop
return exp
if __name__ == "__main__":
extract_factors_and_implement("/home/xuyang1/workspace/report.pdf")
rdagent/app/factor_extraction_and_implementation/factor_extract_and_implement.py
-24
View File
@@ -1,24 +0,0 @@
# %%
from dotenv import load_dotenv
from rdagent.scenarios.qlib.factor_experiment_loader.pdf_loader import FactorExperimentLoaderFromPDFfiles
from rdagent.scenarios.qlib.factor_task_implementation import QlibFactorCoSTEER
from rdagent.log import rdagent_logger as logger
assert load_dotenv()
def extract_factors_and_implement(report_file_path: str) -> None:
with logger.tag('extract_factors_and_implement'):
with logger.tag('load_factor_tasks'):
factor_tasks = FactorExperimentLoaderFromPDFfiles().load(report_file_path)
with logger.tag('implement_factors'):
implementation_result = QlibFactorCoSTEER().generate(factor_tasks)
# Qlib to run the implementation
return implementation_result
if __name__ == "__main__":
extract_factors_and_implement("/home/xuyang1/workspace/report.pdf")
# %%
rdagent/app/model_extraction_and_code/model_extraction_and_implementation.py
+21
View File
@@ -0,0 +1,21 @@
# %%
from dotenv import load_dotenv
from rdagent.components.coder.model_coder.task_loader import (
ModelExperimentLoaderFromPDFfiles,
)
from rdagent.scenarios.qlib.developer.model_coder import QlibModelCoSTEER
from rdagent.scenarios.qlib.experiment.model_experiment import QlibModelScenario
def extract_models_and_implement(report_file_path: str = "../test_doc") -> None:
scenario = QlibModelScenario()
exp = ModelExperimentLoaderFromPDFfiles().load(report_file_path)
exp = QlibModelCoSTEER(scenario).develop(exp)
return exp
import fire
if __name__ == "__main__":
fire.Fire(extract_models_and_implement)
rdagent/app/model_extraction_and_implementation/model_extraction_and_implementation.py
-19
View File
@@ -1,19 +0,0 @@
# %%
from dotenv import load_dotenv
from rdagent.components.coder.model_coder.one_shot import ModelCodeWriter
from rdagent.components.coder.model_coder.task_loader import (
ModelExperimentLoaderFromPDFfiles,
)
def extract_models_and_implement(report_file_path: str = "../test_doc") -> None:
factor_tasks = ModelExperimentLoaderFromPDFfiles().load(report_file_path)
implementation_result = ModelCodeWriter().generate(factor_tasks)
return implementation_result
import fire
if __name__ == "__main__":
fire.Fire(extract_models_and_implement)
rdagent/app/model_implementation/eval.py
+4 -4
View File
@@ -1,7 +1,7 @@
from pathlib import Path
from rdagent.components.coder.model_coder.CoSTEER import ModelCoSTEER
from rdagent.components.loader.task_loader import ModelImpLoader, ModelTaskLoaderJson
from rdagent.components.loader.task_loader import ModelTaskLoaderJson, ModelWsLoader
from rdagent.scenarios.qlib.experiment.model_experiment import (
QlibModelExperiment,
QlibModelScenario,
@@ -23,17 +23,17 @@ model_experiment = QlibModelExperiment(sub_tasks=task_l)
# mtg = ModelCodeWriter(scen=QlibModelScenario())
mtg = ModelCoSTEER(scen=QlibModelScenario())
model_experiment = mtg.generate(model_experiment)
model_experiment = mtg.develop(model_experiment)
# TODO: Align it with the benchmark framework after @wenjun's refine the evaluation part.
# Currently, we just handcraft a workflow for fast evaluation.
mil = ModelImpLoader(bench_folder / "gt_code")
mil = ModelWsLoader(bench_folder / "gt_code")
mie = ModelImpValEval()
# Evaluation:
eval_l = []
for impl in model_experiment.sub_implementations:
for impl in model_experiment.sub_workspace_list:
print(impl.target_task)
gt_impl = mil.load(impl.target_task)
eval_l.append(mie.evaluate(gt_impl, impl))
rdagent/app/qlib_rd_loop/conf.py
+21 -17
View File
@@ -1,30 +1,34 @@
from pydantic_settings import BaseSettings
from pathlib import Path
from pydantic_settings import BaseSettings
class PropSetting(BaseSettings):
""""""
class Config:
env_prefix = "QLIB_" # Use MODEL_CODER_ as prefix for environment variables
protected_namespaces = () # Add 'model_' to the protected namespaces
qlib_factor_scen: str = "rdagent.scenarios.qlib.experiment.factor_experiment.QlibFactorScenario"
qlib_factor_hypothesis_gen: str = "rdagent.scenarios.qlib.factor_proposal.QlibFactorHypothesisGen"
qlib_factor_hypothesis2experiment: str = "rdagent.scenarios.qlib.factor_proposal.QlibFactorHypothesis2Experiment"
qlib_factor_coder: str = "rdagent.scenarios.qlib.factor_task_implementation.QlibFactorCoSTEER"
qlib_factor_runner: str = "rdagent.scenarios.qlib.task_generator.data.QlibFactorRunner"
qlib_factor_summarizer: str = (
"rdagent.scenarios.qlib.task_generator.feedback.QlibFactorHypothesisExperiment2Feedback"
factor_scen: str = "rdagent.scenarios.qlib.experiment.factor_experiment.QlibFactorScenario"
factor_hypothesis_gen: str = "rdagent.scenarios.qlib.proposal.factor_proposal.QlibFactorHypothesisGen"
factor_hypothesis2experiment: str = (
"rdagent.scenarios.qlib.proposal.factor_proposal.QlibFactorHypothesis2Experiment"
)
factor_coder: str = "rdagent.scenarios.qlib.developer.factor_coder.QlibFactorCoSTEER"
factor_runner: str = "rdagent.scenarios.qlib.developer.factor_runner.QlibFactorRunner"
factor_summarizer: str = "rdagent.scenarios.qlib.developer.feedback.QlibFactorHypothesisExperiment2Feedback"
# TODO: model part is not finished yet
qlib_model_scen: str = "rdagent.scenarios.qlib.experiment.model_experiment.QlibModelScenario"
qlib_model_hypothesis_gen: str = "rdagent.scenarios.qlib.model_proposal.QlibModelHypothesisGen"
qlib_model_hypothesis2experiment: str = "rdagent.scenarios.qlib.model_proposal.QlibModelHypothesis2Experiment"
qlib_model_coder: str = "rdagent.scenarios.qlib.model_task_implementation.QlibModelCoSTEER"
qlib_model_runner: str = "rdagent.scenarios.qlib.task_generator.model.QlibModelRunner"
qlib_model_summarizer: str = "rdagent.scenarios.qlib.task_generator.feedback.QlibModelHypothesisExperiment2Feedback"
model_scen: str = "rdagent.scenarios.qlib.experiment.model_experiment.QlibModelScenario"
model_hypothesis_gen: str = "rdagent.scenarios.qlib.proposal.model_proposal.QlibModelHypothesisGen"
model_hypothesis2experiment: str = "rdagent.scenarios.qlib.proposal.model_proposal.QlibModelHypothesis2Experiment"
model_coder: str = "rdagent.scenarios.qlib.developer.model_coder.QlibModelCoSTEER"
model_runner: str = "rdagent.scenarios.qlib.developer.model_runner.QlibModelRunner"
model_summarizer: str = "rdagent.scenarios.qlib.developer.feedback.QlibModelHypothesisExperiment2Feedback"
evolving_n: int = 10
py_bin: str = "/usr/bin/python"
local_qlib_folder: Path = Path("/home/rdagent/qlib")
PROP_SETTING = PropSetting()
rdagent/app/qlib_rd_loop/factor.py
+19 -13
View File
@@ -4,39 +4,45 @@ TODO: Factor Structure RD-Loop
from dotenv import load_dotenv
from rdagent.core.exception import FactorEmptyException
from rdagent.core.scenario import Scenario
from rdagent.log import rdagent_logger as logger
load_dotenv(override=True)
from rdagent.app.qlib_rd_loop.conf import PROP_SETTING
from rdagent.core.developer import Developer
from rdagent.core.proposal import (
Hypothesis2Experiment,
HypothesisExperiment2Feedback,
HypothesisGen,
Trace,
)
from rdagent.core.task_generator import TaskGenerator
from rdagent.core.utils import import_class
scen: Scenario = import_class(PROP_SETTING.qlib_factor_scen)()
scen: Scenario = import_class(PROP_SETTING.factor_scen)()
hypothesis_gen: HypothesisGen = import_class(PROP_SETTING.qlib_factor_hypothesis_gen)(scen)
hypothesis_gen: HypothesisGen = import_class(PROP_SETTING.factor_hypothesis_gen)(scen)
hypothesis2experiment: Hypothesis2Experiment = import_class(PROP_SETTING.qlib_factor_hypothesis2experiment)()
hypothesis2experiment: Hypothesis2Experiment = import_class(PROP_SETTING.factor_hypothesis2experiment)()
qlib_factor_coder: TaskGenerator = import_class(PROP_SETTING.qlib_factor_coder)(scen)
qlib_factor_coder: Developer = import_class(PROP_SETTING.factor_coder)(scen)
qlib_factor_runner: TaskGenerator = import_class(PROP_SETTING.qlib_factor_runner)(scen)
qlib_factor_runner: Developer = import_class(PROP_SETTING.factor_runner)(scen)
qlib_factor_summarizer: HypothesisExperiment2Feedback = import_class(PROP_SETTING.qlib_factor_summarizer)(scen)
qlib_factor_summarizer: HypothesisExperiment2Feedback = import_class(PROP_SETTING.factor_summarizer)(scen)
trace = Trace(scen=scen)
for _ in range(PROP_SETTING.evolving_n):
hypothesis = hypothesis_gen.gen(trace)
exp = hypothesis2experiment.convert(hypothesis, trace)
exp = qlib_factor_coder.generate(exp)
exp = qlib_factor_runner.generate(exp)
feedback = qlib_factor_summarizer.generateFeedback(exp, hypothesis, trace)
try:
hypothesis = hypothesis_gen.gen(trace)
exp = hypothesis2experiment.convert(hypothesis, trace)
exp = qlib_factor_coder.develop(exp)
exp = qlib_factor_runner.develop(exp)
feedback = qlib_factor_summarizer.generateFeedback(exp, hypothesis, trace)
trace.hist.append((hypothesis, exp, feedback))
trace.hist.append((hypothesis, exp, feedback))
except FactorEmptyException as e:
logger.warning(e)
continue
rdagent/app/qlib_rd_loop/model.py
+19 -13
View File
@@ -6,6 +6,8 @@ TODO: move the following code to a new class: Model_RD_Agent
# import_from
from rdagent.app.qlib_rd_loop.conf import PROP_SETTING
from rdagent.core.developer import Developer
from rdagent.core.exception import ModelEmptyException
from rdagent.core.proposal import (
Hypothesis2Experiment,
HypothesisExperiment2Feedback,
@@ -13,26 +15,30 @@ from rdagent.core.proposal import (
Trace,
)
from rdagent.core.scenario import Scenario
from rdagent.core.task_generator import TaskGenerator
from rdagent.core.utils import import_class
from rdagent.log import rdagent_logger as logger
scen: Scenario = import_class(PROP_SETTING.qlib_model_scen)()
scen: Scenario = import_class(PROP_SETTING.model_scen)()
hypothesis_gen: HypothesisGen = import_class(PROP_SETTING.qlib_model_hypothesis_gen)(scen)
hypothesis_gen: HypothesisGen = import_class(PROP_SETTING.model_hypothesis_gen)(scen)
hypothesis2experiment: Hypothesis2Experiment = import_class(PROP_SETTING.qlib_model_hypothesis2experiment)()
hypothesis2experiment: Hypothesis2Experiment = import_class(PROP_SETTING.model_hypothesis2experiment)()
qlib_model_coder: TaskGenerator = import_class(PROP_SETTING.qlib_model_coder)(scen)
qlib_model_runner: TaskGenerator = import_class(PROP_SETTING.qlib_model_runner)(scen)
qlib_model_coder: Developer = import_class(PROP_SETTING.model_coder)(scen)
qlib_model_runner: Developer = import_class(PROP_SETTING.model_runner)(scen)
qlib_model_summarizer: HypothesisExperiment2Feedback = import_class(PROP_SETTING.qlib_model_summarizer)(scen)
qlib_model_summarizer: HypothesisExperiment2Feedback = import_class(PROP_SETTING.model_summarizer)(scen)
trace = Trace(scen=scen)
for _ in range(PROP_SETTING.evolving_n):
hypothesis = hypothesis_gen.gen(trace)
exp = hypothesis2experiment.convert(hypothesis, trace)
exp = qlib_model_coder.generate(exp)
exp = qlib_model_runner.generate(exp)
feedback = qlib_model_summarizer.generateFeedback(exp, hypothesis, trace)
try:
hypothesis = hypothesis_gen.gen(trace)
exp = hypothesis2experiment.convert(hypothesis, trace)
exp = qlib_model_coder.develop(exp)
exp = qlib_model_runner.develop(exp)
feedback = qlib_model_summarizer.generateFeedback(exp, hypothesis, trace)
trace.hist.append((hypothesis, exp, feedback))
trace.hist.append((hypothesis, exp, feedback))
except ModelEmptyException as e:
logger.warning(e)
continue
rdagent/app/quant_factor_benchmark/eval.py
+4 -3
View File
@@ -1,10 +1,11 @@
from rdagent.components.benchmark.conf import BenchmarkSettings
from rdagent.components.benchmark.eval_method import FactorImplementEval
from rdagent.core.utils import import_class
from rdagent.scenarios.qlib.factor_task_loader.json_loader import (
FactorTestCaseLoaderFromJsonFile,
)
from rdagent.components.benchmark.conf import BenchmarkSettings
from rdagent.components.benchmark.eval_method import FactorImplementEval
from rdagent.core.utils import import_class
# 1.read the settings
bs = BenchmarkSettings()
rdagent/components/benchmark/eval_method.py
+17 -16
View File
@@ -15,10 +15,11 @@ from rdagent.components.coder.factor_coder.CoSTEER.evaluators import (
FactorRowCountEvaluator,
FactorSingleColumnEvaluator,
)
from rdagent.components.coder.factor_coder.factor import FileBasedFactorImplementation
from rdagent.core.exception import ImplementRunException
from rdagent.core.experiment import Implementation, Task
from rdagent.core.task_generator import TaskGenerator
from rdagent.components.coder.factor_coder.factor import FactorFBWorkspace
from rdagent.core.conf import RD_AGENT_SETTINGS
from rdagent.core.developer import Developer
from rdagent.core.exception import CoderException
from rdagent.core.experiment import Task, Workspace
from rdagent.core.utils import multiprocessing_wrapper
@@ -26,7 +27,7 @@ class TestCase:
def __init__(
self,
target_task: list[Task] = [],
ground_truth: list[Implementation] = [],
ground_truth: list[Workspace] = [],
):
self.ground_truth = ground_truth
self.target_task = target_task
@@ -41,7 +42,7 @@ class BaseEval:
self,
evaluator_l: List[FactorEvaluator],
test_cases: List[TestCase],
generate_method: TaskGenerator,
generate_method: Developer,
catch_eval_except: bool = True,
):
"""Parameters
@@ -62,12 +63,12 @@ class BaseEval:
self,
path: Union[Path, str],
**kwargs,
) -> List[Implementation]:
) -> List[Workspace]:
path = Path(path)
fi_l = []
for tc in self.test_cases:
try:
fi = FileBasedFactorImplementation.from_folder(tc.task, path, **kwargs)
fi = FactorFBWorkspace.from_folder(tc.task, path, **kwargs)
fi_l.append(fi)
except FileNotFoundError:
print("Fail to load test case for factor: ", tc.task.factor_name)
@@ -75,8 +76,8 @@ class BaseEval:
def eval_case(
self,
case_gt: Implementation,
case_gen: Implementation,
case_gt: Workspace,
case_gen: Workspace,
) -> List[Union[Tuple[FactorEvaluator, object], Exception]]:
"""Parameters
----------
@@ -96,7 +97,7 @@ class BaseEval:
try:
eval_res.append((ev, ev.evaluate(implementation=case_gen, gt_implementation=case_gt)))
# if the corr ev is successfully evaluated and achieve the best performance, then break
except ImplementRunException as e:
except CoderException as e:
return e
except Exception as e:
# exception when evaluation
@@ -111,7 +112,7 @@ class FactorImplementEval(BaseEval):
def __init__(
self,
test_cases: TestCase,
method: TaskGenerator,
method: Developer,
*args,
test_round: int = 10,
**kwargs,
@@ -137,17 +138,17 @@ class FactorImplementEval(BaseEval):
print(f"Eval {_}-th times...")
print("========================================================\n")
try:
gen_factor_l = self.generate_method.generate(self.test_cases.target_task)
gen_factor_l = self.generate_method.develop(self.test_cases.target_task)
except KeyboardInterrupt:
# TODO: Why still need to save result after KeyboardInterrupt?
print("Manually interrupted the evaluation. Saving existing results")
break
if len(gen_factor_l.sub_implementations) != len(self.test_cases.ground_truth):
if len(gen_factor_l.sub_workspace_list) != len(self.test_cases.ground_truth):
raise ValueError(
"The number of cases to eval should be equal to the number of test cases.",
)
gen_factor_l_all_rounds.extend(gen_factor_l.sub_implementations)
gen_factor_l_all_rounds.extend(gen_factor_l.sub_workspace_list)
test_cases_all_rounds.extend(self.test_cases.ground_truth)
eval_res_list = multiprocessing_wrapper(
@@ -155,7 +156,7 @@ class FactorImplementEval(BaseEval):
(self.eval_case, (gt_case, gen_factor))
for gt_case, gen_factor in zip(test_cases_all_rounds, gen_factor_l_all_rounds)
],
n=FACTOR_IMPLEMENT_SETTINGS.evo_multi_proc_n,
n=RD_AGENT_SETTINGS.multi_proc_n,
)
for gt_case, eval_res, gen_factor in tqdm(zip(test_cases_all_rounds, eval_res_list, gen_factor_l_all_rounds)):
rdagent/components/coder/factor_coder/CoSTEER/__init__.py
+14 -7
View File
@@ -9,6 +9,9 @@ from rdagent.components.coder.factor_coder.CoSTEER.evaluators import (
from rdagent.components.coder.factor_coder.CoSTEER.evolvable_subjects import (
FactorEvolvingItem,
)
from rdagent.components.coder.factor_coder.CoSTEER.evolving_agent import (
FactorRAGEvoAgent,
)
from rdagent.components.coder.factor_coder.CoSTEER.evolving_strategy import (
FactorEvolvingStrategyWithGraph,
)
@@ -18,18 +21,19 @@ from rdagent.components.coder.factor_coder.CoSTEER.knowledge_management import (
FactorKnowledgeBaseV1,
)
from rdagent.components.coder.factor_coder.factor import FactorExperiment
from rdagent.core.developer import Developer
from rdagent.core.evolving_agent import RAGEvoAgent
from rdagent.core.scenario import Scenario
from rdagent.core.task_generator import TaskGenerator
class FactorCoSTEER(TaskGenerator[FactorExperiment]):
class FactorCoSTEER(Developer[FactorExperiment]):
def __init__(
self,
*args,
with_knowledge: bool = True,
with_feedback: bool = True,
knowledge_self_gen: bool = True,
filter_final_evo: bool = True,
**kwargs,
) -> None:
super().__init__(*args, **kwargs)
@@ -47,6 +51,7 @@ class FactorCoSTEER(TaskGenerator[FactorExperiment]):
self.with_knowledge = with_knowledge
self.with_feedback = with_feedback
self.knowledge_self_gen = knowledge_self_gen
self.filter_final_evo = filter_final_evo
self.evolving_strategy = FactorEvolvingStrategyWithGraph(scen=self.scen)
# declare the factor evaluator
self.factor_evaluator = FactorMultiEvaluator(FactorEvaluatorForCoder(scen=self.scen), scen=self.scen)
@@ -72,7 +77,7 @@ class FactorCoSTEER(TaskGenerator[FactorExperiment]):
)
return factor_knowledge_base
def generate(self, exp: FactorExperiment) -> FactorExperiment:
def develop(self, exp: FactorExperiment) -> FactorExperiment:
# init knowledge base
factor_knowledge_base = self.load_or_init_knowledge_base(
former_knowledge_base_path=self.knowledge_base_path,
@@ -84,7 +89,9 @@ class FactorCoSTEER(TaskGenerator[FactorExperiment]):
# init intermediate items
factor_experiment = FactorEvolvingItem(sub_tasks=exp.sub_tasks)
self.evolve_agent = RAGEvoAgent(max_loop=self.max_loop, evolving_strategy=self.evolving_strategy, rag=self.rag)
self.evolve_agent = FactorRAGEvoAgent(
max_loop=self.max_loop, evolving_strategy=self.evolving_strategy, rag=self.rag
)
factor_experiment = self.evolve_agent.multistep_evolve(
factor_experiment,
@@ -92,11 +99,11 @@ class FactorCoSTEER(TaskGenerator[FactorExperiment]):
with_knowledge=self.with_knowledge,
with_feedback=self.with_feedback,
knowledge_self_gen=self.knowledge_self_gen,
filter_final_evo=self.filter_final_evo,
)
# save new knowledge base
if self.new_knowledge_base_path is not None:
pickle.dump(factor_knowledge_base, open(self.new_knowledge_base_path, "wb"))
self.knowledge_base = factor_knowledge_base
factor_experiment.based_experiments = exp.based_experiments
return factor_experiment
exp.sub_workspace_list = factor_experiment.sub_workspace_list
return exp
rdagent/components/coder/factor_coder/CoSTEER/evaluators.py
+36 -58
View File
@@ -8,7 +8,6 @@ from typing import List, Tuple
import pandas as pd
from jinja2 import Environment, StrictUndefined
from rdagent.components.coder.factor_coder.config import FACTOR_IMPLEMENT_SETTINGS
from rdagent.components.coder.factor_coder.CoSTEER.evolvable_subjects import (
FactorEvolvingItem,
)
@@ -16,10 +15,10 @@ from rdagent.components.coder.factor_coder.factor import FactorTask
from rdagent.core.conf import RD_AGENT_SETTINGS
from rdagent.core.evaluation import Evaluator
from rdagent.core.evolving_framework import Feedback, QueriedKnowledge
from rdagent.core.experiment import Implementation, Task
from rdagent.log import rdagent_logger as logger
from rdagent.core.experiment import Task, Workspace
from rdagent.core.prompts import Prompts
from rdagent.core.utils import multiprocessing_wrapper
from rdagent.log import rdagent_logger as logger
from rdagent.oai.llm_utils import APIBackend
evaluate_prompts = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")
@@ -33,8 +32,8 @@ class FactorEvaluator(Evaluator):
def evaluate(
self,
target_task: Task,
implementation: Implementation,
gt_implementation: Implementation,
implementation: Workspace,
gt_implementation: Workspace,
**kwargs,
) -> Tuple[str, object]:
"""You can get the dataframe by
@@ -53,7 +52,7 @@ class FactorEvaluator(Evaluator):
"""
raise NotImplementedError("Please implement the `evaluator` method")
def _get_df(self, gt_implementation: Implementation, implementation: Implementation):
def _get_df(self, gt_implementation: Workspace, implementation: Workspace):
if gt_implementation is not None:
_, gt_df = gt_implementation.execute()
if isinstance(gt_df, pd.Series):
@@ -78,10 +77,10 @@ class FactorCodeEvaluator(FactorEvaluator):
def evaluate(
self,
target_task: FactorTask,
implementation: Implementation,
implementation: Workspace,
execution_feedback: str,
factor_value_feedback: str = "",
gt_implementation: Implementation = None,
gt_implementation: Workspace = None,
**kwargs,
):
factor_information = target_task.get_task_information()
@@ -130,8 +129,8 @@ class FactorCodeEvaluator(FactorEvaluator):
class FactorSingleColumnEvaluator(FactorEvaluator):
def evaluate(
self,
implementation: Implementation,
gt_implementation: Implementation,
implementation: Workspace,
gt_implementation: Workspace,
) -> Tuple[str, object]:
_, gen_df = self._get_df(gt_implementation, implementation)
@@ -147,8 +146,8 @@ class FactorSingleColumnEvaluator(FactorEvaluator):
class FactorOutputFormatEvaluator(FactorEvaluator):
def evaluate(
self,
implementation: Implementation,
gt_implementation: Implementation,
implementation: Workspace,
gt_implementation: Workspace,
) -> Tuple[str, object]:
gt_df, gen_df = self._get_df(gt_implementation, implementation)
if gen_df is None:
@@ -184,8 +183,8 @@ class FactorOutputFormatEvaluator(FactorEvaluator):
class FactorDatetimeDailyEvaluator(FactorEvaluator):
def evaluate(
self,
implementation: Implementation,
gt_implementation: Implementation,
implementation: Workspace,
gt_implementation: Workspace,
) -> Tuple[str | object]:
_, gen_df = self._get_df(gt_implementation, implementation)
if gen_df is None:
@@ -214,8 +213,8 @@ class FactorDatetimeDailyEvaluator(FactorEvaluator):
class FactorRowCountEvaluator(FactorEvaluator):
def evaluate(
self,
implementation: Implementation,
gt_implementation: Implementation,
implementation: Workspace,
gt_implementation: Workspace,
) -> Tuple[str, object]:
gt_df, gen_df = self._get_df(gt_implementation, implementation)
@@ -231,8 +230,8 @@ class FactorRowCountEvaluator(FactorEvaluator):
class FactorIndexEvaluator(FactorEvaluator):
def evaluate(
self,
implementation: Implementation,
gt_implementation: Implementation,
implementation: Workspace,
gt_implementation: Workspace,
) -> Tuple[str, object]:
gt_df, gen_df = self._get_df(gt_implementation, implementation)
@@ -248,8 +247,8 @@ class FactorIndexEvaluator(FactorEvaluator):
class FactorMissingValuesEvaluator(FactorEvaluator):
def evaluate(
self,
implementation: Implementation,
gt_implementation: Implementation,
implementation: Workspace,
gt_implementation: Workspace,
) -> Tuple[str, object]:
gt_df, gen_df = self._get_df(gt_implementation, implementation)
@@ -265,8 +264,8 @@ class FactorMissingValuesEvaluator(FactorEvaluator):
class FactorEqualValueCountEvaluator(FactorEvaluator):
def evaluate(
self,
implementation: Implementation,
gt_implementation: Implementation,
implementation: Workspace,
gt_implementation: Workspace,
) -> Tuple[str, object]:
gt_df, gen_df = self._get_df(gt_implementation, implementation)
@@ -296,8 +295,8 @@ class FactorCorrelationEvaluator(FactorEvaluator):
def evaluate(
self,
implementation: Implementation,
gt_implementation: Implementation,
implementation: Workspace,
gt_implementation: Workspace,
) -> Tuple[str, object]:
gt_df, gen_df = self._get_df(gt_implementation, implementation)
@@ -327,11 +326,10 @@ class FactorCorrelationEvaluator(FactorEvaluator):
class FactorValueEvaluator(FactorEvaluator):
def evaluate(
self,
implementation: Implementation,
gt_implementation: Implementation,
implementation: Workspace,
gt_implementation: Workspace,
**kwargs,
) -> Tuple:
conclusions = []
@@ -508,8 +506,8 @@ class FactorEvaluatorForCoder(FactorEvaluator):
def evaluate(
self,
target_task: FactorTask,
implementation: Implementation,
gt_implementation: Implementation = None,
implementation: Workspace,
gt_implementation: Workspace = None,
queried_knowledge: QueriedKnowledge = None,
**kwargs,
) -> FactorSingleFeedback:
@@ -603,41 +601,21 @@ class FactorMultiEvaluator(Evaluator):
queried_knowledge: QueriedKnowledge = None,
**kwargs,
) -> FactorMultiFeedback:
multi_implementation_feedback = FactorMultiFeedback()
# for index in range(len(evo.sub_tasks)):
# corresponding_implementation = evo.sub_implementations[index]
# corresponding_gt_implementation = (
# evo.sub_gt_implementations[index] if evo.sub_gt_implementations is not None else None
# )
# multi_implementation_feedback.append(
# self.single_factor_implementation_evaluator.evaluate(
# target_task=evo.sub_tasks[index],
# implementation=corresponding_implementation,
# gt_implementation=corresponding_gt_implementation,
# queried_knowledge=queried_knowledge,
# )
# )
calls = []
for index in range(len(evo.sub_tasks)):
corresponding_implementation = evo.sub_implementations[index]
corresponding_gt_implementation = (
evo.sub_gt_implementations[index] if evo.sub_gt_implementations is not None else None
)
calls.append(
multi_implementation_feedback = multiprocessing_wrapper(
[
(
self.single_factor_implementation_evaluator.evaluate,
(
evo.sub_tasks[index],
corresponding_implementation,
corresponding_gt_implementation,
evo.sub_workspace_list[index],
evo.sub_gt_implementations[index] if evo.sub_gt_implementations is not None else None,
queried_knowledge,
),
),
)
multi_implementation_feedback = multiprocessing_wrapper(calls, n=FACTOR_IMPLEMENT_SETTINGS.evo_multi_proc_n)
)
for index in range(len(evo.sub_tasks))
],
n=RD_AGENT_SETTINGS.multi_proc_n,
)
final_decision = [
None if single_feedback is None else single_feedback.final_decision
rdagent/components/coder/factor_coder/CoSTEER/evolvable_subjects.py
+2 -2
View File
@@ -1,7 +1,7 @@
from rdagent.components.coder.factor_coder.factor import (
FactorExperiment,
FactorFBWorkspace,
FactorTask,
FileBasedFactorImplementation,
)
from rdagent.core.evolving_framework import EvolvableSubjects
from rdagent.log import rdagent_logger as logger
@@ -15,7 +15,7 @@ class FactorEvolvingItem(FactorExperiment, EvolvableSubjects):
def __init__(
self,
sub_tasks: list[FactorTask],
sub_gt_implementations: list[FileBasedFactorImplementation] = None,
sub_gt_implementations: list[FactorFBWorkspace] = None,
):
FactorExperiment.__init__(self, sub_tasks=sub_tasks)
self.corresponding_selection: list = None
rdagent/components/coder/factor_coder/CoSTEER/evolving_agent.py
+19
View File
@@ -0,0 +1,19 @@
from rdagent.components.coder.factor_coder.CoSTEER.evaluators import FactorMultiFeedback
from rdagent.components.coder.factor_coder.CoSTEER.evolvable_subjects import (
FactorEvolvingItem,
)
from rdagent.core.evaluation import Feedback
from rdagent.core.evolving_agent import RAGEvoAgent
from rdagent.core.evolving_framework import EvolvableSubjects
class FactorRAGEvoAgent(RAGEvoAgent):
def filter_evolvable_subjects_by_feedback(self, evo: EvolvableSubjects, feedback: Feedback) -> EvolvableSubjects:
assert isinstance(evo, FactorEvolvingItem)
assert isinstance(feedback, list)
assert len(evo.sub_workspace_list) == len(feedback)
for index in range(len(evo.sub_workspace_list)):
if not feedback[index].final_decision:
evo.sub_workspace_list[index].clear()
return evo
rdagent/components/coder/factor_coder/CoSTEER/evolving_strategy.py
+22 -38
View File
@@ -16,14 +16,11 @@ from rdagent.components.coder.factor_coder.CoSTEER.scheduler import (
LLMSelect,
RandomSelect,
)
from rdagent.components.coder.factor_coder.factor import (
FactorTask,
FileBasedFactorImplementation,
)
from rdagent.components.coder.factor_coder.factor import FactorFBWorkspace, FactorTask
from rdagent.components.coder.factor_coder.utils import get_data_folder_intro
from rdagent.core.conf import RD_AGENT_SETTINGS
from rdagent.core.evolving_framework import EvolvingStrategy, QueriedKnowledge
from rdagent.core.experiment import Implementation
from rdagent.core.experiment import Workspace
from rdagent.core.prompts import Prompts
from rdagent.core.utils import multiprocessing_wrapper
from rdagent.oai.llm_utils import APIBackend
@@ -43,7 +40,7 @@ class MultiProcessEvolvingStrategy(EvolvingStrategy):
self,
target_task: FactorTask,
queried_knowledge: QueriedKnowledge = None,
) -> Implementation:
) -> Workspace:
raise NotImplementedError
def evolve(
@@ -53,15 +50,12 @@ class MultiProcessEvolvingStrategy(EvolvingStrategy):
queried_knowledge: FactorQueriedKnowledge | None = None,
**kwargs,
) -> FactorEvolvingItem:
self.num_loop += 1
new_evo = deepcopy(evo)
# 1.找出需要evolve的factor
to_be_finished_task_index = []
for index, target_factor_task in enumerate(new_evo.sub_tasks):
for index, target_factor_task in enumerate(evo.sub_tasks):
target_factor_task_desc = target_factor_task.get_task_information()
if target_factor_task_desc in queried_knowledge.success_task_to_knowledge_dict:
new_evo.sub_implementations[index] = queried_knowledge.success_task_to_knowledge_dict[
evo.sub_workspace_list[index] = queried_knowledge.success_task_to_knowledge_dict[
target_factor_task_desc
].implementation
elif (
@@ -87,30 +81,27 @@ class MultiProcessEvolvingStrategy(EvolvingStrategy):
to_be_finished_task_index = LLMSelect(
to_be_finished_task_index,
implementation_factors_per_round,
new_evo,
evo,
queried_knowledge.former_traces,
self.scen,
)
result = multiprocessing_wrapper(
[
(self.implement_one_factor, (new_evo.sub_tasks[target_index], queried_knowledge))
(self.implement_one_factor, (evo.sub_tasks[target_index], queried_knowledge))
for target_index in to_be_finished_task_index
],
n=FACTOR_IMPLEMENT_SETTINGS.evo_multi_proc_n,
n=RD_AGENT_SETTINGS.multi_proc_n,
)
for index, target_index in enumerate(to_be_finished_task_index):
new_evo.sub_implementations[target_index] = result[index]
if evo.sub_workspace_list[target_index] is None:
evo.sub_workspace_list[target_index] = FactorFBWorkspace(target_task=evo.sub_tasks[target_index])
evo.sub_workspace_list[target_index].inject_code(**{"factor.py": result[index]})
# for target_index in to_be_finished_task_index:
# new_evo.sub_implementations[target_index] = self.implement_one_factor(
# new_evo.sub_tasks[target_index], queried_knowledge
# )
evo.corresponding_selection = to_be_finished_task_index
new_evo.corresponding_selection = to_be_finished_task_index
return new_evo
return evo
class FactorEvolvingStrategy(MultiProcessEvolvingStrategy):
@@ -118,7 +109,7 @@ class FactorEvolvingStrategy(MultiProcessEvolvingStrategy):
self,
target_task: FactorTask,
queried_knowledge: FactorQueriedKnowledgeV1 = None,
) -> Implementation:
) -> str:
factor_information_str = target_task.get_task_information()
if queried_knowledge is not None and factor_information_str in queried_knowledge.success_task_to_knowledge_dict:
@@ -149,7 +140,7 @@ class FactorEvolvingStrategy(MultiProcessEvolvingStrategy):
queried_former_failed_knowledge=queried_former_failed_knowledge_to_render,
)
)
session = APIBackend(use_chat_cache=False).build_chat_session(
session = APIBackend(use_chat_cache=FACTOR_IMPLEMENT_SETTINGS.coder_use_cache).build_chat_session(
session_system_prompt=system_prompt,
)
@@ -185,14 +176,8 @@ class FactorEvolvingStrategy(MultiProcessEvolvingStrategy):
json_mode=True,
),
)["code"]
# ast.parse(code)
factor_implementation = FileBasedFactorImplementation(
target_task,
)
factor_implementation.prepare()
factor_implementation.inject_code(**{"factor.py": code})
return factor_implementation
return code
class FactorEvolvingStrategyWithGraph(MultiProcessEvolvingStrategy):
@@ -205,7 +190,7 @@ class FactorEvolvingStrategyWithGraph(MultiProcessEvolvingStrategy):
self,
target_task: FactorTask,
queried_knowledge,
) -> Implementation:
) -> str:
error_summary = FACTOR_IMPLEMENT_SETTINGS.v2_error_summary
# 1. 提取因子的背景信息
target_factor_task_information = target_task.get_task_information()
@@ -249,7 +234,7 @@ class FactorEvolvingStrategyWithGraph(MultiProcessEvolvingStrategy):
)
)
session = APIBackend(use_chat_cache=False).build_chat_session(
session = APIBackend(use_chat_cache=FACTOR_IMPLEMENT_SETTINGS.coder_use_cache).build_chat_session(
session_system_prompt=system_prompt,
)
@@ -276,7 +261,9 @@ class FactorEvolvingStrategyWithGraph(MultiProcessEvolvingStrategy):
)
.strip("\n")
)
session_summary = APIBackend(use_chat_cache=False).build_chat_session(
session_summary = APIBackend(
use_chat_cache=FACTOR_IMPLEMENT_SETTINGS.coder_use_cache
).build_chat_session(
session_system_prompt=error_summary_system_prompt,
)
for _ in range(10): # max attempt to reduce the length of error_summary_user_prompt
@@ -335,7 +322,4 @@ class FactorEvolvingStrategyWithGraph(MultiProcessEvolvingStrategy):
json_mode=True,
)
code = json.loads(response)["code"]
factor_implementation = FileBasedFactorImplementation(target_task)
factor_implementation.prepare()
factor_implementation.inject_code(**{"factor.py": code})
return factor_implementation
return code
rdagent/components/coder/factor_coder/CoSTEER/knowledge_management.py
+23 -21
View File
@@ -27,9 +27,9 @@ from rdagent.core.evolving_framework import (
QueriedKnowledge,
RAGStrategy,
)
from rdagent.core.experiment import Implementation
from rdagent.log import rdagent_logger as logger
from rdagent.core.experiment import Workspace
from rdagent.core.prompts import Prompts
from rdagent.log import rdagent_logger as logger
from rdagent.oai.llm_utils import (
APIBackend,
calculate_embedding_distance_between_str_list,
@@ -40,7 +40,7 @@ class FactorKnowledge(Knowledge):
def __init__(
self,
target_task: FactorTask,
implementation: Implementation,
implementation: Workspace,
feedback: FactorSingleFeedback,
) -> None:
"""
@@ -53,7 +53,7 @@ class FactorKnowledge(Knowledge):
None
"""
self.target_task = target_task
self.implementation = implementation
self.implementation = implementation.copy()
self.feedback = feedback
def get_implementation_and_feedback_str(self) -> str:
@@ -115,7 +115,7 @@ class FactorRAGStrategyV1(RAGStrategy):
for task_index in range(len(implementations.sub_tasks)):
target_task = implementations.sub_tasks[task_index]
target_task_information = target_task.get_task_information()
implementation = implementations.sub_implementations[task_index]
implementation = implementations.sub_workspace_list[task_index]
single_feedback = feedback[task_index]
if single_feedback is None:
continue
@@ -149,9 +149,9 @@ class FactorRAGStrategyV1(RAGStrategy):
for target_factor_task in evo.sub_tasks:
target_factor_task_information = target_factor_task.get_task_information()
if target_factor_task_information in self.knowledgebase.success_task_info_set:
queried_knowledge.success_task_to_knowledge_dict[target_factor_task_information] = (
self.knowledgebase.implementation_trace[target_factor_task_information][-1]
)
queried_knowledge.success_task_to_knowledge_dict[
target_factor_task_information
] = self.knowledgebase.implementation_trace[target_factor_task_information][-1]
elif (
len(
self.knowledgebase.implementation_trace.setdefault(
@@ -163,12 +163,14 @@ class FactorRAGStrategyV1(RAGStrategy):
):
queried_knowledge.failed_task_info_set.add(target_factor_task_information)
else:
queried_knowledge.working_task_to_former_failed_knowledge_dict[target_factor_task_information] = (
self.knowledgebase.implementation_trace.setdefault(
target_factor_task_information,
[],
)[-v1_query_former_trace_limit:]
)
queried_knowledge.working_task_to_former_failed_knowledge_dict[
target_factor_task_information
] = self.knowledgebase.implementation_trace.setdefault(
target_factor_task_information,
[],
)[
-v1_query_former_trace_limit:
]
knowledge_base_success_task_list = list(
self.knowledgebase.success_task_info_set,
@@ -189,9 +191,9 @@ class FactorRAGStrategyV1(RAGStrategy):
)[-1]
for index in similar_indexes
]
queried_knowledge.working_task_to_similar_successful_knowledge_dict[target_factor_task_information] = (
similar_successful_knowledge
)
queried_knowledge.working_task_to_similar_successful_knowledge_dict[
target_factor_task_information
] = similar_successful_knowledge
return queried_knowledge
@@ -234,7 +236,7 @@ class FactorGraphRAGStrategy(RAGStrategy):
single_feedback = feedback[task_index]
target_task = implementations.sub_tasks[task_index]
target_task_information = target_task.get_task_information()
implementation = implementations.sub_implementations[task_index]
implementation = implementations.sub_workspace_list[task_index]
single_feedback = feedback[task_index]
if single_feedback is None:
continue
@@ -425,9 +427,9 @@ class FactorGraphRAGStrategy(RAGStrategy):
else:
current_index += 1
factor_implementation_queried_graph_knowledge.former_traces[target_factor_task_information] = (
former_trace_knowledge[-v2_query_former_trace_limit:]
)
factor_implementation_queried_graph_knowledge.former_traces[
target_factor_task_information
] = former_trace_knowledge[-v2_query_former_trace_limit:]
else:
factor_implementation_queried_graph_knowledge.former_traces[target_factor_task_information] = []
rdagent/components/coder/factor_coder/CoSTEER/scheduler.py
+1 -1
View File
@@ -9,9 +9,9 @@ from rdagent.components.coder.factor_coder.CoSTEER.evolvable_subjects import (
)
from rdagent.components.coder.factor_coder.utils import get_data_folder_intro
from rdagent.core.conf import RD_AGENT_SETTINGS
from rdagent.log import rdagent_logger as logger
from rdagent.core.prompts import Prompts
from rdagent.core.scenario import Scenario
from rdagent.log import rdagent_logger as logger
from rdagent.oai.llm_utils import APIBackend
scheduler_prompts = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")
rdagent/components/coder/factor_coder/config.py
+9 -8
View File
@@ -7,16 +7,17 @@ SELECT_METHOD = Literal["random", "scheduler"]
class FactorImplementSettings(BaseSettings):
factor_data_folder: str = str(
class Config:
env_prefix = "FACTOR_CODER_" # Use FACTOR_CODER_ as prefix for environment variables
coder_use_cache: bool = False
data_folder: str = str(
(Path().cwd() / "git_ignore_folder" / "factor_implementation_source_data").absolute(),
)
factor_data_folder_debug: str = str(
data_folder_debug: str = str(
(Path().cwd() / "git_ignore_folder" / "factor_implementation_source_data_debug").absolute(),
)
factor_execution_workspace: str = str(
(Path().cwd() / "git_ignore_folder" / "factor_implementation_workspace").absolute(),
)
factor_cache_location: str = str(
cache_location: str = str(
(Path().cwd() / "git_ignore_folder" / "factor_implementation_execution_cache").absolute(),
)
enable_execution_cache: bool = True # whether to enable the execution cache
@@ -35,8 +36,6 @@ class FactorImplementSettings(BaseSettings):
v2_error_summary: bool = False
v2_knowledge_sampler: float = 1.0
evo_multi_proc_n: int = 16 # how many processes to use for evolving (including eval & generation)
file_based_execution_timeout: int = 120 # seconds for each factor implementation execution
select_method: SELECT_METHOD = "random"
@@ -47,5 +46,7 @@ class FactorImplementSettings(BaseSettings):
knowledge_base_path: Union[str, None] = None
new_knowledge_base_path: Union[str, None] = None
python_bin: str = "python"
FACTOR_IMPLEMENT_SETTINGS = FactorImplementSettings()
rdagent/components/coder/factor_coder/factor.py
+10 -18
View File
@@ -15,7 +15,7 @@ from rdagent.core.exception import (
NoOutputException,
RuntimeErrorException,
)
from rdagent.core.experiment import Experiment, FBImplementation, Task
from rdagent.core.experiment import Experiment, FBWorkspace, Task
from rdagent.log import rdagent_logger as logger
from rdagent.oai.llm_utils import md5_hash
@@ -51,7 +51,7 @@ variables: {str(self.variables)}"""
return f"<{self.__class__.__name__}[{self.factor_name}]>"
class FileBasedFactorImplementation(FBImplementation):
class FactorFBWorkspace(FBWorkspace):
"""
This class is used to implement a factor by writing the code to a file.
Input data and output factor value are also written to files.
@@ -90,15 +90,6 @@ class FileBasedFactorImplementation(FBImplementation):
check=False,
)
def execute_desc(self):
raise NotImplementedError
def prepare(self, *args, **kwargs):
self.workspace_path = Path(
FACTOR_IMPLEMENT_SETTINGS.factor_execution_workspace,
) / str(uuid.uuid4())
self.workspace_path.mkdir(exist_ok=True, parents=True)
def execute(self, store_result: bool = False, data_type: str = "Debug") -> Tuple[str, pd.DataFrame]:
"""
execute the implementation and get the factor value by the following steps:
@@ -112,6 +103,7 @@ class FileBasedFactorImplementation(FBImplementation):
parameters:
store_result: if True, store the factor value in the instance variable, this feature is to be used in the gt implementation to avoid multiple execution on the same gt implementation
"""
super().execute()
if self.code_dict is None or "factor.py" not in self.code_dict:
if self.raise_exception:
raise CodeFormatException(self.FB_CODE_NOT_SET)
@@ -122,8 +114,8 @@ class FileBasedFactorImplementation(FBImplementation):
if FACTOR_IMPLEMENT_SETTINGS.enable_execution_cache:
# NOTE: cache the result for the same code and same data type
target_file_name = md5_hash(data_type + self.code_dict["factor.py"])
cache_file_path = Path(FACTOR_IMPLEMENT_SETTINGS.factor_cache_location) / f"{target_file_name}.pkl"
Path(FACTOR_IMPLEMENT_SETTINGS.factor_cache_location).mkdir(exist_ok=True, parents=True)
cache_file_path = Path(FACTOR_IMPLEMENT_SETTINGS.cache_location) / f"{target_file_name}.pkl"
Path(FACTOR_IMPLEMENT_SETTINGS.cache_location).mkdir(exist_ok=True, parents=True)
if cache_file_path.exists() and not self.raise_exception:
cached_res = pickle.load(open(cache_file_path, "rb"))
if store_result and cached_res[1] is not None:
@@ -135,11 +127,11 @@ class FileBasedFactorImplementation(FBImplementation):
source_data_path = (
Path(
FACTOR_IMPLEMENT_SETTINGS.factor_data_folder_debug,
FACTOR_IMPLEMENT_SETTINGS.data_folder_debug,
)
if data_type == "Debug"
else Path(
FACTOR_IMPLEMENT_SETTINGS.factor_data_folder,
FACTOR_IMPLEMENT_SETTINGS.data_folder,
)
)
@@ -151,7 +143,7 @@ class FileBasedFactorImplementation(FBImplementation):
execution_feedback = self.FB_EXECUTION_SUCCEEDED
try:
subprocess.check_output(
f"python {code_path}",
f"{FACTOR_IMPLEMENT_SETTINGS.python_bin} {code_path}",
shell=True,
cwd=self.workspace_path,
stderr=subprocess.STDOUT,
@@ -215,7 +207,7 @@ class FileBasedFactorImplementation(FBImplementation):
for file_path in path.iterdir():
if file_path.suffix == ".py":
code_dict[file_path.name] = file_path.read_text()
return FileBasedFactorImplementation(target_task=task, code_dict=code_dict, **kwargs)
return FactorFBWorkspace(target_task=task, code_dict=code_dict, **kwargs)
class FactorExperiment(Experiment[FactorTask, FileBasedFactorImplementation]): ...
FactorExperiment = Experiment
rdagent/components/coder/factor_coder/utils.py
+1 -1
View File
@@ -22,7 +22,7 @@ def get_data_folder_intro():
It is for preparing prompting message.
"""
content_l = []
for p in Path(FACTOR_IMPLEMENT_SETTINGS.factor_data_folder).iterdir():
for p in Path(FACTOR_IMPLEMENT_SETTINGS.data_folder).iterdir():
if p.name.endswith(".h5"):
df = pd.read_hdf(p)
# get df.head() as string with full width
rdagent/components/coder/model_coder/CoSTEER/__init__.py
+12 -7
View File
@@ -8,6 +8,7 @@ from rdagent.components.coder.model_coder.CoSTEER.evaluators import (
from rdagent.components.coder.model_coder.CoSTEER.evolvable_subjects import (
ModelEvolvingItem,
)
from rdagent.components.coder.model_coder.CoSTEER.evolving_agent import ModelRAGEvoAgent
from rdagent.components.coder.model_coder.CoSTEER.evolving_strategy import (
ModelCoderEvolvingStrategy,
)
@@ -16,17 +17,18 @@ from rdagent.components.coder.model_coder.CoSTEER.knowledge_management import (
ModelRAGStrategy,
)
from rdagent.components.coder.model_coder.model import ModelExperiment
from rdagent.core.developer import Developer
from rdagent.core.evolving_agent import RAGEvoAgent
from rdagent.core.task_generator import TaskGenerator
class ModelCoSTEER(TaskGenerator[ModelExperiment]):
class ModelCoSTEER(Developer[ModelExperiment]):
def __init__(
self,
*args,
with_knowledge: bool = True,
with_feedback: bool = True,
knowledge_self_gen: bool = True,
filter_final_evo: bool = True,
**kwargs,
) -> None:
super().__init__(*args, **kwargs)
@@ -44,6 +46,7 @@ class ModelCoSTEER(TaskGenerator[ModelExperiment]):
self.with_knowledge = with_knowledge
self.with_feedback = with_feedback
self.knowledge_self_gen = knowledge_self_gen
self.filter_final_evo = filter_final_evo
self.evolving_strategy = ModelCoderEvolvingStrategy(scen=self.scen)
self.model_evaluator = ModelCoderMultiEvaluator(scen=self.scen)
@@ -57,7 +60,7 @@ class ModelCoSTEER(TaskGenerator[ModelExperiment]):
return model_knowledge_base
def generate(self, exp: ModelExperiment) -> ModelExperiment:
def develop(self, exp: ModelExperiment) -> ModelExperiment:
# init knowledge base
model_knowledge_base = self.load_or_init_knowledge_base(
former_knowledge_base_path=self.knowledge_base_path,
@@ -69,7 +72,9 @@ class ModelCoSTEER(TaskGenerator[ModelExperiment]):
# init intermediate items
model_experiment = ModelEvolvingItem(sub_tasks=exp.sub_tasks)
self.evolve_agent = RAGEvoAgent(max_loop=self.max_loop, evolving_strategy=self.evolving_strategy, rag=self.rag)
self.evolve_agent = ModelRAGEvoAgent(
max_loop=self.max_loop, evolving_strategy=self.evolving_strategy, rag=self.rag
)
model_experiment = self.evolve_agent.multistep_evolve(
model_experiment,
@@ -77,11 +82,11 @@ class ModelCoSTEER(TaskGenerator[ModelExperiment]):
with_knowledge=self.with_knowledge,
with_feedback=self.with_feedback,
knowledge_self_gen=self.knowledge_self_gen,
filter_final_evo=self.filter_final_evo,
)
# save new knowledge base
if self.new_knowledge_base_path is not None:
pickle.dump(model_knowledge_base, open(self.new_knowledge_base_path, "wb"))
self.knowledge_base = model_knowledge_base
model_experiment.based_experiments = exp.based_experiments
return model_experiment
exp.sub_workspace_list = model_experiment.sub_workspace_list
return exp
rdagent/components/coder/model_coder/CoSTEER/evaluators.py
+24 -29
View File
@@ -11,14 +11,14 @@ from rdagent.components.coder.model_coder.conf import MODEL_IMPL_SETTINGS
from rdagent.components.coder.model_coder.CoSTEER.evolvable_subjects import (
ModelEvolvingItem,
)
from rdagent.components.coder.model_coder.model import ModelImplementation, ModelTask
from rdagent.components.coder.model_coder.model import ModelFBWorkspace, ModelTask
from rdagent.core.conf import RD_AGENT_SETTINGS
from rdagent.core.evaluation import Evaluator
from rdagent.core.evolving_framework import QueriedKnowledge
from rdagent.core.experiment import Implementation, Task
from rdagent.log import rdagent_logger as logger
from rdagent.core.experiment import Task, Workspace
from rdagent.core.prompts import Prompts
from rdagent.core.utils import multiprocessing_wrapper
from rdagent.log import rdagent_logger as logger
from rdagent.oai.llm_utils import APIBackend
evaluate_prompts = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")
@@ -62,15 +62,15 @@ class ModelCodeEvaluator(Evaluator):
def evaluate(
self,
target_task: Task,
implementation: Implementation,
gt_implementation: Implementation,
implementation: Workspace,
gt_implementation: Workspace,
model_execution_feedback: str = "",
model_value_feedback: str = "",
):
assert isinstance(target_task, ModelTask)
assert isinstance(implementation, ModelImplementation)
assert isinstance(implementation, ModelFBWorkspace)
if gt_implementation is not None:
assert isinstance(gt_implementation, ModelImplementation)
assert isinstance(gt_implementation, ModelFBWorkspace)
model_task_information = target_task.get_task_information()
code = implementation.code
@@ -120,16 +120,16 @@ class ModelFinalEvaluator(Evaluator):
def evaluate(
self,
target_task: Task,
implementation: Implementation,
gt_implementation: Implementation,
implementation: Workspace,
gt_implementation: Workspace,
model_execution_feedback: str,
model_value_feedback: str,
model_code_feedback: str,
):
assert isinstance(target_task, ModelTask)
assert isinstance(implementation, ModelImplementation)
assert isinstance(implementation, ModelFBWorkspace)
if gt_implementation is not None:
assert isinstance(gt_implementation, ModelImplementation)
assert isinstance(gt_implementation, ModelFBWorkspace)
system_prompt = (
Environment(undefined=StrictUndefined)
@@ -219,8 +219,8 @@ class ModelCoderEvaluator(Evaluator):
def evaluate(
self,
target_task: Task,
implementation: Implementation,
gt_implementation: Implementation,
implementation: Workspace,
gt_implementation: Workspace,
queried_knowledge: QueriedKnowledge = None,
**kwargs,
) -> ModelCoderFeedback:
@@ -248,7 +248,7 @@ class ModelCoderEvaluator(Evaluator):
input_value = 0.4
param_init_value = 0.6
assert isinstance(implementation, ModelImplementation)
assert isinstance(implementation, ModelFBWorkspace)
model_execution_feedback, gen_tensor = implementation.execute(
batch_size=batch_size,
num_features=num_features,
@@ -257,7 +257,7 @@ class ModelCoderEvaluator(Evaluator):
param_init_value=param_init_value,
)
if gt_implementation is not None:
assert isinstance(gt_implementation, ModelImplementation)
assert isinstance(gt_implementation, ModelFBWorkspace)
_, gt_tensor = gt_implementation.execute(
batch_size=batch_size,
num_features=num_features,
@@ -303,26 +303,21 @@ class ModelCoderMultiEvaluator(Evaluator):
queried_knowledge: QueriedKnowledge = None,
**kwargs,
) -> List[ModelCoderFeedback]:
multi_implementation_feedback = []
calls = []
for index in range(len(evo.sub_tasks)):
corresponding_implementation = evo.sub_implementations[index]
corresponding_gt_implementation = (
evo.sub_gt_implementations[index] if evo.sub_gt_implementations is not None else None
)
calls.append(
multi_implementation_feedback = multiprocessing_wrapper(
[
(
ModelCoderEvaluator(scen=self.scen).evaluate,
(
evo.sub_tasks[index],
corresponding_implementation,
corresponding_gt_implementation,
evo.sub_workspace_list[index],
evo.sub_gt_implementations[index] if evo.sub_gt_implementations is not None else None,
queried_knowledge,
),
),
)
multi_implementation_feedback = multiprocessing_wrapper(calls, n=MODEL_IMPL_SETTINGS.evo_multi_proc_n)
)
for index in range(len(evo.sub_tasks))
],
n=RD_AGENT_SETTINGS.multi_proc_n,
)
final_decision = [
None if single_feedback is None else single_feedback.final_decision
rdagent/components/coder/model_coder/CoSTEER/evolvable_subjects.py
+2 -2
View File
@@ -1,6 +1,6 @@
from rdagent.components.coder.model_coder.model import (
ModelExperiment,
ModelImplementation,
ModelFBWorkspace,
ModelTask,
)
from rdagent.core.evolving_framework import EvolvableSubjects
@@ -15,7 +15,7 @@ class ModelEvolvingItem(ModelExperiment, EvolvableSubjects):
def __init__(
self,
sub_tasks: list[ModelTask],
sub_gt_implementations: list[ModelImplementation] = None,
sub_gt_implementations: list[ModelFBWorkspace] = None,
):
ModelExperiment.__init__(self, sub_tasks=sub_tasks)
if sub_gt_implementations is not None and len(
rdagent/components/coder/model_coder/CoSTEER/evolving_agent.py
+19
View File
@@ -0,0 +1,19 @@
from rdagent.components.coder.model_coder.CoSTEER.evaluators import ModelCoderFeedback
from rdagent.components.coder.model_coder.CoSTEER.evolvable_subjects import (
ModelEvolvingItem,
)
from rdagent.core.evaluation import Feedback
from rdagent.core.evolving_agent import RAGEvoAgent
from rdagent.core.evolving_framework import EvolvableSubjects
class ModelRAGEvoAgent(RAGEvoAgent):
def filter_evolvable_subjects_by_feedback(self, evo: EvolvableSubjects, feedback: Feedback) -> EvolvableSubjects:
assert isinstance(evo, ModelEvolvingItem)
assert isinstance(feedback, list)
assert len(evo.sub_workspace_list) == len(feedback)
for index in range(len(evo.sub_workspace_list)):
if not feedback[index].final_decision:
evo.sub_workspace_list[index].clear()
return evo
rdagent/components/coder/model_coder/CoSTEER/evolving_strategy.py
+15 -24
View File
@@ -11,7 +11,7 @@ from rdagent.components.coder.model_coder.CoSTEER.evolvable_subjects import (
from rdagent.components.coder.model_coder.CoSTEER.knowledge_management import (
ModelQueriedKnowledge,
)
from rdagent.components.coder.model_coder.model import ModelImplementation, ModelTask
from rdagent.components.coder.model_coder.model import ModelFBWorkspace, ModelTask
from rdagent.core.conf import RD_AGENT_SETTINGS
from rdagent.core.evolving_framework import EvolvingStrategy
from rdagent.core.prompts import Prompts
@@ -26,7 +26,7 @@ class ModelCoderEvolvingStrategy(EvolvingStrategy):
self,
target_task: ModelTask,
queried_knowledge: ModelQueriedKnowledge = None,
) -> ModelImplementation:
) -> str:
model_information_str = target_task.get_task_information()
if queried_knowledge is not None and model_information_str in queried_knowledge.success_task_to_knowledge_dict:
@@ -86,19 +86,15 @@ class ModelCoderEvolvingStrategy(EvolvingStrategy):
queried_similar_successful_knowledge_to_render = queried_similar_successful_knowledge_to_render[1:]
code = json.loads(
APIBackend(use_chat_cache=False).build_messages_and_create_chat_completion(
APIBackend(
use_chat_cache=MODEL_IMPL_SETTINGS.coder_use_cache
).build_messages_and_create_chat_completion(
user_prompt=user_prompt,
system_prompt=system_prompt,
json_mode=True,
),
)["code"]
model_implementation = ModelImplementation(
target_task,
)
model_implementation.prepare()
model_implementation.inject_code(**{"model.py": code})
return model_implementation
return code
def evolve(
self,
@@ -107,14 +103,12 @@ class ModelCoderEvolvingStrategy(EvolvingStrategy):
queried_knowledge: ModelQueriedKnowledge | None = None,
**kwargs,
) -> ModelEvolvingItem:
new_evo = deepcopy(evo)
# 1.找出需要evolve的model
to_be_finished_task_index = []
for index, target_model_task in enumerate(new_evo.sub_tasks):
for index, target_model_task in enumerate(evo.sub_tasks):
target_model_task_desc = target_model_task.get_task_information()
if target_model_task_desc in queried_knowledge.success_task_to_knowledge_dict:
new_evo.sub_implementations[index] = queried_knowledge.success_task_to_knowledge_dict[
evo.sub_workspace_list[index] = queried_knowledge.success_task_to_knowledge_dict[
target_model_task_desc
].implementation
elif (
@@ -125,20 +119,17 @@ class ModelCoderEvolvingStrategy(EvolvingStrategy):
result = multiprocessing_wrapper(
[
(self.implement_one_model, (new_evo.sub_tasks[target_index], queried_knowledge))
(self.implement_one_model, (evo.sub_tasks[target_index], queried_knowledge))
for target_index in to_be_finished_task_index
],
n=MODEL_IMPL_SETTINGS.evo_multi_proc_n,
n=RD_AGENT_SETTINGS.multi_proc_n,
)
for index, target_index in enumerate(to_be_finished_task_index):
new_evo.sub_implementations[target_index] = result[index]
if evo.sub_workspace_list[target_index] is None:
evo.sub_workspace_list[target_index] = ModelFBWorkspace(target_task=evo.sub_tasks[target_index])
evo.sub_workspace_list[target_index].inject_code(**{"model.py": result[index]})
# for target_index in to_be_finished_task_index:
# new_evo.sub_implementations[target_index] = self.implement_one_model(
# new_evo.sub_tasks[target_index], queried_knowledge
# )
evo.corresponding_selection = to_be_finished_task_index
new_evo.corresponding_selection = to_be_finished_task_index
return new_evo
return evo
rdagent/components/coder/model_coder/CoSTEER/knowledge_management.py
+18 -16
View File
@@ -9,7 +9,7 @@ from rdagent.core.evolving_framework import (
QueriedKnowledge,
RAGStrategy,
)
from rdagent.core.experiment import Implementation
from rdagent.core.experiment import Workspace
from rdagent.oai.llm_utils import calculate_embedding_distance_between_str_list
@@ -17,7 +17,7 @@ class ModelKnowledge(Knowledge):
def __init__(
self,
target_task: ModelTask,
implementation: Implementation,
implementation: Workspace,
feedback: ModelCoderFeedback,
) -> None:
"""
@@ -30,7 +30,7 @@ class ModelKnowledge(Knowledge):
None
"""
self.target_task = target_task
self.implementation = implementation
self.implementation = implementation.copy()
self.feedback = feedback
def get_implementation_and_feedback_str(self) -> str:
@@ -87,7 +87,7 @@ class ModelRAGStrategy(RAGStrategy):
for task_index in range(len(implementations.sub_tasks)):
target_task = implementations.sub_tasks[task_index]
target_task_information = target_task.get_task_information()
implementation = implementations.sub_implementations[task_index]
implementation = implementations.sub_workspace_list[task_index]
single_feedback = feedback[task_index]
if single_feedback is None:
continue
@@ -121,9 +121,9 @@ class ModelRAGStrategy(RAGStrategy):
for target_model_task in evo.sub_tasks:
target_model_task_information = target_model_task.get_task_information()
if target_model_task_information in self.knowledgebase.success_task_info_set:
queried_knowledge.success_task_to_knowledge_dict[target_model_task_information] = (
self.knowledgebase.implementation_trace[target_model_task_information][-1]
)
queried_knowledge.success_task_to_knowledge_dict[
target_model_task_information
] = self.knowledgebase.implementation_trace[target_model_task_information][-1]
elif (
len(
self.knowledgebase.implementation_trace.setdefault(
@@ -135,12 +135,14 @@ class ModelRAGStrategy(RAGStrategy):
):
queried_knowledge.failed_task_info_set.add(target_model_task_information)
else:
queried_knowledge.working_task_to_former_failed_knowledge_dict[target_model_task_information] = (
self.knowledgebase.implementation_trace.setdefault(
target_model_task_information,
[],
)[-query_former_trace_limit:]
)
queried_knowledge.working_task_to_former_failed_knowledge_dict[
target_model_task_information
] = self.knowledgebase.implementation_trace.setdefault(
target_model_task_information,
[],
)[
-query_former_trace_limit:
]
knowledge_base_success_task_list = list(
self.knowledgebase.success_task_info_set,
@@ -161,7 +163,7 @@ class ModelRAGStrategy(RAGStrategy):
)[-1]
for index in similar_indexes
]
queried_knowledge.working_task_to_similar_successful_knowledge_dict[target_model_task_information] = (
similar_successful_knowledge
)
queried_knowledge.working_task_to_similar_successful_knowledge_dict[
target_model_task_information
] = similar_successful_knowledge
return queried_knowledge
rdagent/components/coder/model_coder/benchmark/eval.py
+2 -2
View File
@@ -1,7 +1,7 @@
# TODO: inherent from the benchmark base class
import torch
from rdagent.components.coder.model_coder.model import ModelImplementation
from rdagent.components.coder.model_coder.model import ModelFBWorkspace
def get_data_conf(init_val):
@@ -32,7 +32,7 @@ class ModelImpValEval:
For each hidden output, we can calculate a correlation. The average correlation will be the metrics.
"""
def evaluate(self, gt: ModelImplementation, gen: ModelImplementation):
def evaluate(self, gt: ModelFBWorkspace, gen: ModelFBWorkspace):
round_n = 10
eval_pairs: list[tuple] = []
rdagent/components/coder/model_coder/conf.py
+4 -7
View File
@@ -6,12 +6,11 @@ from pydantic_settings import BaseSettings
class ModelImplSettings(BaseSettings):
class Config:
env_prefix = "MODEL_IMPL_" # Use MODEL_IMPL_ as prefix for environment variables
env_prefix = "MODEL_CODER_" # Use MODEL_CODER_ as prefix for environment variables
model_execution_workspace: str = str(
(Path().cwd() / "git_ignore_folder" / "model_implementation_workspace").absolute(),
)
model_cache_location: str = str(
coder_use_cache: bool = False
cache_location: str = str(
(Path().cwd() / "git_ignore_folder" / "model_implementation_execution_cache").absolute(),
)
@@ -24,8 +23,6 @@ class ModelImplSettings(BaseSettings):
query_similar_success_limit: int = 5
fail_task_trial_limit: int = 20
evo_multi_proc_n: int = 1
enable_execution_cache: bool = True # whether to enable the execution cache
rdagent/components/coder/model_coder/main.py
+5 -3
View File
@@ -10,6 +10,10 @@ import os
import torch
from dotenv import load_dotenv
from rdagent.components.coder.model_coder.CoSTEER.evaluators import (
shape_evaluator,
value_evaluator,
)
from rdagent.oai.llm_utils import APIBackend
assert load_dotenv()
@@ -68,8 +72,6 @@ for test_mode in ["zeros", "ones", "randn"]:
os.system("rm node_features.pt")
# load the output and print the shape
from evaluator import shape_evaluator, value_evaluator
try:
llm_output = torch.load("llm_output.pt")
except:
@@ -80,7 +82,7 @@ for test_mode in ["zeros", "ones", "randn"]:
average_value_eval.append(value_evaluator(llm_output, gt_output)[1])
print("Shape evaluation: ", average_shape_eval[-1])
print("Value evaluation:super().generate(task_l) ", average_value_eval[-1])
print("Value evaluation:super().develop(task_l) ", average_value_eval[-1])
os.system("rm llm_output.pt")
os.system("rm gt_output.pt")
rdagent/components/coder/model_coder/model.py
+6 -17
View File
@@ -9,7 +9,7 @@ import torch
from rdagent.components.coder.model_coder.conf import MODEL_IMPL_SETTINGS
from rdagent.core.exception import CodeFormatException
from rdagent.core.experiment import Experiment, FBImplementation, Task
from rdagent.core.experiment import Experiment, FBWorkspace, Task
from rdagent.oai.llm_utils import md5_hash
from rdagent.utils import get_module_by_module_path
@@ -40,7 +40,7 @@ model_type: {self.model_type}
return f"<{self.__class__.__name__} {self.name}>"
class ModelImplementation(FBImplementation):
class ModelFBWorkspace(FBWorkspace):
"""
It is a Pytorch model implementation task;
All the things are placed in a folder.
@@ -60,18 +60,6 @@ class ModelImplementation(FBImplementation):
"""
def __init__(self, target_task: Task) -> None:
super().__init__(target_task)
def prepare(self) -> None:
"""
Prepare for the workspace;
"""
unique_id = uuid.uuid4()
self.workspace_path = Path(MODEL_IMPL_SETTINGS.model_execution_workspace) / f"M{unique_id}"
# start with `M` so that it can be imported via python
self.workspace_path.mkdir(parents=True, exist_ok=True)
def execute(
self,
batch_size: int = 8,
@@ -80,14 +68,15 @@ class ModelImplementation(FBImplementation):
input_value: float = 1.0,
param_init_value: float = 1.0,
):
super().execute()
try:
if MODEL_IMPL_SETTINGS.enable_execution_cache:
# NOTE: cache the result for the same code
target_file_name = md5_hash(
f"{batch_size}_{num_features}_{num_timesteps}_{input_value}_{param_init_value}_{self.code_dict['model.py']}"
)
cache_file_path = Path(MODEL_IMPL_SETTINGS.model_cache_location) / f"{target_file_name}.pkl"
Path(MODEL_IMPL_SETTINGS.model_cache_location).mkdir(exist_ok=True, parents=True)
cache_file_path = Path(MODEL_IMPL_SETTINGS.cache_location) / f"{target_file_name}.pkl"
Path(MODEL_IMPL_SETTINGS.cache_location).mkdir(exist_ok=True, parents=True)
if cache_file_path.exists():
return pickle.load(open(cache_file_path, "rb"))
mod = get_module_by_module_path(str(self.workspace_path / "model.py"))
@@ -115,4 +104,4 @@ class ModelImplementation(FBImplementation):
return f"Execution error: {e}", None
class ModelExperiment(Experiment[ModelTask, ModelImplementation]): ...
ModelExperiment = Experiment
rdagent/components/coder/model_coder/one_shot/__init__.py
+6 -9
View File
@@ -3,22 +3,19 @@ from pathlib import Path
from jinja2 import Environment, StrictUndefined
from rdagent.components.coder.model_coder.model import (
ModelExperiment,
ModelImplementation,
)
from rdagent.components.coder.model_coder.model import ModelExperiment, ModelFBWorkspace
from rdagent.core.developer import Developer
from rdagent.core.prompts import Prompts
from rdagent.core.task_generator import TaskGenerator
from rdagent.oai.llm_utils import APIBackend
DIRNAME = Path(__file__).absolute().resolve().parent
class ModelCodeWriter(TaskGenerator[ModelExperiment]):
def generate(self, exp: ModelExperiment) -> ModelExperiment:
class ModelCodeWriter(Developer[ModelExperiment]):
def develop(self, exp: ModelExperiment) -> ModelExperiment:
mti_l = []
for t in exp.sub_tasks:
mti = ModelImplementation(t)
mti = ModelFBWorkspace(t)
mti.prepare()
pr = Prompts(file_path=DIRNAME / "prompt.yaml")
@@ -40,5 +37,5 @@ class ModelCodeWriter(TaskGenerator[ModelExperiment]):
code = match.group(1)
mti.inject_code(**{"model.py": code})
mti_l.append(mti)
exp.sub_implementations = mti_l
exp.sub_workspace_list = mti_l
return exp
rdagent/components/coder/model_coder/task_loader.py
+4 -2
View File
@@ -9,12 +9,14 @@ from rdagent.components.document_reader.document_reader import (
load_and_process_pdfs_by_langchain,
)
from rdagent.components.loader.task_loader import ModelTaskLoader
from rdagent.log import rdagent_logger as logger
from rdagent.core.prompts import Prompts
from rdagent.log import rdagent_logger as logger
from rdagent.oai.llm_utils import APIBackend
from rdagent.scenarios.qlib.experiment.model_experiment import QlibModelExperiment
document_process_prompts = Prompts(file_path=Path(__file__).parent / "prompts.yaml")
def extract_model_from_doc(doc_content: str) -> dict:
"""
Extract model information from document content.
@@ -107,7 +109,7 @@ class ModelExperimentLoaderFromDict(ModelTaskLoader):
key=model_name,
)
task_l.append(task)
return ModelExperiment(sub_tasks=task_l)
return QlibModelExperiment(sub_tasks=task_l)
class ModelExperimentLoaderFromPDFfiles(ModelTaskLoader):
rdagent/components/knowledge_management/vector_base.py
+1
View File
@@ -8,6 +8,7 @@ from scipy.spatial.distance import cosine
from rdagent.log import rdagent_logger as logger
from rdagent.oai.llm_utils import APIBackend
class KnowledgeMetaData:
def __init__(self, content: str = "", label: str = None, embedding=None, identity=None):
self.label = label
rdagent/components/loader/task_loader.py
+5 -6
View File
@@ -3,8 +3,8 @@ from pathlib import Path
from typing import Sequence
from rdagent.components.coder.factor_coder.factor import FactorTask
from rdagent.components.coder.model_coder.model import ModelImplementation, ModelTask
from rdagent.core.experiment import ImpLoader, Loader
from rdagent.components.coder.model_coder.model import ModelFBWorkspace, ModelTask
from rdagent.core.experiment import Loader, WsLoader
class FactorTaskLoader(Loader[FactorTask]):
@@ -80,16 +80,15 @@ class ModelTaskLoaderJson(ModelTaskLoader):
return model_impl_task_list
class ModelImpLoader(ImpLoader[ModelTask, ModelImplementation]):
class ModelWsLoader(WsLoader[ModelTask, ModelFBWorkspace]):
def __init__(self, path: Path) -> None:
self.path = Path(path)
def load(self, task: ModelTask) -> ModelImplementation:
def load(self, task: ModelTask) -> ModelFBWorkspace:
assert task.name is not None
mti = ModelImplementation(task)
mti = ModelFBWorkspace(task)
mti.prepare()
with open(self.path / f"{task.name}.py", "r") as f:
code = f.read()
mti.inject_code(**{"model.py": code})
return mti
return mti
rdagent/components/proposal/factor_proposal.py
+9 -6
View File
@@ -27,10 +27,12 @@ class FactorHypothesisGen(HypothesisGen):
# The following methods are scenario related so they should be implemented in the subclass
@abstractmethod
def prepare_context(self, trace: Trace) -> Tuple[dict, bool]: ...
def prepare_context(self, trace: Trace) -> Tuple[dict, bool]:
...
@abstractmethod
def convert_response(self, response: str) -> FactorHypothesis: ...
def convert_response(self, response: str) -> FactorHypothesis:
...
def gen(self, trace: Trace) -> FactorHypothesis:
context_dict, json_flag = self.prepare_context(trace)
@@ -62,12 +64,13 @@ class FactorHypothesisGen(HypothesisGen):
class FactorHypothesis2Experiment(Hypothesis2Experiment[FactorExperiment]):
@abstractmethod
def prepare_context(self, hypothesis: Hypothesis, trace: Trace) -> Tuple[dict, bool]:
...
@abstractmethod
def prepare_context(self, hypothesis: Hypothesis, trace: Trace) -> Tuple[dict, bool]: ...
@abstractmethod
def convert_response(self, response: str, trace: Trace) -> FactorExperiment: ...
def convert_response(self, response: str, trace: Trace) -> FactorExperiment:
...
def convert(self, hypothesis: Hypothesis, trace: Trace) -> FactorExperiment:
context, json_flag = self.prepare_context(hypothesis, trace)
rdagent/components/proposal/model_proposal.py
+8 -5
View File
@@ -21,13 +21,14 @@ ModelHypothesis = Hypothesis
class ModelHypothesisGen(HypothesisGen):
# The following methods are scenario related so they should be implemented in the subclass
@abstractmethod
def prepare_context(self, trace: Trace) -> Tuple[dict, bool]: ...
def prepare_context(self, trace: Trace) -> Tuple[dict, bool]:
...
@abstractmethod
def convert_response(self, response: str) -> ModelHypothesis: ...
def convert_response(self, response: str) -> ModelHypothesis:
...
def gen(self, trace: Trace) -> ModelHypothesis:
context_dict, json_flag = self.prepare_context(trace)
@@ -63,10 +64,12 @@ class ModelHypothesis2Experiment(Hypothesis2Experiment[ModelExperiment]):
super().__init__()
@abstractmethod
def prepare_context(self, hypothesis: Hypothesis, trace: Trace) -> Tuple[dict, bool]: ...
def prepare_context(self, hypothesis: Hypothesis, trace: Trace) -> Tuple[dict, bool]:
...
@abstractmethod
def convert_response(self, response: str, trace: Trace) -> ModelExperiment: ...
def convert_response(self, response: str, trace: Trace) -> ModelExperiment:
...
def convert(self, hypothesis: Hypothesis, trace: Trace) -> ModelExperiment:
context, json_flag = self.prepare_context(hypothesis, trace)
rdagent/components/runner/__init__.py
+5 -5
View File
@@ -3,12 +3,12 @@ from pathlib import Path
from typing import Tuple
from rdagent.components.runner.conf import RUNNER_SETTINGS
from rdagent.core.developer import Developer
from rdagent.core.experiment import ASpecificExp, Experiment
from rdagent.core.task_generator import TaskGenerator
from rdagent.oai.llm_utils import md5_hash
class CachedRunner(TaskGenerator[ASpecificExp]):
class CachedRunner(Developer[ASpecificExp]):
def get_cache_key(self, exp: Experiment) -> str:
all_tasks = []
for based_exp in exp.based_experiments:
@@ -20,8 +20,8 @@ class CachedRunner(TaskGenerator[ASpecificExp]):
def get_cache_result(self, exp: Experiment) -> Tuple[bool, object]:
task_info_key = self.get_cache_key(exp)
Path(RUNNER_SETTINGS.runner_cache_path).mkdir(parents=True, exist_ok=True)
cache_path = Path(RUNNER_SETTINGS.runner_cache_path) / f"{task_info_key}.pkl"
Path(RUNNER_SETTINGS.cache_path).mkdir(parents=True, exist_ok=True)
cache_path = Path(RUNNER_SETTINGS.cache_path) / f"{task_info_key}.pkl"
if cache_path.exists():
return True, pickle.load(open(cache_path, "rb"))
else:
@@ -29,5 +29,5 @@ class CachedRunner(TaskGenerator[ASpecificExp]):
def dump_cache_result(self, exp: Experiment, result: object):
task_info_key = self.get_cache_key(exp)
cache_path = Path(RUNNER_SETTINGS.runner_cache_path) / f"{task_info_key}.pkl"
cache_path = Path(RUNNER_SETTINGS.cache_path) / f"{task_info_key}.pkl"
pickle.dump(result, open(cache_path, "wb"))
rdagent/components/runner/conf.py
+5 -2
View File
@@ -12,8 +12,11 @@ from pydantic_settings import BaseSettings
class RunnerSettings(BaseSettings):
runner_cache_result: bool = True # whether to cache the result of the docker execution
runner_cache_path: str = str(Path.cwd() / "runner_cache/") # the path to store the cache
class Config:
env_prefix = "RUNNER_" # Use MODEL_CODER_ as prefix for environment variables
cache_result: bool = True # whether to cache the result of the docker execution
cache_path: str = str(Path.cwd() / "runner_cache/") # the path to store the cache
RUNNER_SETTINGS = RunnerSettings()
rdagent/core/conf.py
+7 -1
View File
@@ -18,7 +18,7 @@ class RDAgentSettings(BaseSettings):
# TODO: (xiao) I think LLMSetting may be a better name.
# TODO: (xiao) I think most of the config should be in oai.config
# Log configs
# TODO: (xiao) think it can be a seperate config.
# TODO: (xiao) think it can be a separate config.
log_trace_path: str | None = None
log_llm_chat_content: bool = True
@@ -100,5 +100,11 @@ class RDAgentSettings(BaseSettings):
max_input_duplicate_factor_group: int = 600
max_output_duplicate_factor_group: int = 20
# workspace conf
workspace_path: Path = Path.cwd() / "git_ignore_folder" / "RD-Agent_workspace"
# multi processing conf
multi_proc_n: int = 1
RD_AGENT_SETTINGS = RDAgentSettings()
rdagent/core/task_generator.py → rdagent/core/developer.py
+2 -13
View File
@@ -5,12 +5,12 @@ from rdagent.core.experiment import ASpecificExp
from rdagent.core.scenario import Scenario
class TaskGenerator(ABC, Generic[ASpecificExp]):
class Developer(ABC, Generic[ASpecificExp]):
def __init__(self, scen: Scenario) -> None:
self.scen: Scenario = scen
@abstractmethod
def generate(self, exp: ASpecificExp) -> ASpecificExp:
def develop(self, exp: ASpecificExp) -> ASpecificExp:
"""
Task Generator should take in an experiment.
@@ -19,14 +19,3 @@ class TaskGenerator(ABC, Generic[ASpecificExp]):
"""
raise NotImplementedError("generate method is not implemented.")
def collect_feedback(self, feedback_obj_l: List[object]):
"""
When online evaluation.
The previous feedbacks will be collected to support advanced factor generator
Parameters
----------
feedback_obj_l : List[object]
"""
rdagent/core/evaluation.py
+3 -3
View File
@@ -1,6 +1,6 @@
from abc import ABC, abstractmethod
from rdagent.core.experiment import Implementation, Task
from rdagent.core.experiment import Task, Workspace
from rdagent.core.scenario import Scenario
@@ -19,8 +19,8 @@ class Evaluator(ABC):
def evaluate(
self,
target_task: Task,
implementation: Implementation,
gt_implementation: Implementation,
implementation: Workspace,
gt_implementation: Workspace,
**kwargs,
):
raise NotImplementedError
rdagent/core/evolving_agent.py
+10 -4
View File
@@ -1,5 +1,5 @@
from abc import ABC, abstractmethod
from typing import Any
from typing import Any, List
from tqdm import tqdm
@@ -14,14 +14,18 @@ class EvoAgent(ABC):
@abstractmethod
def multistep_evolve(self, evo: EvolvableSubjects, eva: Evaluator | Feedback, **kwargs: Any) -> EvolvableSubjects:
pass
...
@abstractmethod
def filter_evolvable_subjects_by_feedback(self, evo: EvolvableSubjects, feedback: Feedback) -> EvolvableSubjects:
...
class RAGEvoAgent(EvoAgent):
def __init__(self, max_loop, evolving_strategy, rag) -> None:
super().__init__(max_loop, evolving_strategy)
self.rag = rag
self.evolving_trace = []
self.evolving_trace: List[EvoStep] = []
def multistep_evolve(
self,
@@ -31,6 +35,7 @@ class RAGEvoAgent(EvoAgent):
with_knowledge: bool = False,
with_feedback: bool = True,
knowledge_self_gen: bool = False,
filter_final_evo: bool = False,
) -> EvolvableSubjects:
for _ in tqdm(range(self.max_loop), "Implementing"):
# 1. knowledge self-evolving
@@ -60,5 +65,6 @@ class RAGEvoAgent(EvoAgent):
# 6. update trace
self.evolving_trace.append(es)
if with_feedback and filter_final_evo:
evo = self.filter_evolvable_subjects_by_feedback(evo, self.evolving_trace[-1].feedback)
return evo
rdagent/core/evolving_framework.py
+2 -1
View File
@@ -32,7 +32,8 @@ class EvolvableSubjects:
return copy.deepcopy(self)
class QlibEvolvableSubjects(EvolvableSubjects): ...
class QlibEvolvableSubjects(EvolvableSubjects):
...
@dataclass
rdagent/core/exception.py
+22 -4
View File
@@ -1,4 +1,4 @@
class ImplementRunException(Exception):
class CoderException(Exception):
"""
Exceptions raised when Implementing and running code.
- start: FactorTask => FactorGenerator
@@ -8,19 +8,37 @@ class ImplementRunException(Exception):
"""
class CodeFormatException(ImplementRunException):
class CodeFormatException(CoderException):
"""
The generated code is not found due format error.
"""
class RuntimeErrorException(ImplementRunException):
class RuntimeErrorException(CoderException):
"""
The generated code fail to execute the script.
"""
class NoOutputException(ImplementRunException):
class NoOutputException(CoderException):
"""
The code fail to generate output file.
"""
class RunnerException(Exception):
"""
Exceptions raised when running the code output.
"""
class FactorEmptyException(Exception):
"""
Exceptions raised when no factor is generated correctly
"""
class ModelEmptyException(Exception):
"""
Exceptions raised when no model is generated correctly
"""
rdagent/core/experiment.py
+78 -42
View File
@@ -1,17 +1,20 @@
from __future__ import annotations
import shutil
import uuid
from abc import ABC, abstractmethod
from copy import deepcopy
from pathlib import Path
from typing import Any, Dict, Generic, Optional, Sequence, TypeVar
from rdagent.core.conf import RD_AGENT_SETTINGS
"""
This file contains the all the class about organizing the task in RD-Agent.
"""
class Task(ABC):
# TODO: 把name放在这里作为主键
# Please refer to rdagent/model_implementation/task.py for the implementation
# I think the task version applies to the base class.
@abstractmethod
def get_task_information(self):
"""
@@ -23,39 +26,44 @@ class Task(ABC):
ASpecificTask = TypeVar("ASpecificTask", bound=Task)
class Implementation(ABC, Generic[ASpecificTask]):
# TODO: workspace;
# - code or data(optional)
# - Execute logic
# - `env is not included`. It is a underlying infra
def __init__(self, target_task: ASpecificTask) -> None:
self.target_task = target_task
class Workspace(ABC, Generic[ASpecificTask]):
"""
A workspace is a place to store the task implementation. It evolves as the developer implements the task.
To get a snapshot of the workspace, make sure call `copy` to get a copy of the workspace.
"""
def __init__(self, target_task: ASpecificTask = None) -> None:
self.target_task: ASpecificTask = target_task
@abstractmethod
def execute(self, *args, **kwargs) -> object:
raise NotImplementedError("execute method is not implemented.")
ASpecificImp = TypeVar("ASpecificImp", bound=Implementation)
class ImpLoader(ABC, Generic[ASpecificTask, ASpecificImp]):
@abstractmethod
def load(self, task: ASpecificTask) -> ASpecificImp:
def copy(self) -> Workspace:
raise NotImplementedError("copy method is not implemented.")
ASpecificWS = TypeVar("ASpecificWS", bound=Workspace)
class WsLoader(ABC, Generic[ASpecificTask, ASpecificWS]):
@abstractmethod
def load(self, task: ASpecificTask) -> ASpecificWS:
raise NotImplementedError("load method is not implemented.")
class FBImplementation(Implementation):
class FBWorkspace(Workspace):
"""
File-based task implementation
File-based task workspace
The implemented task will be a folder which contains related elements.
- Data
- Code Implementation
- Code Workspace
- Output
- After execution, it will generate the final output as file.
A typical way to run the pipeline of FBImplementation will be
A typical way to run the pipeline of FBWorkspace will be
(We didn't add it as a method due to that we may pass arguments into `prepare` or `execute` based on our requirements.)
.. code-block:: python
@@ -67,15 +75,12 @@ class FBImplementation(Implementation):
"""
# TODO:
# FileBasedFactorImplementation should inherit from it.
# Why not directly reuse FileBasedFactorImplementation.
# Because it has too much concrete dependencies.
# e.g. dataframe, factors
def __init__(self, *args, code_dict: Dict[str, str] = None, **kwargs) -> None:
def __init__(self, *args, **kwargs) -> None:
super().__init__(*args, **kwargs)
self.code_dict = code_dict # The code to be injected into the folder, store them in the variable
self.workspace_path: Optional[Path] = None
self.code_dict = (
{}
) # The code injected into the folder, store them in the variable to reproduce the former result
self.workspace_path: Path = RD_AGENT_SETTINGS.workspace_path / uuid.uuid4().hex
@property
def code(self) -> str:
@@ -84,28 +89,26 @@ class FBImplementation(Implementation):
code_string += f"File: {file_name}\n{code}\n"
return code_string
@abstractmethod
def prepare(self, *args, **kwargs):
"""
Prepare all the files except the injected code
Prepare the workspace except the injected code
- Data
- Documentation
- TODO: env? Env is implicitly defined by the document?
typical usage of `*args, **kwargs`:
Different methods shares the same data. The data are passed by the arguments.
"""
# TODO: model and factor prepare;
self.workspace_path.mkdir(parents=True, exist_ok=True)
def inject_code(self, **files: str):
"""
Inject the code into the folder.
{
"model.py": "<model code>"
<file name>: <code>
}
"""
self.code_dict = files
self.prepare()
for k, v in files.items():
self.code_dict[k] = v
with open(self.workspace_path / k, "w") as f:
f.write(v)
@@ -114,22 +117,55 @@ class FBImplementation(Implementation):
Get the environment description.
To be general, we only return a list of filenames.
How to summarize the environment is the responsibility of the TaskGenerator.
How to summarize the environment is the responsibility of the Developer.
"""
return list(self.workspace_path.iterdir())
def inject_code_from_folder(self, folder_path: Path):
"""
Load the workspace from the folder
"""
for file_path in folder_path.iterdir():
if file_path.suffix == ".py" or file_path.suffix == ".yaml":
self.inject_code(**{file_path.name: file_path.read_text()})
class Experiment(ABC, Generic[ASpecificTask, ASpecificImp]):
def copy(self) -> FBWorkspace:
"""
copy the workspace from the original one
"""
return deepcopy(self)
def clear(self) -> None:
"""
Clear the workspace
"""
shutil.rmtree(self.workspace_path)
self.code_dict = {}
@abstractmethod
def execute(self, *args, **kwargs) -> object:
"""
Before each execution, make sure to prepare and inject code
"""
self.prepare()
self.inject_code(**self.code_dict)
ASpecificWSForExperiment = TypeVar("ASpecificWSForExperiment", bound=Workspace)
ASpecificWSForSubTasks = TypeVar("ASpecificWSForSubTasks", bound=Workspace)
class Experiment(ABC, Generic[ASpecificTask, ASpecificWSForExperiment, ASpecificWSForSubTasks]):
"""
The experiment is a sequence of tasks and the implementations of the tasks after generated by the TaskGenerator.
The experiment is a sequence of tasks and the implementations of the tasks after generated by the Developer.
"""
def __init__(self, sub_tasks: Sequence[ASpecificTask]) -> None:
self.sub_tasks = sub_tasks
self.sub_implementations: Sequence[ASpecificImp] = [None for _ in self.sub_tasks]
self.based_experiments: Sequence[Experiment] = []
self.sub_workspace_list: Sequence[ASpecificWSForSubTasks] = [None for _ in self.sub_tasks]
self.based_experiments: Sequence[ASpecificWSForExperiment] = []
self.result: object = None # The result of the experiment, can be different types in different scenarios.
self.exp_ws: ASpecificImp = None
self.experiment_workspace: ASpecificWSForExperiment = None
ASpecificExp = TypeVar("ASpecificExp", bound=Experiment)
rdagent/core/proposal.py
+9 -3
View File
@@ -45,6 +45,13 @@ class HypothesisFeedback(Feedback):
def __bool__(self):
return self.decision
def __str__(self) -> str:
return f"""Observations: {self.observations}
Hypothesis Evaluation: {self.hypothesis_evaluation}
New Hypothesis: {self.new_hypothesis}
Decision: {self.decision}
Reason: {self.reason}"""
ASpecificScen = TypeVar("ASpecificScen", bound=Scenario)
@@ -64,10 +71,11 @@ class Trace(Generic[ASpecificScen]):
return None, None
class HypothesisGen:
class HypothesisGen(ABC):
def __init__(self, scen: Scenario):
self.scen = scen
@abstractmethod
def gen(self, trace: Trace) -> Hypothesis:
# def gen(self, scenario_desc: str, ) -> Hypothesis:
"""
@@ -107,5 +115,3 @@ class HypothesisExperiment2Feedback:
For example: `mlflow` of Qlib will be included.
"""
raise NotImplementedError("generateFeedback method is not implemented.")
# def generateResultComparison()
rdagent/log/__init__.py
+3 -3
View File
@@ -1,4 +1,4 @@
from .logger import RDAgentLog
from .utils import LogColors
from rdagent.log.logger import RDAgentLog
from rdagent.log.utils import LogColors
rdagent_logger: RDAgentLog = RDAgentLog()
rdagent_logger: RDAgentLog = RDAgentLog()
rdagent/log/base.py
+4 -3
View File
@@ -47,16 +47,17 @@ class View:
Display the content in the storage
"""
# TODO: pleas fix me
@abstractmethod
def display(s: Storage, watch: bool=False):
def display(s: Storage, watch: bool = False):
"""
Parameters
----------
s : Storage
watch : bool
should we watch the new content and display them
"""
...
...
rdagent/log/logger.py
+34 -28
View File
@@ -1,16 +1,17 @@
import sys
import os
from loguru import logger
from rdagent.core.utils import SingletonBaseClass
from rdagent.core.conf import RD_AGENT_SETTINGS
from pathlib import Path
from psutil import Process
import sys
from contextlib import contextmanager
from datetime import datetime, timezone
from functools import partial
from contextlib import contextmanager
from multiprocessing import Pipe
from multiprocessing.connection import Connection
from pathlib import Path
from loguru import logger
from psutil import Process
from rdagent.core.conf import RD_AGENT_SETTINGS
from rdagent.core.utils import SingletonBaseClass
from .storage import FileStorage
from .utils import LogColors, get_caller_info
@@ -22,7 +23,7 @@ class RDAgentLog(SingletonBaseClass):
Here is an example tag
.. code-block::
a
- b
- c
@@ -38,6 +39,7 @@ class RDAgentLog(SingletonBaseClass):
- 1233-365 ...
"""
# TODO: Simplify it to introduce less concepts ( We may merge RDAgentLog, Storage &)
# Solution: Storage => PipeLog, View => PipeLogView, RDAgentLog is an instance of PipeLogger
# PipeLogger.info(...) , PipeLogger.get_resp() to get feedback from frontend.
@@ -48,16 +50,15 @@ class RDAgentLog(SingletonBaseClass):
_tag: str = ""
def __init__(self, log_trace_path: str | None = RD_AGENT_SETTINGS.log_trace_path) -> None:
if log_trace_path is None:
timestamp = datetime.now(timezone.utc).strftime("%Y-%m-%d_%H-%M-%S-%f")
log_trace_path: Path = Path.cwd() / "log" / timestamp
self.log_trace_path = Path(log_trace_path)
self.log_trace_path.mkdir(parents=True, exist_ok=True)
self.storage = FileStorage(log_trace_path)
self.main_pid = os.getpid()
@contextmanager
@@ -70,13 +71,13 @@ class RDAgentLog(SingletonBaseClass):
# TODO: It may result in error in mutithreading or co-routine
self._tag = self._tag + tag
yield
self._tag = self._tag[:-len(tag)]
self._tag = self._tag[: -len(tag)]
def get_pids(self) -> str:
'''
"""
Returns a string of pids from the current process to the main process.
Split by '-'.
'''
"""
pid = os.getpid()
process = Process(pid)
pid_chain = f"{pid}"
@@ -87,7 +88,7 @@ class RDAgentLog(SingletonBaseClass):
process = parent_process
return pid_chain
def file_format(self, record, raw: bool=False):
def file_format(self, record, raw: bool = False):
record["message"] = LogColors.remove_ansi_codes(record["message"])
if raw:
return "{message}"
@@ -95,24 +96,25 @@ class RDAgentLog(SingletonBaseClass):
def log_object(self, obj: object, *, tag: str = "") -> None:
caller_info = get_caller_info()
tag = f"{self._tag}.{tag}.{self.get_pids()}".strip('.')
tag = f"{self._tag}.{tag}.{self.get_pids()}".strip(".")
logp = self.storage.log(obj, name=tag, save_type="pkl")
file_handler_id = logger.add(self.log_trace_path / tag.replace('.','/') / "common_logs.log", format=self.file_format)
file_handler_id = logger.add(
self.log_trace_path / tag.replace(".", "/") / "common_logs.log", format=self.file_format
)
logger.patch(lambda r: r.update(caller_info)).info(f"Logging object in {logp.absolute()}")
logger.remove(file_handler_id)
def info(self, msg: str, *, tag: str = "", raw: bool=False) -> None:
def info(self, msg: str, *, tag: str = "", raw: bool = False) -> None:
# TODO: too much duplicated. due to we have no logger with stream context;
caller_info = get_caller_info()
if raw:
logger.remove()
logger.add(sys.stderr, format=lambda r: "{message}")
tag = f"{self._tag}.{tag}.{self.get_pids()}".strip('.')
log_file_path = self.log_trace_path / tag.replace('.','/') / "common_logs.log"
tag = f"{self._tag}.{tag}.{self.get_pids()}".strip(".")
log_file_path = self.log_trace_path / tag.replace(".", "/") / "common_logs.log"
if raw:
file_handler_id = logger.add(log_file_path, format=partial(self.file_format, raw=True))
else:
@@ -131,15 +133,19 @@ class RDAgentLog(SingletonBaseClass):
# getattr(logger.patch(lambda r: r.update(caller_info)), level)(msg)
caller_info = get_caller_info()
tag = f"{self._tag}.{tag}.{self.get_pids()}".strip('.')
file_handler_id = logger.add(self.log_trace_path / tag.replace('.','/') / "common_logs.log", format=self.file_format)
tag = f"{self._tag}.{tag}.{self.get_pids()}".strip(".")
file_handler_id = logger.add(
self.log_trace_path / tag.replace(".", "/") / "common_logs.log", format=self.file_format
)
logger.patch(lambda r: r.update(caller_info)).warning(msg)
logger.remove(file_handler_id)
def error(self, msg: str, *, tag: str = "") -> None:
caller_info = get_caller_info()
tag = f"{self._tag}.{tag}.{self.get_pids()}".strip('.')
file_handler_id = logger.add(self.log_trace_path / tag.replace('.','/') / "common_logs.log", format=self.file_format)
tag = f"{self._tag}.{tag}.{self.get_pids()}".strip(".")
file_handler_id = logger.add(
self.log_trace_path / tag.replace(".", "/") / "common_logs.log", format=self.file_format
)
logger.patch(lambda r: r.update(caller_info)).error(msg)
logger.remove(file_handler_id)
rdagent/log/storage.py
+11 -9
View File
@@ -1,11 +1,12 @@
import json
import pickle
from datetime import datetime, timezone
from pathlib import Path
from typing import Literal
from pathlib import Path
from datetime import datetime, timezone
from .base import Storage
class FileStorage(Storage):
"""
The info are logginged to the file systems
@@ -17,18 +18,19 @@ class FileStorage(Storage):
self.path = Path(path)
self.path.mkdir(parents=True, exist_ok=True)
def log(self,
obj: object,
name: str = "",
save_type: Literal["json", "text", "pkl"] = "text",
timestamp: datetime | None = None,
) -> Path:
def log(
self,
obj: object,
name: str = "",
save_type: Literal["json", "text", "pkl"] = "text",
timestamp: datetime | None = None,
) -> Path:
# TODO: We can remove the timestamp after we implement PipeLog
if timestamp is None:
timestamp = datetime.now(timezone.utc)
else:
timestamp = timestamp.astimezone(timezone.utc)
cur_p = self.path / name.replace(".", "/")
cur_p.mkdir(parents=True, exist_ok=True)
rdagent/log/ui/web.py
+5 -6
View File
@@ -1,20 +1,18 @@
from rdagent.log.base import View, Storage
from pathlib import Path
from rdagent.log.base import Storage, View
class ProcessView(View):
def __init__(self, trace_path: Path):
# Save logs to your desired data structure
# ...
pass
def display(s: Storage, watch: bool = False):
pass
class WebView(View):
r"""
@@ -55,6 +53,7 @@ class WebView(View):
2. Map path like `a.b.c` to frontend components
3. Display logic
"""
def __init__(self, trace_path: Path):
pass
# Save logs to your desired data structure
@@ -74,7 +73,7 @@ class STLWindow:
...
def consume_msg(self, msg):
... # update it's view
... # update it's view
class STLUI:
rdagent/log/utils.py
+6 -6
View File
@@ -23,7 +23,7 @@ class LogColors:
END = "\033[0m"
@classmethod
def get_all_colors(cls: type['LogColors']) -> list:
def get_all_colors(cls: type["LogColors"]) -> list:
names = dir(cls)
names = [name for name in names if not name.startswith("__") and not callable(getattr(cls, name))]
return [getattr(cls, name) for name in names]
@@ -52,8 +52,8 @@ class LogColors:
"""
It is for removing ansi ctrl characters in the string(e.g. colored text)
"""
ansi_escape = re.compile(r'\x1B\[[0-?]*[ -/]*[@-~]')
return ansi_escape.sub('', s)
ansi_escape = re.compile(r"\x1B\[[0-?]*[ -/]*[@-~]")
return ansi_escape.sub("", s)
def get_caller_info():
@@ -63,8 +63,8 @@ def get_caller_info():
caller_info = stack[2]
frame = caller_info[0]
info = {
'line': caller_info.lineno,
'name': frame.f_globals['__name__'], # Get the module name from the frame's globals
'function': frame.f_code.co_name, # Get the caller's function name
"line": caller_info.lineno,
"name": frame.f_globals["__name__"], # Get the module name from the frame's globals
"function": frame.f_code.co_name, # Get the caller's function name
}
return info
rdagent/oai/llm_utils.py
+7 -5
View File
@@ -19,11 +19,13 @@ import numpy as np
import tiktoken
from rdagent.core.conf import RD_AGENT_SETTINGS
from rdagent.log import LogColors, rdagent_logger as logger
from rdagent.core.utils import SingletonBaseClass
from rdagent.log import LogColors
from rdagent.log import rdagent_logger as logger
DEFAULT_QLIB_DOT_PATH = Path("./")
def md5_hash(input_string: str) -> str:
hash_md5 = hashlib.md5(usedforsecurity=False)
input_bytes = input_string.encode("utf-8")
@@ -203,7 +205,7 @@ class ChatSession:
user prompt should always be provided
"""
messages = self.build_chat_completion_message(user_prompt)
with logger.tag(f"session_{self.conversation_id}"):
response = self.api_backend._try_create_chat_completion_or_embedding( # noqa: SLF001
messages=messages,
@@ -651,7 +653,7 @@ class APIBackend:
# TODO: with logger.config(stream=self.chat_stream): and add a `stream_start` flag to add timestamp for first message.
if self.cfg.log_llm_chat_content:
logger.info(f"{LogColors.CYAN}Response:{LogColors.END}", tag="llm_messages")
for chunk in response:
content = (
chunk.choices[0].delta.content
@@ -663,10 +665,10 @@ class APIBackend:
resp += content
if len(chunk.choices) > 0 and chunk.choices[0].finish_reason is not None:
finish_reason = chunk.choices[0].finish_reason
if self.cfg.log_llm_chat_content:
logger.info("\n", raw=True, tag="llm_messages")
else:
resp = response.choices[0].message.content
finish_reason = response.choices[0].finish_reason
rdagent/scenarios/qlib/factor_task_implementation/__init__.py → rdagent/scenarios/qlib/developer/factor_coder.py
View File
rdagent/scenarios/qlib/task_generator/data.py → rdagent/scenarios/qlib/developer/factor_runner.py
+16 -36
View File
@@ -1,17 +1,14 @@
import pickle
import shutil
from pathlib import Path
from typing import List, Tuple
from typing import List
import pandas as pd
from rdagent.components.runner import CachedRunner
from rdagent.components.runner.conf import RUNNER_SETTINGS
from rdagent.core.exception import FactorEmptyException
from rdagent.log import rdagent_logger as logger
from rdagent.core.task_generator import TaskGenerator
from rdagent.oai.llm_utils import md5_hash
from rdagent.scenarios.qlib.experiment.factor_experiment import QlibFactorExperiment
from rdagent.utils.env import QTDockerEnv
DIRNAME = Path(__file__).absolute().resolve().parent
DIRNAME_local = Path.cwd()
@@ -40,15 +37,15 @@ class QlibFactorRunner(CachedRunner[QlibFactorExperiment]):
- results in `mlflow`
"""
def generate(self, exp: QlibFactorExperiment) -> QlibFactorExperiment:
def develop(self, exp: QlibFactorExperiment) -> QlibFactorExperiment:
"""
Generate the experiment by processing and combining factor data,
then passing the combined data to Docker for backtest results.
"""
if exp.based_experiments and exp.based_experiments[-1].result is None:
exp.based_experiments[-1] = self.generate(exp.based_experiments[-1])
exp.based_experiments[-1] = self.develop(exp.based_experiments[-1])
if RUNNER_SETTINGS.runner_cache_result:
if RUNNER_SETTINGS.cache_result:
cache_hit, result = self.get_cache_result(exp)
if cache_hit:
exp.result = result
@@ -56,12 +53,15 @@ class QlibFactorRunner(CachedRunner[QlibFactorExperiment]):
if exp.based_experiments:
SOTA_factor = None
if exp.based_experiments.__len__() != 1:
if len(exp.based_experiments) > 1:
SOTA_factor = self.process_factor_data(exp.based_experiments)
# Process the new factors data
new_factors = self.process_factor_data(exp)
if new_factors.empty:
raise FactorEmptyException("No valid factor data found to merge.")
# Combine the SOTA factor and new factors if SOTA factor exists
if SOTA_factor is not None and not SOTA_factor.empty:
combined_factors = pd.concat([SOTA_factor, new_factors], axis=1).dropna()
@@ -73,36 +73,16 @@ class QlibFactorRunner(CachedRunner[QlibFactorExperiment]):
new_columns = pd.MultiIndex.from_product([["feature"], combined_factors.columns])
combined_factors.columns = new_columns
# Save the combined factors to a pickle file
combined_factors_path = DIRNAME / "env_factor/combined_factors_df.pkl"
with open(combined_factors_path, "wb") as f:
# Save the combined factors to the workspace
with open(exp.experiment_workspace.workspace_path / "combined_factors_df.pkl", "wb") as f:
pickle.dump(combined_factors, f)
# Docker run
# Call Docker, pass the combined factors to Docker, and generate backtest results
qtde = QTDockerEnv()
qtde.prepare()
# Run the Docker command
execute_log = qtde.run(local_path=str(DIRNAME / "env_factor"), entry="rm -r mlruns")
# Run the Qlib backtest
execute_log = qtde.run(
local_path=str(DIRNAME / "env_factor"),
entry=f"qrun conf.yaml" if len(exp.based_experiments) == 0 else "qrun conf_combined.yaml",
result = exp.experiment_workspace.execute(
qlib_config_name=f"conf.yaml" if len(exp.based_experiments) == 0 else "conf_combined.yaml"
)
execute_log = qtde.run(local_path=str(DIRNAME / "env_factor"), entry="python read_exp_res.py")
csv_path = DIRNAME / "env_factor/qlib_res.csv"
if not csv_path.exists():
logger.error(f"File {csv_path} does not exist.")
return None
result = pd.read_csv(csv_path, index_col=0).iloc[:, 0]
exp.result = result
if RUNNER_SETTINGS.runner_cache_result:
if RUNNER_SETTINGS.cache_result:
self.dump_cache_result(exp, result)
return exp
@@ -124,11 +104,11 @@ class QlibFactorRunner(CachedRunner[QlibFactorExperiment]):
# Collect all exp's dataframes
for exp in exp_or_list:
# Iterate over sub-implementations and execute them to get each factor data
for implementation in exp.sub_implementations:
for implementation in exp.sub_workspace_list:
message, df = implementation.execute(data_type="All")
# Check if factor generation was successful
if df is not None:
if df is not None and "datetime" in df.index.names:
time_diff = df.index.get_level_values("datetime").to_series().diff().dropna().unique()
if pd.Timedelta(minutes=1) not in time_diff:
factor_dfs.append(df)
rdagent/scenarios/qlib/task_generator/feedback.py → rdagent/scenarios/qlib/developer/feedback.py
+4 -15
View File
@@ -7,7 +7,6 @@ from pathlib import Path
from jinja2 import Environment, StrictUndefined
from rdagent.core.experiment import Experiment
from rdagent.log import rdagent_logger as logger
from rdagent.core.prompts import Prompts
from rdagent.core.proposal import (
Hypothesis,
@@ -15,6 +14,7 @@ from rdagent.core.proposal import (
HypothesisFeedback,
Trace,
)
from rdagent.log import rdagent_logger as logger
from rdagent.oai.llm_utils import APIBackend
feedback_prompts = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")
@@ -76,8 +76,7 @@ class QlibFactorHypothesisExperiment2Feedback(HypothesisExperiment2Feedback):
reason = response_json.get("Reasoning", "No reasoning provided")
decision = response_json.get("Replace Best Result", "no").lower() == "yes"
# Create HypothesisFeedback object
hypothesis_feedback = HypothesisFeedback(
return HypothesisFeedback(
observations=observations,
hypothesis_evaluation=hypothesis_evaluation,
new_hypothesis=new_hypothesis,
@@ -85,17 +84,6 @@ class QlibFactorHypothesisExperiment2Feedback(HypothesisExperiment2Feedback):
decision=decision,
)
logger.info(
"Generated Hypothesis Feedback:\n"
f"Observations: {observations}\n"
f"Feedback for Hypothesis: {hypothesis_evaluation}\n"
f"New Hypothesis: {new_hypothesis}\n"
f"Reason: {reason}\n"
f"Replace Best Result: {'Yes' if decision else 'No'}"
)
return hypothesis_feedback
class QlibModelHypothesisExperiment2Feedback(HypothesisExperiment2Feedback):
"""Generated feedbacks on the hypothesis from **Executed** Implementations of different tasks & their comparisons with previous performances"""
@@ -106,6 +94,7 @@ class QlibModelHypothesisExperiment2Feedback(HypothesisExperiment2Feedback):
For example: `mlflow` of Qlib will be included.
"""
logger.info("Generating feedback...")
# Define the system prompt for hypothesis feedback
system_prompt = feedback_prompts["model_feedback_generation"]["system"]
@@ -140,5 +129,5 @@ class QlibModelHypothesisExperiment2Feedback(HypothesisExperiment2Feedback):
hypothesis_evaluation=response_json_hypothesis.get("Feedback for Hypothesis", "No feedback provided"),
new_hypothesis=response_json_hypothesis.get("New Hypothesis", "No new hypothesis provided"),
reason=response_json_hypothesis.get("Reasoning", "No reasoning provided"),
decision=response_json_hypothesis.get("Decision", "false").lower() == "true",
decision=str(response_json_hypothesis.get("Decision", "false")).lower() == "true",
)
rdagent/scenarios/qlib/model_task_implementation/__init__.py → rdagent/scenarios/qlib/developer/model_coder.py
View File
rdagent/scenarios/qlib/developer/model_runner.py
+55
View File
@@ -0,0 +1,55 @@
import shutil
import uuid
from pathlib import Path
import pandas as pd
from rdagent.components.coder.model_coder.model import ModelExperiment, ModelFBWorkspace
from rdagent.components.runner import CachedRunner
from rdagent.components.runner.conf import RUNNER_SETTINGS
from rdagent.core.developer import Developer
from rdagent.core.exception import ModelEmptyException
from rdagent.log import rdagent_logger as logger
from rdagent.scenarios.qlib.experiment.model_experiment import QlibModelExperiment
from rdagent.utils.env import QTDockerEnv
class QlibModelRunner(CachedRunner[QlibModelExperiment]):
"""
Docker run
Everything in a folder
- config.yaml
- Pytorch `model.py`
- results in `mlflow`
https://github.com/microsoft/qlib/blob/main/qlib/contrib/model/pytorch_nn.py
- pt_model_uri: hard-code `model.py:Net` in the config
- let LLM modify model.py
"""
def develop(self, exp: QlibModelExperiment) -> QlibModelExperiment:
if RUNNER_SETTINGS.cache_result:
cache_hit, result = self.get_cache_result(exp)
if cache_hit:
exp.result = result
return exp
if exp.sub_workspace_list[0].code_dict.get("model.py") is None:
raise ModelEmptyException("model.py is empty")
# to replace & inject code
exp.experiment_workspace.inject_code(**{"model.py": exp.sub_workspace_list[0].code_dict["model.py"]})
env_to_use = {"PYTHONPATH": "./"}
if exp.sub_tasks[0].model_type == "TimeSeries":
env_to_use.update({"dataset_cls": "TSDatasetH", "step_len": 20, "num_timesteps": 20})
elif exp.sub_tasks[0].model_type == "Tabular":
env_to_use.update({"dataset_cls": "DatasetH"})
result = exp.experiment_workspace.execute(qlib_config_name="conf.yaml", run_env=env_to_use)
exp.result = result
if RUNNER_SETTINGS.cache_result:
self.dump_cache_result(exp, result)
return exp
rdagent/scenarios/qlib/experiment/factor_experiment.py
+11 -2
View File
@@ -1,13 +1,22 @@
from pathlib import Path
from rdagent.components.coder.factor_coder.factor import FactorExperiment
from rdagent.components.coder.factor_coder.factor import (
FactorExperiment,
FactorFBWorkspace,
FactorTask,
)
from rdagent.components.coder.factor_coder.utils import get_data_folder_intro
from rdagent.core.prompts import Prompts
from rdagent.core.scenario import Scenario
from rdagent.scenarios.qlib.experiment.workspace import QlibFBWorkspace
prompt_dict = Prompts(file_path=Path(__file__).parent / "prompts.yaml")
QlibFactorExperiment = FactorExperiment
class QlibFactorExperiment(FactorExperiment[FactorTask, QlibFBWorkspace, FactorFBWorkspace]):
def __init__(self, *args, **kwargs) -> None:
super().__init__(*args, **kwargs)
self.experiment_workspace = QlibFBWorkspace(template_folder_path=Path(__file__).parent / "factor_template")
class QlibFactorScenario(Scenario):
rdagent/scenarios/qlib/task_generator/env_factor/conf.yaml → rdagent/scenarios/qlib/experiment/factor_template/conf.yaml
View File
rdagent/scenarios/qlib/task_generator/env_factor/conf_combined.yaml → rdagent/scenarios/qlib/experiment/factor_template/conf_combined.yaml
+1 -1
View File
@@ -21,7 +21,7 @@ data_handler_config: &data_handler_config
- ["LABEL0"]
- class: qlib.data.dataset.loader.StaticDataLoader
kwargs:
# config: "/home/finco/v-yuanteli/RD-Agent/rdagent/scenarios/qlib/task_generator/env_factor/combined_factors_df.pkl"
# config: "/home/finco/v-yuanteli/RD-Agent/rdagent/scenarios/qlib/task_generator/factor_template/combined_factors_df.pkl"
config: "combined_factors_df.pkl"
learn_processors:
rdagent/scenarios/qlib/task_generator/env_factor/read_exp_res.py → rdagent/scenarios/qlib/experiment/factor_template/read_exp_res.py
View File
rdagent/scenarios/qlib/experiment/model_experiment.py
+11 -2
View File
@@ -1,12 +1,21 @@
from pathlib import Path
from rdagent.components.coder.model_coder.model import ModelExperiment
from rdagent.components.coder.model_coder.model import (
ModelExperiment,
ModelFBWorkspace,
ModelTask,
)
from rdagent.core.prompts import Prompts
from rdagent.core.scenario import Scenario
from rdagent.scenarios.qlib.experiment.workspace import QlibFBWorkspace
prompt_dict = Prompts(file_path=Path(__file__).parent / "prompts.yaml")
QlibModelExperiment = ModelExperiment
class QlibModelExperiment(ModelExperiment[ModelTask, QlibFBWorkspace, ModelFBWorkspace]):
def __init__(self, *args, **kwargs) -> None:
super().__init__(*args, **kwargs)
self.experiment_workspace = QlibFBWorkspace(template_folder_path=Path(__file__).parent / "model_template")
class QlibModelScenario(Scenario):
rdagent/scenarios/qlib/task_generator/model_template/README.md → rdagent/scenarios/qlib/experiment/model_template/README.md
View File
rdagent/scenarios/qlib/task_generator/model_template/conf.yaml → rdagent/scenarios/qlib/experiment/model_template/conf.yaml
View File
rdagent/scenarios/qlib/task_generator/model_template/read_exp_res.py → rdagent/scenarios/qlib/experiment/model_template/read_exp_res.py
View File
rdagent/scenarios/qlib/experiment/workspace.py
+44
View File
@@ -0,0 +1,44 @@
from pathlib import Path
import pandas as pd
from rdagent.core.experiment import FBWorkspace
from rdagent.log import rdagent_logger as logger
from rdagent.utils.env import QTDockerEnv
class QlibFBWorkspace(FBWorkspace):
def __init__(self, template_folder_path: Path, *args, **kwargs) -> None:
super().__init__(*args, **kwargs)
self.inject_code_from_folder(template_folder_path)
def execute(self, qlib_config_name: str = "conf.yaml", run_env: dict = {}, *args, **kwargs) -> str:
qtde = QTDockerEnv()
qtde.prepare()
# Run the Docker command
execute_log = qtde.run(
local_path=str(self.workspace_path),
entry="rm -r mlruns",
env=run_env,
)
# Run the Qlib backtest
execute_log = qtde.run(
local_path=str(self.workspace_path),
entry=f"qrun {qlib_config_name}",
env=run_env,
)
execute_log = qtde.run(
local_path=str(self.workspace_path),
entry="python read_exp_res.py",
env=run_env,
)
csv_path = self.workspace_path / "qlib_res.csv"
if not csv_path.exists():
logger.error(f"File {csv_path} does not exist.")
return None
return pd.read_csv(csv_path, index_col=0).iloc[:, 0]
rdagent/scenarios/qlib/factor_experiment_loader/json_loader.py
+4 -3
View File
@@ -4,11 +4,12 @@ from pathlib import Path
from rdagent.components.benchmark.eval_method import TestCase
from rdagent.components.coder.factor_coder.factor import (
FactorExperiment,
FactorFBWorkspace,
FactorTask,
FileBasedFactorImplementation,
)
from rdagent.components.loader.experiment_loader import FactorExperimentLoader
from rdagent.core.experiment import Loader
from rdagent.scenarios.qlib.experiment.factor_experiment import QlibFactorExperiment
class FactorExperimentLoaderFromDict(FactorExperimentLoader):
@@ -23,7 +24,7 @@ class FactorExperimentLoaderFromDict(FactorExperimentLoader):
variables=factor_data["variables"],
)
task_l.append(task)
exp = FactorExperiment(sub_tasks=task_l)
exp = QlibFactorExperiment(sub_tasks=task_l)
return exp
@@ -54,7 +55,7 @@ class FactorTestCaseLoaderFromJsonFile:
factor_formulation=factor_data["formulation"],
variables=factor_data["variables"],
)
gt = FileBasedFactorImplementation(task, code=factor_data["gt_code"])
gt = FactorFBWorkspace(task, code=factor_data["gt_code"])
gt.execute()
TestData.target_task.append(task)
TestData.ground_truth.append(gt)
rdagent/scenarios/qlib/factor_experiment_loader/pdf_loader.py
+2 -1
View File
@@ -18,8 +18,8 @@ from rdagent.components.document_reader.document_reader import (
)
from rdagent.components.loader.experiment_loader import FactorExperimentLoader
from rdagent.core.conf import RD_AGENT_SETTINGS
from rdagent.log import rdagent_logger as logger
from rdagent.core.prompts import Prompts
from rdagent.log import rdagent_logger as logger
from rdagent.oai.llm_utils import APIBackend, create_embedding_with_multiprocessing
from rdagent.scenarios.qlib.factor_experiment_loader.json_loader import (
FactorExperimentLoaderFromDict,
@@ -27,6 +27,7 @@ from rdagent.scenarios.qlib.factor_experiment_loader.json_loader import (
document_process_prompts = Prompts(file_path=Path(__file__).parent / "prompts.yaml")
def classify_report_from_dict(
report_dict: Mapping[str, str],
vote_time: int = 1,
rdagent/scenarios/qlib/factor_proposal.py → rdagent/scenarios/qlib/proposal/factor_proposal.py
+4 -3
View File
@@ -12,8 +12,9 @@ from rdagent.components.proposal.factor_proposal import (
)
from rdagent.core.prompts import Prompts
from rdagent.core.proposal import Hypothesis, Scenario, Trace
from rdagent.scenarios.qlib.experiment.factor_experiment import QlibFactorExperiment
prompt_dict = Prompts(file_path=Path(__file__).parent / "prompts.yaml")
prompt_dict = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")
QlibFactorHypothesis = FactorHypothesis
@@ -75,8 +76,8 @@ class QlibFactorHypothesis2Experiment(FactorHypothesis2Experiment):
formulation = response_dict[factor_name]["formulation"]
variables = response_dict[factor_name]["variables"]
tasks.append(FactorTask(factor_name, description, formulation, variables))
exp = FactorExperiment(tasks)
exp = QlibFactorExperiment(tasks)
exp.based_experiments = [t[1] for t in trace.hist if t[2]]
if len(exp.based_experiments) == 0:
exp.based_experiments.append(FactorExperiment(sub_tasks=[]))
exp.based_experiments.append(QlibFactorExperiment(sub_tasks=[]))
return exp
rdagent/scenarios/qlib/model_proposal.py → rdagent/scenarios/qlib/proposal/model_proposal.py
+3 -2
View File
@@ -12,8 +12,9 @@ from rdagent.components.proposal.model_proposal import (
)
from rdagent.core.prompts import Prompts
from rdagent.core.proposal import Hypothesis, Scenario, Trace
from rdagent.scenarios.qlib.experiment.model_experiment import QlibModelExperiment
prompt_dict = Prompts(file_path=Path(__file__).parent / "prompts.yaml")
prompt_dict = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")
QlibModelHypothesis = ModelHypothesis
@@ -76,6 +77,6 @@ class QlibModelHypothesis2Experiment(ModelHypothesis2Experiment):
variables = response_dict[model_name]["variables"]
model_type = response_dict[model_name]["model_type"]
tasks.append(ModelTask(model_name, description, formulation, variables, model_type))
exp = ModelExperiment(tasks)
exp = QlibModelExperiment(tasks)
exp.based_experiments = [t[1] for t in trace.hist if t[2]]
return exp
rdagent/scenarios/qlib/task_generator/model.py
-84
View File
@@ -1,84 +0,0 @@
import shutil
import uuid
from pathlib import Path
import pandas as pd
from rdagent.components.coder.model_coder.model import (
ModelExperiment,
ModelImplementation,
)
from rdagent.components.runner import CachedRunner
from rdagent.components.runner.conf import RUNNER_SETTINGS
from rdagent.log import rdagent_logger as logger
from rdagent.core.task_generator import TaskGenerator
from rdagent.utils.env import QTDockerEnv
class QlibModelRunner(CachedRunner[ModelImplementation]):
"""
Docker run
Everything in a folder
- config.yaml
- Pytorch `model.py`
- results in `mlflow`
https://github.com/microsoft/qlib/blob/main/qlib/contrib/model/pytorch_nn.py
- pt_model_uri: hard-code `model.py:Net` in the config
- let LLM modify model.py
"""
def generate(self, exp: ModelExperiment) -> ModelExperiment:
if RUNNER_SETTINGS.runner_cache_result:
cache_hit, result = self.get_cache_result(exp)
if cache_hit:
exp.result = result
return exp
TEMPLATE_PATH = Path(__file__).parent / "model_template" # Can be updated
# To set the experiment level workspace and prepare the workspaces use the first task as the target task
exp.exp_ws = ModelImplementation(target_task=exp.sub_tasks[0])
exp.exp_ws.prepare()
# to copy_template_to_workspace
for file_path in TEMPLATE_PATH.iterdir():
shutil.copyfile(file_path, exp.exp_ws.workspace_path / file_path.name)
# to replace & inject code
exp.exp_ws.inject_code(**{"model.py": exp.sub_implementations[0].code_dict["model.py"]})
env_to_use = {}
if exp.sub_tasks[0].model_type == "TimeSeries":
env_to_use = {"dataset_cls": "TSDatasetH", "step_len": 20, "num_timesteps": 20}
elif exp.sub_tasks[0].model_type == "Tabular":
env_to_use = {"dataset_cls": "DatasetH"}
# to execute
qtde = QTDockerEnv()
# Preparing the Docker environment
qtde.prepare()
# Run the Docker container with the specified entry
execute_log = qtde.run(
local_path=exp.exp_ws.workspace_path, entry="qrun conf.yaml", env={"PYTHONPATH": "./", **env_to_use}
)
# Run the experiment analysis code
execute_log = qtde.run(local_path=exp.exp_ws.workspace_path, entry="python read_exp_res.py")
csv_path = exp.exp_ws.workspace_path / "qlib_res.csv"
if not csv_path.exists():
logger.error(f"File {csv_path} does not exist.")
return None
result = pd.read_csv(csv_path, index_col=0).iloc[:,0]
exp.result = result
if RUNNER_SETTINGS.runner_cache_result:
self.dump_cache_result(exp, result)
return exp
rdagent/utils/env.py
+6 -3
View File
@@ -112,9 +112,9 @@ class LocalEnv(Env[LocalConf]):
class DockerConf(BaseSettings):
build_from_dockerfile: bool = False
dockerfile_folder_path: Optional[Path] = (
None # the path to the dockerfile optional path provided when build_from_dockerfile is False
)
dockerfile_folder_path: Optional[
Path
] = None # the path to the dockerfile optional path provided when build_from_dockerfile is False
image: str # the image you want to build
mount_path: str # the path in the docker image to mount the folder
default_entry: str # the entry point of the image
@@ -128,6 +128,9 @@ class DockerConf(BaseSettings):
class QlibDockerConf(DockerConf):
class Config:
env_prefix = "QLIB_DOCKER_" # Use QLIB_DOCKER_ as prefix for environment variables
build_from_dockerfile: bool = True
dockerfile_folder_path: Path = Path(__file__).parent.parent / "scenarios" / "qlib" / "docker"
image: str = "local_qlib:latest"
test/utils/env_tpl/read_exp.py
+3 -1
View File
@@ -1,9 +1,11 @@
import qlib
from mlflow.tracking import MlflowClient
from mlflow.entities import ViewType
from mlflow.tracking import MlflowClient
qlib.init()
from qlib.workflow import R
# here is the documents of the https://qlib.readthedocs.io/en/latest/component/recorder.html
# TODO: list all the recorder and metrics