mirror of
https://github.com/NicolasBohn/NexQuant.git
synced 2026-08-04 02:37:44 +00:00
Merge pull request #65 from microsoft/ytli_update
Completed and integrated the entire factor process
This commit is contained in:
WinstonLiyt
@@ -153,3 +153,8 @@ git_ignore_folder/
|
||||
|
||||
# DB files
|
||||
*.db
|
||||
|
||||
# Docker
|
||||
env_factor/mlruns/
|
||||
env_tpl
|
||||
mlruns/
|
||||
@@ -1,4 +1,5 @@
|
||||
from pydantic_settings import BaseSettings
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
class PropSetting(BaseSettings):
|
||||
@@ -22,6 +23,8 @@ class PropSetting(BaseSettings):
|
||||
qlib_model_summarizer: str = "rdagent.scenarios.qlib.task_generator.feedback.QlibModelHypothesisExperiment2Feedback"
|
||||
|
||||
evolving_n: int = 10
|
||||
|
||||
|
||||
|
||||
py_bin: str = "/usr/bin/python"
|
||||
local_qlib_folder: Path = Path("/home/rdagent/qlib")
|
||||
|
||||
PROP_SETTING = PropSetting()
|
||||
|
||||
@@ -25,9 +25,10 @@ hypothesis_gen: HypothesisGen = import_class(PROP_SETTING.qlib_factor_hypothesis
|
||||
hypothesis2experiment: Hypothesis2Experiment = import_class(PROP_SETTING.qlib_factor_hypothesis2experiment)()
|
||||
|
||||
qlib_factor_coder: TaskGenerator = import_class(PROP_SETTING.qlib_factor_coder)(scen)
|
||||
|
||||
qlib_factor_runner: TaskGenerator = import_class(PROP_SETTING.qlib_factor_runner)(scen)
|
||||
|
||||
qlib_factor_summarizer: HypothesisExperiment2Feedback = import_class(PROP_SETTING.qlib_factor_summarizer)()
|
||||
qlib_factor_summarizer: HypothesisExperiment2Feedback = import_class(PROP_SETTING.qlib_factor_summarizer)(scen)
|
||||
|
||||
|
||||
trace = Trace(scen=scen)
|
||||
@@ -38,4 +39,4 @@ for _ in range(PROP_SETTING.evolving_n):
|
||||
exp = qlib_factor_runner.generate(exp)
|
||||
feedback = qlib_factor_summarizer.generateFeedback(exp, hypothesis, trace)
|
||||
|
||||
trace.hist.append((hypothesis, exp, feedback))
|
||||
trace.hist.append((hypothesis, exp, feedback))
|
||||
@@ -25,7 +25,7 @@ hypothesis2experiment: Hypothesis2Experiment = import_class(PROP_SETTING.qlib_mo
|
||||
qlib_model_coder: TaskGenerator = import_class(PROP_SETTING.qlib_model_coder)(scen)
|
||||
qlib_model_runner: TaskGenerator = import_class(PROP_SETTING.qlib_model_runner)(scen)
|
||||
|
||||
qlib_model_summarizer: HypothesisExperiment2Feedback = import_class(PROP_SETTING.qlib_model_hypothesis2experiment)()
|
||||
qlib_model_summarizer: HypothesisExperiment2Feedback = import_class(PROP_SETTING.qlib_model_summarizer)()
|
||||
|
||||
trace = Trace(scen=scen)
|
||||
for _ in range(PROP_SETTING.evolving_n):
|
||||
|
||||
@@ -98,4 +98,5 @@ class FactorCoSTEER(TaskGenerator[FactorExperiment]):
|
||||
if self.new_knowledge_base_path is not None:
|
||||
pickle.dump(factor_knowledge_base, open(self.new_knowledge_base_path, "wb"))
|
||||
self.knowledge_base = factor_knowledge_base
|
||||
factor_experiment.based_experiments = exp.based_experiments
|
||||
return factor_experiment
|
||||
|
||||
@@ -84,7 +84,7 @@ class FactorCodeEvaluator(FactorEvaluator):
|
||||
gt_implementation: Implementation = None,
|
||||
**kwargs,
|
||||
):
|
||||
factor_information = target_task.get_factor_information()
|
||||
factor_information = target_task.get_task_information()
|
||||
code = implementation.code
|
||||
|
||||
system_prompt = (
|
||||
@@ -181,6 +181,36 @@ class FactorOutputFormatEvaluator(FactorEvaluator):
|
||||
)
|
||||
|
||||
|
||||
class FactorDatetimeDailyEvaluator(FactorEvaluator):
|
||||
def evaluate(
|
||||
self,
|
||||
implementation: Implementation,
|
||||
gt_implementation: Implementation,
|
||||
) -> Tuple[str | object]:
|
||||
_, gen_df = self._get_df(gt_implementation, implementation)
|
||||
if gen_df is None:
|
||||
return "The source dataframe is None. Skip the evaluation of the datetime format.", False
|
||||
|
||||
if "datetime" not in gen_df.index.names:
|
||||
return "The source dataframe does not have a datetime index. Please check the implementation.", False
|
||||
|
||||
try:
|
||||
pd.to_datetime(gen_df.index.get_level_values("datetime"))
|
||||
except Exception:
|
||||
return (
|
||||
"The source dataframe has a datetime index but it is not in the correct format (maybe a regular string or other objects). Please check the implementation.",
|
||||
False,
|
||||
)
|
||||
|
||||
time_diff = gen_df.index.get_level_values("datetime").to_series().diff().dropna().unique()
|
||||
if pd.Timedelta(minutes=1) in time_diff:
|
||||
return (
|
||||
"The generated dataframe is not daily. The implementation is definitely wrong. Please check the implementation.",
|
||||
False,
|
||||
)
|
||||
return "The generated dataframe is daily.", True
|
||||
|
||||
|
||||
class FactorRowCountEvaluator(FactorEvaluator):
|
||||
def evaluate(
|
||||
self,
|
||||
@@ -314,6 +344,9 @@ class FactorValueEvaluator(FactorEvaluator):
|
||||
feedback_str, _ = FactorOutputFormatEvaluator(self.scen).evaluate(implementation, gt_implementation)
|
||||
conclusions.append(feedback_str)
|
||||
|
||||
feedback_str, _ = FactorDatetimeDailyEvaluator(self.scen).evaluate(implementation, gt_implementation)
|
||||
conclusions.append(feedback_str)
|
||||
|
||||
# Check if both dataframe have the same rows count
|
||||
if gt_implementation is not None:
|
||||
feedback_str, _ = FactorRowCountEvaluator(self.scen).evaluate(implementation, gt_implementation)
|
||||
@@ -373,7 +406,7 @@ class FactorFinalDecisionEvaluator(Evaluator):
|
||||
evaluate_prompts["evaluator_final_decision_v1_user"],
|
||||
)
|
||||
.render(
|
||||
factor_information=target_task.get_factor_information(),
|
||||
factor_information=target_task.get_task_information(),
|
||||
execution_feedback=execution_feedback_to_render,
|
||||
code_feedback=code_feedback,
|
||||
factor_value_feedback=(
|
||||
@@ -475,7 +508,7 @@ class FactorEvaluatorForCoder(FactorEvaluator):
|
||||
if implementation is None:
|
||||
return None
|
||||
|
||||
target_task_information = target_task.get_factor_information()
|
||||
target_task_information = target_task.get_task_information()
|
||||
if (
|
||||
queried_knowledge is not None
|
||||
and target_task_information in queried_knowledge.success_task_to_knowledge_dict
|
||||
|
||||
@@ -59,7 +59,7 @@ class MultiProcessEvolvingStrategy(EvolvingStrategy):
|
||||
# 1.找出需要evolve的factor
|
||||
to_be_finished_task_index = []
|
||||
for index, target_factor_task in enumerate(new_evo.sub_tasks):
|
||||
target_factor_task_desc = target_factor_task.get_factor_information()
|
||||
target_factor_task_desc = target_factor_task.get_task_information()
|
||||
if target_factor_task_desc in queried_knowledge.success_task_to_knowledge_dict:
|
||||
new_evo.sub_implementations[index] = queried_knowledge.success_task_to_knowledge_dict[
|
||||
target_factor_task_desc
|
||||
@@ -119,7 +119,7 @@ class FactorEvolvingStrategy(MultiProcessEvolvingStrategy):
|
||||
target_task: FactorTask,
|
||||
queried_knowledge: FactorQueriedKnowledgeV1 = None,
|
||||
) -> Implementation:
|
||||
factor_information_str = target_task.get_factor_information()
|
||||
factor_information_str = target_task.get_task_information()
|
||||
|
||||
if queried_knowledge is not None and factor_information_str in queried_knowledge.success_task_to_knowledge_dict:
|
||||
return queried_knowledge.success_task_to_knowledge_dict[factor_information_str].implementation
|
||||
@@ -208,7 +208,7 @@ class FactorEvolvingStrategyWithGraph(MultiProcessEvolvingStrategy):
|
||||
) -> Implementation:
|
||||
error_summary = FACTOR_IMPLEMENT_SETTINGS.v2_error_summary
|
||||
# 1. 提取因子的背景信息
|
||||
target_factor_task_information = target_task.get_factor_information()
|
||||
target_factor_task_information = target_task.get_task_information()
|
||||
|
||||
# 2. 检查该因子是否需要继续做（是否已经作对，是否做错太多）
|
||||
if (
|
||||
|
||||
@@ -114,7 +114,7 @@ class FactorRAGStrategyV1(RAGStrategy):
|
||||
feedback = evo_step.feedback
|
||||
for task_index in range(len(implementations.sub_tasks)):
|
||||
target_task = implementations.sub_tasks[task_index]
|
||||
target_task_information = target_task.get_factor_information()
|
||||
target_task_information = target_task.get_task_information()
|
||||
implementation = implementations.sub_implementations[task_index]
|
||||
single_feedback = feedback[task_index]
|
||||
if single_feedback is None:
|
||||
@@ -147,7 +147,7 @@ class FactorRAGStrategyV1(RAGStrategy):
|
||||
|
||||
queried_knowledge = FactorQueriedKnowledgeV1()
|
||||
for target_factor_task in evo.sub_tasks:
|
||||
target_factor_task_information = target_factor_task.get_factor_information()
|
||||
target_factor_task_information = target_factor_task.get_task_information()
|
||||
if target_factor_task_information in self.knowledgebase.success_task_info_set:
|
||||
queried_knowledge.success_task_to_knowledge_dict[target_factor_task_information] = (
|
||||
self.knowledgebase.implementation_trace[target_factor_task_information][-1]
|
||||
@@ -233,7 +233,7 @@ class FactorGraphRAGStrategy(RAGStrategy):
|
||||
for task_index in range(len(implementations.sub_tasks)):
|
||||
single_feedback = feedback[task_index]
|
||||
target_task = implementations.sub_tasks[task_index]
|
||||
target_task_information = target_task.get_factor_information()
|
||||
target_task_information = target_task.get_task_information()
|
||||
implementation = implementations.sub_implementations[task_index]
|
||||
single_feedback = feedback[task_index]
|
||||
if single_feedback is None:
|
||||
@@ -395,7 +395,7 @@ class FactorGraphRAGStrategy(RAGStrategy):
|
||||
fail_task_trial_limit = FACTOR_IMPLEMENT_SETTINGS.fail_task_trial_limit
|
||||
|
||||
for target_factor_task in evo.sub_tasks:
|
||||
target_factor_task_information = target_factor_task.get_factor_information()
|
||||
target_factor_task_information = target_factor_task.get_task_information()
|
||||
if (
|
||||
target_factor_task_information not in self.knowledgebase.success_task_to_knowledge_dict
|
||||
and target_factor_task_information in self.knowledgebase.working_trace_knowledge
|
||||
@@ -442,7 +442,7 @@ class FactorGraphRAGStrategy(RAGStrategy):
|
||||
) -> QueriedKnowledge | None:
|
||||
# queried_component_knowledge = FactorQueriedGraphComponentKnowledge()
|
||||
for target_factor_task in evo.sub_tasks:
|
||||
target_factor_task_information = target_factor_task.get_factor_information()
|
||||
target_factor_task_information = target_factor_task.get_task_information()
|
||||
if (
|
||||
target_factor_task_information in self.knowledgebase.success_task_to_knowledge_dict
|
||||
or target_factor_task_information in factor_implementation_queried_graph_knowledge.failed_task_info_set
|
||||
@@ -582,7 +582,7 @@ class FactorGraphRAGStrategy(RAGStrategy):
|
||||
) -> QueriedKnowledge | None:
|
||||
# queried_error_knowledge = FactorQueriedGraphErrorKnowledge()
|
||||
for task_index, target_factor_task in enumerate(evo.sub_tasks):
|
||||
target_factor_task_information = target_factor_task.get_factor_information()
|
||||
target_factor_task_information = target_factor_task.get_task_information()
|
||||
factor_implementation_queried_graph_knowledge.error_with_success_task[target_factor_task_information] = {}
|
||||
if (
|
||||
target_factor_task_information in self.knowledgebase.success_task_to_knowledge_dict
|
||||
|
||||
@@ -39,7 +39,7 @@ def LLMSelect(
|
||||
tasks = []
|
||||
for i in to_be_finished_task_index:
|
||||
# find corresponding former trace for each task
|
||||
target_factor_task_information = evo.sub_tasks[i].get_factor_information()
|
||||
target_factor_task_information = evo.sub_tasks[i].get_task_information()
|
||||
if target_factor_task_information in former_trace:
|
||||
tasks.append((i, evo.sub_tasks[i], former_trace[target_factor_task_information]))
|
||||
|
||||
|
||||
@@ -7,13 +7,16 @@ SELECT_METHOD = Literal["random", "scheduler"]
|
||||
|
||||
|
||||
class FactorImplementSettings(BaseSettings):
|
||||
file_based_execution_data_folder: str = str(
|
||||
factor_data_folder: str = str(
|
||||
(Path().cwd() / "git_ignore_folder" / "factor_implementation_source_data").absolute(),
|
||||
)
|
||||
file_based_execution_workspace: str = str(
|
||||
factor_data_folder_debug: str = str(
|
||||
(Path().cwd() / "git_ignore_folder" / "factor_implementation_source_data_debug").absolute(),
|
||||
)
|
||||
factor_execution_workspace: str = str(
|
||||
(Path().cwd() / "git_ignore_folder" / "factor_implementation_workspace").absolute(),
|
||||
)
|
||||
implementation_execution_cache_location: str = str(
|
||||
factor_cache_location: str = str(
|
||||
(Path().cwd() / "git_ignore_folder" / "factor_implementation_execution_cache").absolute(),
|
||||
)
|
||||
enable_execution_cache: bool = True # whether to enable the execution cache
|
||||
|
||||
@@ -37,7 +37,7 @@ class FactorTask(Task):
|
||||
self.variables = variables
|
||||
self.factor_resources = resource
|
||||
|
||||
def get_factor_information(self):
|
||||
def get_task_information(self):
|
||||
return f"""factor_name: {self.factor_name}
|
||||
factor_description: {self.factor_description}
|
||||
factor_formulation: {self.factor_formulation}
|
||||
@@ -95,11 +95,11 @@ class FileBasedFactorImplementation(FBImplementation):
|
||||
|
||||
def prepare(self, *args, **kwargs):
|
||||
self.workspace_path = Path(
|
||||
FACTOR_IMPLEMENT_SETTINGS.file_based_execution_workspace,
|
||||
FACTOR_IMPLEMENT_SETTINGS.factor_execution_workspace,
|
||||
) / str(uuid.uuid4())
|
||||
self.workspace_path.mkdir(exist_ok=True, parents=True)
|
||||
|
||||
def execute(self, store_result: bool = False) -> Tuple[str, pd.DataFrame]:
|
||||
def execute(self, store_result: bool = False, data_type: str = "Debug") -> Tuple[str, pd.DataFrame]:
|
||||
"""
|
||||
execute the implementation and get the factor value by the following steps:
|
||||
1. make the directory in workspace path
|
||||
@@ -120,14 +120,10 @@ class FileBasedFactorImplementation(FBImplementation):
|
||||
raise ValueError(self.FB_CODE_NOT_SET)
|
||||
with FileLock(self.workspace_path / "execution.lock"):
|
||||
if FACTOR_IMPLEMENT_SETTINGS.enable_execution_cache:
|
||||
# NOTE: cache the result for the same code
|
||||
target_file_name = md5_hash(self.code_dict["factor.py"])
|
||||
cache_file_path = (
|
||||
Path(FACTOR_IMPLEMENT_SETTINGS.implementation_execution_cache_location) / f"{target_file_name}.pkl"
|
||||
)
|
||||
Path(FACTOR_IMPLEMENT_SETTINGS.implementation_execution_cache_location).mkdir(
|
||||
exist_ok=True, parents=True
|
||||
)
|
||||
# NOTE: cache the result for the same code and same data type
|
||||
target_file_name = md5_hash(data_type + self.code_dict["factor.py"])
|
||||
cache_file_path = Path(FACTOR_IMPLEMENT_SETTINGS.factor_cache_location) / f"{target_file_name}.pkl"
|
||||
Path(FACTOR_IMPLEMENT_SETTINGS.factor_cache_location).mkdir(exist_ok=True, parents=True)
|
||||
if cache_file_path.exists() and not self.raise_exception:
|
||||
cached_res = pickle.load(open(cache_file_path, "rb"))
|
||||
if store_result and cached_res[1] is not None:
|
||||
@@ -137,8 +133,14 @@ class FileBasedFactorImplementation(FBImplementation):
|
||||
if self.executed_factor_value_dataframe is not None:
|
||||
return self.FB_FROM_CACHE, self.executed_factor_value_dataframe
|
||||
|
||||
source_data_path = Path(
|
||||
FACTOR_IMPLEMENT_SETTINGS.file_based_execution_data_folder,
|
||||
source_data_path = (
|
||||
Path(
|
||||
FACTOR_IMPLEMENT_SETTINGS.factor_data_folder_debug,
|
||||
)
|
||||
if data_type == "Debug"
|
||||
else Path(
|
||||
FACTOR_IMPLEMENT_SETTINGS.factor_data_folder,
|
||||
)
|
||||
)
|
||||
|
||||
source_data_path.mkdir(exist_ok=True, parents=True)
|
||||
|
||||
@@ -68,7 +68,7 @@ evolving_strategy_factor_implementation_v1_user: |-
|
||||
--------------Correct code to similar factors:---------------
|
||||
{% for similar_successful_knowledge in queried_similar_successful_knowledge %}
|
||||
=====Factor {{loop.index}}:=====
|
||||
{{ similar_successful_knowledge.target_task.get_factor_information() }}
|
||||
{{ similar_successful_knowledge.target_task.get_task_information() }}
|
||||
=====Code:=====
|
||||
{{ similar_successful_knowledge.implementation.code }}
|
||||
{% endfor %}
|
||||
@@ -94,7 +94,7 @@ evolving_strategy_factor_implementation_v2_user: |-
|
||||
When doing other tasks, you met some similar errors but you finally solve them. Here are some examples:
|
||||
{% for error_content, similar_error_knowledge in queried_similar_error_knowledge %}
|
||||
--------------Factor information to similar error ({{error_content}}):---------------
|
||||
{{ similar_error_knowledge[0].target_task.get_factor_information() }}
|
||||
{{ similar_error_knowledge[0].target_task.get_task_information() }}
|
||||
=====Code with similar error ({{error_content}}):=====
|
||||
{{ similar_error_knowledge[0].implementation.code }}
|
||||
=====Success code to former code with similar error ({{error_content}}):=====
|
||||
@@ -111,7 +111,7 @@ evolving_strategy_factor_implementation_v2_user: |-
|
||||
--------------Correct code to similar factors:---------------
|
||||
{% for similar_component_knowledge in queried_similar_component_knowledge %}
|
||||
=====Factor {{loop.index}}:=====
|
||||
{{ similar_component_knowledge.target_task.get_factor_information() }}
|
||||
{{ similar_component_knowledge.target_task.get_task_information() }}
|
||||
=====Code:=====
|
||||
{{ similar_component_knowledge.implementation.code }}
|
||||
{% endfor %}
|
||||
@@ -137,7 +137,7 @@ evolving_strategy_error_summary_v2_user: |-
|
||||
{% if queried_similar_error_knowledge|length != 0 %}
|
||||
{% for error_content, similar_error_knowledge in queried_similar_error_knowledge %}
|
||||
--------------Factor information to similar error ({{error_content}}):---------------
|
||||
{{ similar_error_knowledge[0].target_task.get_factor_information() }}
|
||||
{{ similar_error_knowledge[0].target_task.get_task_information() }}
|
||||
=====Code with similar error ({{error_content}}):=====
|
||||
{{ similar_error_knowledge[0].implementation.code }}
|
||||
=====Success code to former code with similar error ({{error_content}}):=====
|
||||
|
||||
@@ -22,7 +22,7 @@ def get_data_folder_intro():
|
||||
It is for preparing prompting message.
|
||||
"""
|
||||
content_l = []
|
||||
for p in Path(FACTOR_IMPLEMENT_SETTINGS.file_based_execution_data_folder).iterdir():
|
||||
for p in Path(FACTOR_IMPLEMENT_SETTINGS.factor_data_folder).iterdir():
|
||||
if p.name.endswith(".h5"):
|
||||
df = pd.read_hdf(p)
|
||||
# get df.head() as string with full width
|
||||
|
||||
@@ -83,4 +83,5 @@ class ModelCoSTEER(TaskGenerator[ModelExperiment]):
|
||||
if self.new_knowledge_base_path is not None:
|
||||
pickle.dump(model_knowledge_base, open(self.new_knowledge_base_path, "wb"))
|
||||
self.knowledge_base = model_knowledge_base
|
||||
model_experiment.based_experiments = exp.based_experiments
|
||||
return model_experiment
|
||||
|
||||
@@ -72,7 +72,7 @@ class ModelCodeEvaluator(Evaluator):
|
||||
if gt_implementation is not None:
|
||||
assert isinstance(gt_implementation, ModelImplementation)
|
||||
|
||||
model_task_information = target_task.get_information()
|
||||
model_task_information = target_task.get_task_information()
|
||||
code = implementation.code
|
||||
|
||||
system_prompt = (
|
||||
@@ -146,7 +146,7 @@ class ModelFinalEvaluator(Evaluator):
|
||||
evaluate_prompts["evaluator_final_feedback"]["user"],
|
||||
)
|
||||
.render(
|
||||
model_information=target_task.get_information(),
|
||||
model_information=target_task.get_task_information(),
|
||||
model_execution_feedback=execution_feedback_to_render,
|
||||
model_code_feedback=model_code_feedback,
|
||||
model_value_feedback=model_value_feedback,
|
||||
@@ -224,7 +224,7 @@ class ModelCoderEvaluator(Evaluator):
|
||||
queried_knowledge: QueriedKnowledge = None,
|
||||
**kwargs,
|
||||
) -> ModelCoderFeedback:
|
||||
target_task_information = target_task.get_information()
|
||||
target_task_information = target_task.get_task_information()
|
||||
if (
|
||||
queried_knowledge is not None
|
||||
and target_task_information in queried_knowledge.success_task_to_knowledge_dict
|
||||
|
||||
@@ -27,7 +27,7 @@ class ModelCoderEvolvingStrategy(EvolvingStrategy):
|
||||
target_task: ModelTask,
|
||||
queried_knowledge: ModelQueriedKnowledge = None,
|
||||
) -> ModelImplementation:
|
||||
model_information_str = target_task.get_information()
|
||||
model_information_str = target_task.get_task_information()
|
||||
|
||||
if queried_knowledge is not None and model_information_str in queried_knowledge.success_task_to_knowledge_dict:
|
||||
return queried_knowledge.success_task_to_knowledge_dict[model_information_str].implementation
|
||||
@@ -113,7 +113,7 @@ class ModelCoderEvolvingStrategy(EvolvingStrategy):
|
||||
# 1.找出需要evolve的model
|
||||
to_be_finished_task_index = []
|
||||
for index, target_model_task in enumerate(new_evo.sub_tasks):
|
||||
target_model_task_desc = target_model_task.get_information()
|
||||
target_model_task_desc = target_model_task.get_task_information()
|
||||
if target_model_task_desc in queried_knowledge.success_task_to_knowledge_dict:
|
||||
new_evo.sub_implementations[index] = queried_knowledge.success_task_to_knowledge_dict[
|
||||
target_model_task_desc
|
||||
|
||||
@@ -86,7 +86,7 @@ class ModelRAGStrategy(RAGStrategy):
|
||||
feedback = evo_step.feedback
|
||||
for task_index in range(len(implementations.sub_tasks)):
|
||||
target_task = implementations.sub_tasks[task_index]
|
||||
target_task_information = target_task.get_information()
|
||||
target_task_information = target_task.get_task_information()
|
||||
implementation = implementations.sub_implementations[task_index]
|
||||
single_feedback = feedback[task_index]
|
||||
if single_feedback is None:
|
||||
@@ -119,7 +119,7 @@ class ModelRAGStrategy(RAGStrategy):
|
||||
|
||||
queried_knowledge = ModelQueriedKnowledge()
|
||||
for target_model_task in evo.sub_tasks:
|
||||
target_model_task_information = target_model_task.get_information()
|
||||
target_model_task_information = target_model_task.get_task_information()
|
||||
if target_model_task_information in self.knowledgebase.success_task_info_set:
|
||||
queried_knowledge.success_task_to_knowledge_dict[target_model_task_information] = (
|
||||
self.knowledgebase.implementation_trace[target_model_task_information][-1]
|
||||
|
||||
@@ -8,10 +8,10 @@ class ModelImplSettings(BaseSettings):
|
||||
class Config:
|
||||
env_prefix = "MODEL_IMPL_" # Use MODEL_IMPL_ as prefix for environment variables
|
||||
|
||||
file_based_execution_workspace: str = str(
|
||||
model_execution_workspace: str = str(
|
||||
(Path().cwd() / "git_ignore_folder" / "model_implementation_workspace").absolute(),
|
||||
)
|
||||
implementation_execution_cache_location: str = str(
|
||||
model_cache_location: str = str(
|
||||
(Path().cwd() / "git_ignore_folder" / "model_implementation_execution_cache").absolute(),
|
||||
)
|
||||
|
||||
|
||||
@@ -24,7 +24,7 @@ class ModelTask(Task):
|
||||
self.variables: str = variables
|
||||
self.model_type: str = model_type # Tabular for tabular model, TimesSeries for time series model
|
||||
|
||||
def get_information(self):
|
||||
def get_task_information(self):
|
||||
return f"""name: {self.name}
|
||||
description: {self.description}
|
||||
formulation: {self.formulation}
|
||||
@@ -68,7 +68,7 @@ class ModelImplementation(FBImplementation):
|
||||
Prepare for the workspace;
|
||||
"""
|
||||
unique_id = uuid.uuid4()
|
||||
self.workspace_path = Path(MODEL_IMPL_SETTINGS.file_based_execution_workspace) / f"M{unique_id}"
|
||||
self.workspace_path = Path(MODEL_IMPL_SETTINGS.model_execution_workspace) / f"M{unique_id}"
|
||||
# start with `M` so that it can be imported via python
|
||||
self.workspace_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
@@ -84,10 +84,8 @@ class ModelImplementation(FBImplementation):
|
||||
if MODEL_IMPL_SETTINGS.enable_execution_cache:
|
||||
# NOTE: cache the result for the same code
|
||||
target_file_name = md5_hash(self.code_dict["model.py"])
|
||||
cache_file_path = (
|
||||
Path(MODEL_IMPL_SETTINGS.implementation_execution_cache_location) / f"{target_file_name}.pkl"
|
||||
)
|
||||
Path(MODEL_IMPL_SETTINGS.implementation_execution_cache_location).mkdir(exist_ok=True, parents=True)
|
||||
cache_file_path = Path(MODEL_IMPL_SETTINGS.model_cache_location) / f"{target_file_name}.pkl"
|
||||
Path(MODEL_IMPL_SETTINGS.model_cache_location).mkdir(exist_ok=True, parents=True)
|
||||
if cache_file_path.exists():
|
||||
return pickle.load(open(cache_file_path, "rb"))
|
||||
mod = get_module_by_module_path(str(self.workspace_path / "model.py"))
|
||||
|
||||
@@ -7,17 +7,27 @@ This file contains the all the class about organizing the task in RD-Agent.
|
||||
"""
|
||||
|
||||
|
||||
class Task:
|
||||
class Task(ABC):
|
||||
# TODO: 把name放在这里作为主键
|
||||
# Please refer to rdagent/model_implementation/task.py for the implementation
|
||||
# I think the task version applies to the base class.
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def get_task_information(self):
|
||||
"""
|
||||
Get the task information string to build the unique key
|
||||
"""
|
||||
pass
|
||||
|
||||
|
||||
ASpecificTask = TypeVar("ASpecificTask", bound=Task)
|
||||
|
||||
|
||||
class Implementation(ABC, Generic[ASpecificTask]):
|
||||
# TODO: workspace;
|
||||
# - code or data(optional)
|
||||
# - Execute logic
|
||||
# - `env is not included`. It is a underlying infra
|
||||
def __init__(self, target_task: ASpecificTask) -> None:
|
||||
self.target_task = target_task
|
||||
|
||||
@@ -85,6 +95,7 @@ class FBImplementation(Implementation):
|
||||
typical usage of `*args, **kwargs`:
|
||||
Different methods shares the same data. The data are passed by the arguments.
|
||||
"""
|
||||
# TODO: model and factor prepare;
|
||||
|
||||
def inject_code(self, **files: str):
|
||||
"""
|
||||
@@ -113,11 +124,14 @@ class Experiment(ABC, Generic[ASpecificTask, ASpecificImp]):
|
||||
The experiment is a sequence of tasks and the implementations of the tasks after generated by the TaskGenerator.
|
||||
"""
|
||||
|
||||
result_ws: Optional[FBImplementation]
|
||||
|
||||
def __init__(self, sub_tasks: Sequence[ASpecificTask]) -> None:
|
||||
self.sub_tasks = sub_tasks
|
||||
self.sub_implementations: Sequence[ASpecificImp] = [None for _ in self.sub_tasks]
|
||||
self.based_experiments: Sequence[Experiment] = []
|
||||
self.result: object = None # The result of the experiment, can be different types in different scenarios.
|
||||
self.result_ws = None
|
||||
|
||||
|
||||
TaskOrExperiment = TypeVar("TaskOrExperiment", Task, Experiment)
|
||||
|
||||
@@ -3,10 +3,10 @@
|
||||
"""
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Dict, Generic, List, Tuple, TypeVar
|
||||
from typing import Any, Dict, Generic, List, Tuple, TypeVar
|
||||
|
||||
from rdagent.core.evaluation import Feedback
|
||||
from rdagent.core.experiment import Experiment
|
||||
from rdagent.core.experiment import ASpecificTask, Experiment
|
||||
from rdagent.core.scenario import Scenario
|
||||
|
||||
# class data_ana: XXX
|
||||
@@ -101,9 +101,12 @@ class Hypothesis2Experiment(ABC, Generic[ASpecificExp]):
|
||||
class HypothesisExperiment2Feedback:
|
||||
""" "Generated feedbacks on the hypothesis from **Executed** Implementations of different tasks & their comparisons with previous performances"""
|
||||
|
||||
def generateFeedback(self, ti: Experiment, hypothesis: Hypothesis, trace: Trace) -> HypothesisFeedback:
|
||||
def __init__(self, scen: Scenario):
|
||||
self.scen = scen
|
||||
|
||||
def generateFeedback(self, exp: Experiment, hypothesis: Hypothesis, trace: Trace) -> HypothesisFeedback:
|
||||
"""
|
||||
The `ti` should be executed and the results should be included, as well as the comparison between previous results (done by LLM).
|
||||
The `exp` should be executed and the results should be included, as well as the comparison between previous results (done by LLM).
|
||||
For example: `mlflow` of Qlib will be included.
|
||||
"""
|
||||
raise NotImplementedError("generateFeedback method is not implemented.")
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from dotenv import load_dotenv
|
||||
from pydantic_settings import BaseSettings
|
||||
|
||||
# make sure that env variable is loaded while calling Config()
|
||||
load_dotenv(verbose=True, override=True)
|
||||
|
||||
from pydantic_settings import BaseSettings
|
||||
|
||||
|
||||
class QlibRDAgentSettings(BaseSettings):
|
||||
runner_cache_result: bool = True # whether to cache the result of the docker execution
|
||||
runner_cache_path: str = str(Path.cwd() / "runner_cache/") # the path to store the cache
|
||||
|
||||
|
||||
Qlib_RD_AGENT_SETTINGS = QlibRDAgentSettings()
|
||||
@@ -0,0 +1,23 @@
|
||||
FROM pytorch/pytorch:latest
|
||||
|
||||
# For GPU support, please choose the proper tag from https://hub.docker.com/r/pytorch/pytorch/tags
|
||||
|
||||
RUN apt-get clean && apt-get update && apt-get install -y \
|
||||
curl \
|
||||
vim \
|
||||
git \
|
||||
build-essential \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
RUN git clone https://github.com/microsoft/qlib.git
|
||||
|
||||
WORKDIR /workspace/qlib
|
||||
|
||||
RUN git reset c9ed050ef034fe6519c14b59f3d207abcb693282 --hard
|
||||
|
||||
RUN python -m pip install --upgrade numpy
|
||||
RUN python -m pip install --upgrade cython
|
||||
RUN python -m pip install -e .
|
||||
|
||||
RUN pip install catboost
|
||||
RUN pip install xgboost
|
||||
@@ -77,4 +77,6 @@ class QlibFactorHypothesis2Experiment(FactorHypothesis2Experiment):
|
||||
tasks.append(FactorTask(factor_name, description, formulation, variables))
|
||||
exp = FactorExperiment(tasks)
|
||||
exp.based_experiments = [t[1] for t in trace.hist if t[2]]
|
||||
if len(exp.based_experiments) == 0:
|
||||
exp.based_experiments.append(FactorExperiment(sub_tasks=[]))
|
||||
return exp
|
||||
|
||||
@@ -47,4 +47,36 @@ model_experiment_output_format: |-
|
||||
"model_type": "type of model 1, Tabular or TimesSeries" # Should be one of "Tabular" or "TimeSeries"
|
||||
}
|
||||
# Don't add ellipsis (...) or any filler text that might cause JSON parsing errors here!
|
||||
}
|
||||
}
|
||||
|
||||
data_feedback_generation:
|
||||
system: |-
|
||||
You are a professional result analysis assistant on data driven R&D.
|
||||
The task is described in the following scenario:
|
||||
{{ scenario }}
|
||||
You will receive a hypothesis, multiple tasks with their factors, and some results.
|
||||
Your feedback should specify whether the current result supports or refutes the hypothesis, compare it with previous results, and suggest improvements or new directions.
|
||||
Please provide detailed and constructive feedback for the future exploration.
|
||||
Please respond in JSON format, and example JSON Structure for Result Analysis:
|
||||
{
|
||||
"Observations": "Your overall observations here",
|
||||
"Feedback for Hypothesis": "Observations related to the hypothesis",
|
||||
"New Hypothesis": "Put your new hypothesis here.",
|
||||
"Reasoning": "Provide reasoning for the hypothesis here.",
|
||||
"Replace Best Result": "yes or no"
|
||||
}
|
||||
user: |-
|
||||
Target hypothesis:
|
||||
{{ hypothesis_text }}
|
||||
Tasks and Factors:
|
||||
{{ task_details }}
|
||||
Current Result:
|
||||
{{ current_result }}
|
||||
SOTA Result:
|
||||
{{ sota_result }}
|
||||
Analyze the current result in the context of its ability to:
|
||||
1. Support or refute the hypothesis.
|
||||
2. Show improvement or deterioration compared to the last experiment.
|
||||
3. Demonstrate positive or negative effects when compared to Alpha158.
|
||||
|
||||
Provide detailed feedback and recommend whether to replace the best result if the new factor proves superior.
|
||||
|
||||
@@ -1,5 +1,33 @@
|
||||
import pickle
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
from typing import List, Tuple
|
||||
|
||||
import pandas as pd
|
||||
|
||||
from rdagent.core.log import RDAgentLog
|
||||
from rdagent.core.task_generator import TaskGenerator
|
||||
from rdagent.oai.llm_utils import md5_hash
|
||||
from rdagent.scenarios.qlib.conf import Qlib_RD_AGENT_SETTINGS
|
||||
from rdagent.scenarios.qlib.experiment.factor_experiment import QlibFactorExperiment
|
||||
from rdagent.utils.env import QTDockerEnv
|
||||
|
||||
DIRNAME = Path(__file__).absolute().resolve().parent
|
||||
DIRNAME_local = Path.cwd()
|
||||
logger = RDAgentLog()
|
||||
|
||||
# class QlibFactorExpWorkspace:
|
||||
|
||||
# def prepare():
|
||||
# # create a folder;
|
||||
# # copy template
|
||||
# # place data inside the folder `combined_factors`
|
||||
# #
|
||||
# def execute():
|
||||
# de = DockerEnv()
|
||||
# de.run(local_path=self.ws_path, entry="qrun conf.yaml")
|
||||
|
||||
# TODO: supporting multiprocessing and keep previous results
|
||||
|
||||
|
||||
class QlibFactorRunner(TaskGenerator[QlibFactorExperiment]):
|
||||
@@ -10,9 +38,138 @@ class QlibFactorRunner(TaskGenerator[QlibFactorExperiment]):
|
||||
- price-volume data dumper
|
||||
- `data.py` + Adaptor to Factor implementation
|
||||
- results in `mlflow`
|
||||
|
||||
- TODO: implement a qlib handler
|
||||
"""
|
||||
|
||||
def get_cache_key(self, exp: QlibFactorExperiment) -> str:
|
||||
all_tasks = []
|
||||
for based_exp in exp.based_experiments:
|
||||
all_tasks.extend(based_exp.sub_tasks)
|
||||
all_tasks.extend(exp.sub_tasks)
|
||||
task_info_list = [task.get_task_information() for task in all_tasks]
|
||||
task_info_str = "\n".join(task_info_list)
|
||||
return md5_hash(task_info_str)
|
||||
|
||||
def get_cache_result(self, exp: QlibFactorExperiment) -> Tuple[bool, object]:
|
||||
task_info_key = self.get_cache_key(exp)
|
||||
Path(Qlib_RD_AGENT_SETTINGS.runner_cache_path).mkdir(parents=True, exist_ok=True)
|
||||
cache_path = Path(Qlib_RD_AGENT_SETTINGS.runner_cache_path) / f"{task_info_key}.pkl"
|
||||
if cache_path.exists():
|
||||
return True, pickle.load(open(cache_path, "rb"))
|
||||
else:
|
||||
return False, None
|
||||
|
||||
def dump_cache_result(self, exp: QlibFactorExperiment, result: object):
|
||||
task_info_key = self.get_cache_key(exp)
|
||||
cache_path = Path(Qlib_RD_AGENT_SETTINGS.runner_cache_path) / f"{task_info_key}.pkl"
|
||||
pickle.dump(result, open(cache_path, "wb"))
|
||||
|
||||
def generate(self, exp: QlibFactorExperiment) -> QlibFactorExperiment:
|
||||
return exp # TODO IMPLEMENT THIS
|
||||
"""
|
||||
Generate the experiment by processing and combining factor data,
|
||||
then passing the combined data to Docker for backtest results.
|
||||
"""
|
||||
if exp.based_experiments and exp.based_experiments[-1].result is None:
|
||||
exp.based_experiments[-1] = self.generate(exp.based_experiments[-1])
|
||||
|
||||
if Qlib_RD_AGENT_SETTINGS.runner_cache_result:
|
||||
cache_hit, result = self.get_cache_result(exp)
|
||||
if cache_hit:
|
||||
exp.result = result
|
||||
return exp
|
||||
|
||||
if exp.based_experiments:
|
||||
SOTA_factor = None
|
||||
if exp.based_experiments.__len__() != 1:
|
||||
SOTA_factor = self.process_factor_data(exp.based_experiments)
|
||||
|
||||
# Process the new factors data
|
||||
new_factors = self.process_factor_data(exp)
|
||||
|
||||
# Combine the SOTA factor and new factors if SOTA factor exists
|
||||
if SOTA_factor is not None and not SOTA_factor.empty:
|
||||
combined_factors = pd.concat([SOTA_factor, new_factors], axis=1).dropna()
|
||||
else:
|
||||
combined_factors = new_factors
|
||||
|
||||
# Sort and nest the combined factors under 'feature'
|
||||
combined_factors = combined_factors.sort_index()
|
||||
new_columns = pd.MultiIndex.from_product([["feature"], combined_factors.columns])
|
||||
combined_factors.columns = new_columns
|
||||
|
||||
# Save the combined factors to a pickle file
|
||||
combined_factors_path = DIRNAME / "env_factor/combined_factors_df.pkl"
|
||||
with open(combined_factors_path, "wb") as f:
|
||||
pickle.dump(combined_factors, f)
|
||||
|
||||
# Docker run
|
||||
# Call Docker, pass the combined factors to Docker, and generate backtest results
|
||||
qtde = QTDockerEnv()
|
||||
qtde.prepare()
|
||||
|
||||
# Run the Docker command
|
||||
execute_log = qtde.run(local_path=str(DIRNAME / "env_factor"), entry="rm -r mlruns")
|
||||
# Run the Qlib backtest
|
||||
execute_log = qtde.run(
|
||||
local_path=str(DIRNAME / "env_factor"),
|
||||
entry=f"qrun conf.yaml" if len(exp.based_experiments) == 0 else "qrun conf_combined.yaml",
|
||||
)
|
||||
|
||||
execute_log = qtde.run(local_path=str(DIRNAME / "env_factor"), entry="python read_exp_res.py")
|
||||
|
||||
pkl_path = DIRNAME / "env_factor/qlib_res.pkl"
|
||||
|
||||
if not pkl_path.exists():
|
||||
logger.error(f"File {pkl_path} does not exist.")
|
||||
return None
|
||||
|
||||
with open(pkl_path, "rb") as f:
|
||||
result = pickle.load(f)
|
||||
|
||||
exp.result = result
|
||||
if Qlib_RD_AGENT_SETTINGS.runner_cache_result:
|
||||
self.dump_cache_result(exp, result)
|
||||
|
||||
# Check if the result is valid and is a DataFrame
|
||||
if isinstance(result, pd.DataFrame):
|
||||
if not result.empty:
|
||||
logger.info("Successfully retrieved experiment result.")
|
||||
return exp
|
||||
else:
|
||||
logger.error("Result DataFrame is empty.")
|
||||
return None
|
||||
else:
|
||||
logger.error("Data format error.")
|
||||
return None
|
||||
|
||||
def process_factor_data(self, exp_or_list: List[QlibFactorExperiment] | QlibFactorExperiment) -> pd.DataFrame:
|
||||
"""
|
||||
Process and combine factor data from experiment implementations.
|
||||
|
||||
Args:
|
||||
exp (ASpecificExp): The experiment containing factor data.
|
||||
|
||||
Returns:
|
||||
pd.DataFrame: Combined factor data without NaN values.
|
||||
"""
|
||||
if isinstance(exp_or_list, QlibFactorExperiment):
|
||||
exp_or_list = [exp_or_list]
|
||||
factor_dfs = []
|
||||
|
||||
# Collect all exp's dataframes
|
||||
for exp in exp_or_list:
|
||||
# Iterate over sub-implementations and execute them to get each factor data
|
||||
for implementation in exp.sub_implementations:
|
||||
message, df = implementation.execute(data_type="All")
|
||||
|
||||
# Check if factor generation was successful
|
||||
if df is not None:
|
||||
time_diff = df.index.get_level_values("datetime").to_series().diff().dropna().unique()
|
||||
if pd.Timedelta(minutes=1) not in time_diff:
|
||||
factor_dfs.append(df)
|
||||
|
||||
# Combine all successful factor data
|
||||
if factor_dfs:
|
||||
return pd.concat(factor_dfs, axis=1)
|
||||
else:
|
||||
logger.error("No valid factor data found to merge.")
|
||||
return pd.DataFrame() # Return an empty DataFrame if no valid data
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
qlib_init:
|
||||
provider_uri: "~/.qlib/qlib_data/cn_data"
|
||||
region: cn
|
||||
|
||||
market: &market csi300
|
||||
benchmark: &benchmark SH000300
|
||||
|
||||
data_handler_config: &data_handler_config
|
||||
start_time: 2008-01-01
|
||||
end_time: 2020-08-01
|
||||
fit_start_time: 2008-01-01
|
||||
fit_end_time: 2014-12-31
|
||||
instruments: *market
|
||||
port_analysis_config: &port_analysis_config
|
||||
strategy:
|
||||
class: TopkDropoutStrategy
|
||||
module_path: qlib.contrib.strategy
|
||||
kwargs:
|
||||
signal: <PRED>
|
||||
topk: 50
|
||||
n_drop: 5
|
||||
backtest:
|
||||
start_time: 2017-01-01
|
||||
end_time: 2020-08-01
|
||||
account: 100000000
|
||||
benchmark: *benchmark
|
||||
exchange_kwargs:
|
||||
limit_threshold: 0.095
|
||||
deal_price: close
|
||||
open_cost: 0.0005
|
||||
close_cost: 0.0015
|
||||
min_cost: 5
|
||||
task:
|
||||
model:
|
||||
class: LGBModel
|
||||
module_path: qlib.contrib.model.gbdt
|
||||
kwargs:
|
||||
loss: mse
|
||||
colsample_bytree: 0.8879
|
||||
learning_rate: 0.2
|
||||
subsample: 0.8789
|
||||
lambda_l1: 205.6999
|
||||
lambda_l2: 580.9768
|
||||
max_depth: 8
|
||||
num_leaves: 210
|
||||
num_threads: 20
|
||||
dataset:
|
||||
class: DatasetH
|
||||
module_path: qlib.data.dataset
|
||||
kwargs:
|
||||
handler:
|
||||
class: Alpha158
|
||||
module_path: qlib.contrib.data.handler
|
||||
kwargs: *data_handler_config
|
||||
segments:
|
||||
train: [2008-01-01, 2014-12-31]
|
||||
valid: [2015-01-01, 2016-12-31]
|
||||
test: [2017-01-01, 2020-08-01]
|
||||
record:
|
||||
- class: SignalRecord
|
||||
module_path: qlib.workflow.record_temp
|
||||
kwargs:
|
||||
model: <MODEL>
|
||||
dataset: <DATASET>
|
||||
- class: SigAnaRecord
|
||||
module_path: qlib.workflow.record_temp
|
||||
kwargs:
|
||||
ana_long_short: False
|
||||
ann_scaler: 252
|
||||
- class: PortAnaRecord
|
||||
module_path: qlib.workflow.record_temp
|
||||
kwargs:
|
||||
config: *port_analysis_config
|
||||
@@ -0,0 +1,93 @@
|
||||
qlib_init:
|
||||
provider_uri: "~/.qlib/qlib_data/cn_data"
|
||||
region: cn
|
||||
|
||||
market: &market csi300
|
||||
benchmark: &benchmark SH000300
|
||||
|
||||
data_handler_config: &data_handler_config
|
||||
start_time: 2008-01-01
|
||||
end_time: 2022-08-01
|
||||
instruments: *market
|
||||
data_loader:
|
||||
class: NestedDataLoader
|
||||
kwargs:
|
||||
dataloader_l:
|
||||
- class: qlib.contrib.data.loader.Alpha158DL
|
||||
kwargs:
|
||||
config:
|
||||
label:
|
||||
- ["Ref($close, -2)/Ref($close, -1) - 1"]
|
||||
- ["LABEL0"]
|
||||
- class: qlib.data.dataset.loader.StaticDataLoader
|
||||
kwargs:
|
||||
# config: "/home/finco/v-yuanteli/RD-Agent/rdagent/scenarios/qlib/task_generator/env_factor/combined_factors_df.pkl"
|
||||
config: "combined_factors_df.pkl"
|
||||
|
||||
learn_processors:
|
||||
- class: DropnaLabel
|
||||
- class: CSZScoreNorm
|
||||
kwargs:
|
||||
fields_group: label
|
||||
|
||||
port_analysis_config: &port_analysis_config
|
||||
strategy:
|
||||
class: TopkDropoutStrategy
|
||||
module_path: qlib.contrib.strategy
|
||||
kwargs:
|
||||
signal: <PRED>
|
||||
topk: 50
|
||||
n_drop: 5
|
||||
backtest:
|
||||
start_time: 2017-01-01
|
||||
end_time: 2020-08-01
|
||||
account: 100000000
|
||||
benchmark: *benchmark
|
||||
exchange_kwargs:
|
||||
limit_threshold: 0.095
|
||||
deal_price: close
|
||||
open_cost: 0.0005
|
||||
close_cost: 0.0015
|
||||
min_cost: 5
|
||||
|
||||
task:
|
||||
model:
|
||||
class: LGBModel
|
||||
module_path: qlib.contrib.model.gbdt
|
||||
kwargs:
|
||||
loss: mse
|
||||
colsample_bytree: 0.8879
|
||||
learning_rate: 0.2
|
||||
subsample: 0.8789
|
||||
lambda_l1: 205.6999
|
||||
lambda_l2: 580.9768
|
||||
max_depth: 8
|
||||
num_leaves: 210
|
||||
num_threads: 20
|
||||
dataset:
|
||||
class: DatasetH
|
||||
module_path: qlib.data.dataset
|
||||
kwargs:
|
||||
handler:
|
||||
class: DataHandlerLP
|
||||
module_path: qlib.contrib.data.handler
|
||||
kwargs: *data_handler_config
|
||||
segments:
|
||||
train: [2008-01-01, 2014-12-31]
|
||||
valid: [2015-01-01, 2016-12-31]
|
||||
test: [2017-01-01, 2020-08-01]
|
||||
record:
|
||||
- class: SignalRecord
|
||||
module_path: qlib.workflow.record_temp
|
||||
kwargs:
|
||||
model: <MODEL>
|
||||
dataset: <DATASET>
|
||||
- class: SigAnaRecord
|
||||
module_path: qlib.workflow.record_temp
|
||||
kwargs:
|
||||
ana_long_short: False
|
||||
ann_scaler: 252
|
||||
- class: PortAnaRecord
|
||||
module_path: qlib.workflow.record_temp
|
||||
kwargs:
|
||||
config: *port_analysis_config
|
||||
@@ -0,0 +1,53 @@
|
||||
import os
|
||||
import pickle
|
||||
from pathlib import Path
|
||||
|
||||
import pandas as pd
|
||||
import qlib
|
||||
from mlflow.entities import ViewType
|
||||
from mlflow.tracking import MlflowClient
|
||||
|
||||
qlib.init()
|
||||
|
||||
from qlib.workflow import R
|
||||
|
||||
# here is the documents of the https://qlib.readthedocs.io/en/latest/component/recorder.html
|
||||
|
||||
# TODO: list all the recorder and metrics
|
||||
|
||||
# Assuming you have already listed the experiments
|
||||
experiments = R.list_experiments()
|
||||
|
||||
# Iterate through each experiment to find the latest recorder
|
||||
experiment_name = None
|
||||
latest_recorder = None
|
||||
for experiment in experiments:
|
||||
# print(f"Experiment: {experiment}")
|
||||
recorders = R.list_recorders(experiment_name=experiment)
|
||||
for recorder_id in recorders:
|
||||
if recorder_id is not None:
|
||||
experiment_name = experiment
|
||||
recorder = R.get_recorder(recorder_id=recorder_id, experiment_name=experiment)
|
||||
end_time = recorder.info["end_time"]
|
||||
if latest_recorder is None or end_time > latest_recorder.info["end_time"]:
|
||||
latest_recorder = recorder
|
||||
|
||||
# Check if the latest recorder is found
|
||||
if latest_recorder is None:
|
||||
print("No recorders found")
|
||||
else:
|
||||
print(f"Latest recorder: {latest_recorder}")
|
||||
|
||||
# Load the specified file from the latest recorder
|
||||
file_path = "portfolio_analysis/port_analysis_1day.pkl"
|
||||
indicator_analysis_df = latest_recorder.load_object(file_path)
|
||||
|
||||
# Optionally convert to DataFrame if not already in DataFrame format
|
||||
if not isinstance(indicator_analysis_df, pd.DataFrame):
|
||||
indicator_analysis_df = pd.DataFrame(indicator_analysis_df)
|
||||
|
||||
output_path = os.path.join(str(Path(__file__).resolve().parent), "qlib_res.pkl")
|
||||
with open(output_path, "wb") as f:
|
||||
pickle.dump(indicator_analysis_df, f)
|
||||
|
||||
print(f"Output has been saved to {output_path}")
|
||||
@@ -1,13 +1,101 @@
|
||||
# TODO:
|
||||
# Implement to feedback.
|
||||
|
||||
import json
|
||||
from rdagent.oai.llm_utils import APIBackend
|
||||
from rdagent.core.proposal import HypothesisExperiment2Feedback, Trace, Hypothesis, HypothesisFeedback, Scenario
|
||||
from pathlib import Path
|
||||
|
||||
from jinja2 import Environment, StrictUndefined
|
||||
|
||||
from rdagent.core.experiment import Experiment
|
||||
from rdagent.core.log import RDAgentLog
|
||||
from rdagent.core.prompts import Prompts
|
||||
from rdagent.core.proposal import (
|
||||
Hypothesis,
|
||||
HypothesisExperiment2Feedback,
|
||||
HypothesisFeedback,
|
||||
Trace,
|
||||
)
|
||||
from rdagent.oai.llm_utils import APIBackend
|
||||
|
||||
feedback_prompts = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")
|
||||
DIRNAME = Path(__file__).absolute().resolve().parent
|
||||
logger = RDAgentLog()
|
||||
|
||||
|
||||
class QlibFactorHypothesisExperiment2Feedback(HypothesisExperiment2Feedback): ...
|
||||
class QlibFactorHypothesisExperiment2Feedback(HypothesisExperiment2Feedback):
|
||||
def generateFeedback(self, exp: Experiment, hypothesis: Hypothesis, trace: Trace) -> HypothesisFeedback:
|
||||
"""
|
||||
Generate feedback for the given experiment and hypothesis.
|
||||
|
||||
Args:
|
||||
exp (QlibFactorExperiment): The experiment to generate feedback for.
|
||||
hypothesis (QlibFactorHypothesis): The hypothesis to generate feedback for.
|
||||
trace (Trace): The trace of the experiment.
|
||||
|
||||
Returns:
|
||||
Any: The feedback generated for the given experiment and hypothesis.
|
||||
"""
|
||||
logger.info("Generating feedback...")
|
||||
hypothesis_text = hypothesis.hypothesis
|
||||
current_result = exp.result
|
||||
tasks_factors = [task.get_task_information() for task in exp.sub_tasks]
|
||||
sota_result = exp.based_experiments[-1].result
|
||||
|
||||
# Generate the system prompt
|
||||
sys_prompt = (
|
||||
Environment(undefined=StrictUndefined)
|
||||
.from_string(feedback_prompts["data_feedback_generation"]["system"])
|
||||
.render(scenario=self.scen.get_scenario_all_desc())
|
||||
)
|
||||
|
||||
# Generate the user prompt
|
||||
usr_prompt = (
|
||||
Environment(undefined=StrictUndefined)
|
||||
.from_string(feedback_prompts["data_feedback_generation"]["user"])
|
||||
.render(
|
||||
hypothesis_text=hypothesis_text,
|
||||
task_details=tasks_factors,
|
||||
current_result=current_result,
|
||||
sota_result=sota_result,
|
||||
)
|
||||
)
|
||||
|
||||
# Call the APIBackend to generate the response for hypothesis feedback
|
||||
response = APIBackend().build_messages_and_create_chat_completion(
|
||||
user_prompt=usr_prompt,
|
||||
system_prompt=sys_prompt,
|
||||
json_mode=True,
|
||||
)
|
||||
|
||||
# Parse the JSON response to extract the feedback
|
||||
response_json = json.loads(response)
|
||||
|
||||
# Extract fields from JSON response
|
||||
observations = response_json.get("Observations", "No observations provided")
|
||||
hypothesis_evaluation = response_json.get("Feedback for Hypothesis", "No feedback provided")
|
||||
new_hypothesis = response_json.get("New Hypothesis", "No new hypothesis provided")
|
||||
reason = response_json.get("Reasoning", "No reasoning provided")
|
||||
decision = response_json.get("Replace Best Result", "no").lower() == "yes"
|
||||
|
||||
# Create HypothesisFeedback object
|
||||
hypothesis_feedback = HypothesisFeedback(
|
||||
observations=observations,
|
||||
hypothesis_evaluation=hypothesis_evaluation,
|
||||
new_hypothesis=new_hypothesis,
|
||||
reason=reason,
|
||||
decision=decision,
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"Generated Hypothesis Feedback:\n"
|
||||
f"Observations: {observations}\n"
|
||||
f"Feedback for Hypothesis: {hypothesis_evaluation}\n"
|
||||
f"New Hypothesis: {new_hypothesis}\n"
|
||||
f"Reason: {reason}\n"
|
||||
f"Replace Best Result: {'Yes' if decision else 'No'}"
|
||||
)
|
||||
|
||||
return hypothesis_feedback
|
||||
|
||||
|
||||
class QlibModelHypothesisExperiment2Feedback(HypothesisExperiment2Feedback):
|
||||
@@ -40,7 +128,7 @@ class QlibModelHypothesisExperiment2Feedback(HypothesisExperiment2Feedback):
|
||||
else:
|
||||
last_info_str = "This is the first round. No previous information available."
|
||||
|
||||
usr_prompt_hypothesis = f'''
|
||||
usr_prompt_hypothesis = f"""
|
||||
We are in an experiment of finding hypothesis and validating or rejecting them so that in the end we have a powerful model generated.
|
||||
Here are the context: {context}.
|
||||
{last_info_str}
|
||||
@@ -51,8 +139,8 @@ class QlibModelHypothesisExperiment2Feedback(HypothesisExperiment2Feedback):
|
||||
Result: {exp.result}\n
|
||||
|
||||
Compare and observe. Which result has a better return and lower risk? If the performance increases， the hypothesis should be considered positive (working).
|
||||
Hence, with the hypotheses, relevant reasonings, and results in mind (comparison), provide detailed and constructive feedback and suggest a new hypothesis.
|
||||
'''
|
||||
Hence, with the hypotheses, relevant reasoning, and results in mind (comparison), provide detailed and constructive feedback and suggest a new hypothesis.
|
||||
"""
|
||||
|
||||
try:
|
||||
# Call the APIBackend to generate the response for hypothesis feedback
|
||||
@@ -61,7 +149,7 @@ class QlibModelHypothesisExperiment2Feedback(HypothesisExperiment2Feedback):
|
||||
system_prompt=sys_prompt_hypothesis,
|
||||
json_mode=True,
|
||||
)
|
||||
|
||||
|
||||
# Parse the JSON response to extract the feedback
|
||||
response_json_hypothesis = json.loads(response_hypothesis)
|
||||
hypothesis_feedback = HypothesisFeedback(
|
||||
@@ -69,13 +157,13 @@ class QlibModelHypothesisExperiment2Feedback(HypothesisExperiment2Feedback):
|
||||
hypothesis_evaluation=response_json_hypothesis.get("Feedback for Hypothesis", "No feedback provided"),
|
||||
new_hypothesis=response_json_hypothesis.get("New Hypothesis", "No new hypothesis provided"),
|
||||
reason=response_json_hypothesis.get("Reasoning", "No reasoning provided"),
|
||||
decision=response_json_hypothesis.get("Decision", "false").lower() == "true"
|
||||
decision=response_json_hypothesis.get("Decision", "false").lower() == "true",
|
||||
)
|
||||
|
||||
return hypothesis_feedback
|
||||
|
||||
except json.JSONDecodeError as e:
|
||||
# TODO: (Xiao) I think raising a specific type of ERROR to make caller know sth bad has happend would be more reasonable
|
||||
# TODO: (Xiao) I think raising a specific type of ERROR to make caller know sth bad has happened would be more reasonable
|
||||
print("Error parsing JSON response from LLM for hypothesis feedback:", e)
|
||||
except Exception as e:
|
||||
print("An unexpected error occurred while generating hypothesis feedback:", e)
|
||||
@@ -85,6 +173,5 @@ class QlibModelHypothesisExperiment2Feedback(HypothesisExperiment2Feedback):
|
||||
hypothesis_evaluation="No feedback",
|
||||
new_hypothesis="No new hypothesis",
|
||||
reason="No reasoning",
|
||||
decision=False
|
||||
decision=False,
|
||||
)
|
||||
|
||||
|
||||
+40
-29
@@ -5,14 +5,21 @@ Tries to create uniform environment for the agent to run;
|
||||
- All the code and data is expected included in one folder
|
||||
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import docker
|
||||
import subprocess
|
||||
import sys
|
||||
from abc import abstractmethod
|
||||
from pydantic import BaseModel
|
||||
from typing import Generic, TypeVar, Optional, Dict
|
||||
from pathlib import Path
|
||||
from typing import Dict, Generic, Optional, TypeVar
|
||||
|
||||
import docker
|
||||
import docker.models
|
||||
import docker.models.containers
|
||||
from pydantic import BaseModel
|
||||
from pydantic_settings import BaseSettings
|
||||
|
||||
from rdagent.core.log import RDAgentLog
|
||||
|
||||
ASpecificBaseModel = TypeVar("ASpecificBaseModel", bound=BaseModel)
|
||||
|
||||
@@ -71,6 +78,7 @@ class LocalEnv(Env[LocalConf]):
|
||||
"""
|
||||
Sometimes local environment may be more convinient for testing
|
||||
"""
|
||||
|
||||
def prepare(self):
|
||||
if not (Path("~/.qlib/qlib_data/cn_data").expanduser().resolve().exists()):
|
||||
self.run(
|
||||
@@ -79,10 +87,7 @@ class LocalEnv(Env[LocalConf]):
|
||||
else:
|
||||
print("Data already exists. Download skipped.")
|
||||
|
||||
def run(self,
|
||||
entry: str | None = None,
|
||||
local_path: Optional[str] = None,
|
||||
env: dict | None = None) -> str:
|
||||
def run(self, entry: str | None = None, local_path: Optional[str] = None, env: dict | None = None) -> str:
|
||||
if env is None:
|
||||
env = {}
|
||||
|
||||
@@ -94,15 +99,7 @@ class LocalEnv(Env[LocalConf]):
|
||||
cwd = None
|
||||
if local_path:
|
||||
cwd = Path(local_path).resolve()
|
||||
print(f"CWD: {cwd}")
|
||||
|
||||
result = subprocess.run(
|
||||
command,
|
||||
cwd=cwd,
|
||||
env={**os.environ, **env},
|
||||
capture_output=True,
|
||||
text=True
|
||||
)
|
||||
result = subprocess.run(command, cwd=cwd, env={**os.environ, **env}, capture_output=True, text=True)
|
||||
|
||||
if result.returncode != 0:
|
||||
raise RuntimeError(f"Error while running the command: {result.stderr}")
|
||||
@@ -113,8 +110,10 @@ class LocalEnv(Env[LocalConf]):
|
||||
## Docker Environment -----
|
||||
|
||||
|
||||
class DockerConf(BaseModel):
|
||||
image: str # the image you want to run
|
||||
class DockerConf(BaseSettings):
|
||||
build_from_dockerfile: bool = False
|
||||
dockerfile_folder_path: Path # the path to the dockerfile
|
||||
image: str # the image you want to build
|
||||
mount_path: str # the path in the docker image to mount the folder
|
||||
default_entry: str # the entry point of the image
|
||||
|
||||
@@ -122,14 +121,16 @@ class DockerConf(BaseModel):
|
||||
# Sometime, we need maintain some extra data for the workspace.
|
||||
# And the extra data may be shared and the downloading can be time consuming.
|
||||
# So we just want to download it once.
|
||||
network: str | None = "bridge" # the network mode for the docker
|
||||
|
||||
|
||||
QLIB_TORCH_IMAGE = DockerConf(
|
||||
image="linlanglv/qlib_image_nightly_pytorch:nightly",
|
||||
mount_path="/workspace",
|
||||
default_entry="qrun conf.yaml",
|
||||
extra_volumes={Path("~/.qlib/").expanduser().resolve(): "/root/.qlib/"},
|
||||
)
|
||||
class QlibDockerConf(DockerConf):
|
||||
build_from_dockerfile: bool = True
|
||||
dockerfile_folder_path: Path = Path(__file__).parent.parent / "scenarios" / "qlib" / "docker"
|
||||
image: str = "local_qlib:latest"
|
||||
mount_path: str = "/workspace/qlib_workspace/"
|
||||
default_entry: str = "qrun conf.yaml"
|
||||
extra_volumes: dict = {Path("~/.qlib/").expanduser().resolve(): "/root/.qlib/"}
|
||||
|
||||
|
||||
class DockerEnv(Env[DockerConf]):
|
||||
@@ -140,6 +141,12 @@ class DockerEnv(Env[DockerConf]):
|
||||
Download image if it doesn't exist
|
||||
"""
|
||||
client = docker.from_env()
|
||||
if self.conf.build_from_dockerfile is not None and self.conf.dockerfile_folder_path.exists():
|
||||
RDAgentLog().info(f"Building the image from dockerfile: {self.conf.dockerfile_folder_path}")
|
||||
image, logs = client.images.build(
|
||||
path=str(self.conf.dockerfile_folder_path), tag=self.conf.image, network_mode=self.conf.network
|
||||
)
|
||||
RDAgentLog().info(f"Finished building the image from dockerfile: {self.conf.dockerfile_folder_path}")
|
||||
try:
|
||||
client.images.get(self.conf.image)
|
||||
except docker.errors.ImageNotFound:
|
||||
@@ -164,14 +171,15 @@ class DockerEnv(Env[DockerConf]):
|
||||
|
||||
log_output = ""
|
||||
try:
|
||||
container = client.containers.run(
|
||||
container: docker.models.containers.Container = client.containers.run(
|
||||
image=self.conf.image,
|
||||
command=entry,
|
||||
volumes=volumns,
|
||||
environment=env,
|
||||
detach=True,
|
||||
working_dir=self.conf.mount_path,
|
||||
auto_remove=True,
|
||||
# auto_remove=True, # remove too fast might cause the logs not to be get
|
||||
network=self.conf.network,
|
||||
)
|
||||
logs = container.logs(stream=True)
|
||||
for log in logs:
|
||||
@@ -179,6 +187,8 @@ class DockerEnv(Env[DockerConf]):
|
||||
print(decoded_log)
|
||||
log_output += decoded_log + "\n"
|
||||
container.wait()
|
||||
container.stop()
|
||||
container.remove()
|
||||
return log_output
|
||||
except docker.errors.ContainerError as e:
|
||||
raise RuntimeError(f"Error while running the container: {e}")
|
||||
@@ -191,7 +201,7 @@ class DockerEnv(Env[DockerConf]):
|
||||
class QTDockerEnv(DockerEnv):
|
||||
"""Qlib Torch Docker"""
|
||||
|
||||
def __init__(self, conf: DockerConf = QLIB_TORCH_IMAGE):
|
||||
def __init__(self, conf: DockerConf = QlibDockerConf()):
|
||||
super().__init__(conf)
|
||||
|
||||
def prepare(self):
|
||||
@@ -201,7 +211,8 @@ class QTDockerEnv(DockerEnv):
|
||||
super().prepare()
|
||||
qlib_data_path = next(iter(self.conf.extra_volumes.keys()))
|
||||
if not (Path(qlib_data_path) / "qlib_data" / "cn_data").exists():
|
||||
RDAgentLog().info("We are downloading!")
|
||||
cmd = "python -m qlib.run.get_data qlib_data --target_dir ~/.qlib/qlib_data/cn_data --region cn --interval 1d --delete_old False"
|
||||
self.run(entry=cmd)
|
||||
else:
|
||||
print("Data already exists. Download skipped.")
|
||||
RDAgentLog().info("Data already exists. Download skipped.")
|
||||
|
||||
@@ -18,6 +18,7 @@ matplotlib
|
||||
langchain
|
||||
tiktoken
|
||||
scikit-learn
|
||||
docker
|
||||
|
||||
# azure identity related
|
||||
azure.identity
|
||||
|
||||
@@ -0,0 +1,116 @@
|
||||
# 🐳 Run Docker & Qlib
|
||||
---
|
||||
|
||||
## 📄 Description
|
||||
This guide explains how to run the Qlib Docker test file located at `test/utils/test_env.py` in the RD-Agent repository.
|
||||
|
||||
---
|
||||
|
||||
## 🚀 Running Instructions
|
||||
|
||||
### 1. Install the required Python libraries
|
||||
- Ensure that the `docker` Python library is installed:
|
||||
```sh
|
||||
pip install docker
|
||||
```
|
||||
|
||||
### 2. Run the test script
|
||||
- Execute the test script to verify the Docker environment setup:
|
||||
```sh
|
||||
python test/utils/test_env.py
|
||||
```
|
||||
|
||||
### Troubleshooting
|
||||
- **PermissionError: [Errno 13] Permission denied.**
|
||||
> This error occurs when the current user does not have the necessary permissions to access the Docker socket. To resolve this issue, follow these steps:
|
||||
|
||||
1. **Add the current user to the `docker` group**
|
||||
Docker requires root or `docker` group user permissions to access the Docker socket. Add the current user to the `docker` group:
|
||||
```sh
|
||||
sudo usermod -aG docker $USER
|
||||
```
|
||||
|
||||
2. **Refresh group changes**
|
||||
To apply the group changes, log out and log back in, or use the following command:
|
||||
```sh
|
||||
newgrp docker
|
||||
```
|
||||
|
||||
3. **Verify Docker access**
|
||||
Run the following command to ensure that Docker can be accessed:
|
||||
```sh
|
||||
docker run hello-world
|
||||
```
|
||||
|
||||
4. **Rerun the test script**
|
||||
After completing these steps, rerun the test script:
|
||||
```sh
|
||||
python test/utils/test_env.py
|
||||
```
|
||||
---
|
||||
## 🛠️ Detailed Qlib Docker Function Framework
|
||||
|
||||
Here, we provide an overview of the specific functions within the Qlib Docker framework, their purposes, and examples of how to call them.
|
||||
|
||||
### QTDockerEnv Class in `env.py`
|
||||
|
||||
The `QTDockerEnv` class is responsible for setting up and running Docker environments for Qlib experiments.
|
||||
|
||||
#### Methods:
|
||||
|
||||
1. **prepare()**
|
||||
- **Purpose**: Prepares the Docker environment for running experiments. This includes building the Docker image if necessary.
|
||||
- **Example**:
|
||||
```python
|
||||
qtde = QTDockerEnv()
|
||||
qtde.prepare()
|
||||
```
|
||||
|
||||
2. **run(local_path: str, entry: str) -> str**
|
||||
- **Purpose**: Runs a specified entry point (e.g., a configuration file) in the prepared Docker environment.
|
||||
- **Parameters**:
|
||||
- `local_path`: Path to the local directory to mount into the Docker container.
|
||||
- `entry`: Command or entry point to run inside the Docker container.
|
||||
- **Returns**: The stdout output from the Docker container.
|
||||
- **Example**:
|
||||
```python
|
||||
result = qtde.run(local_path="/path/to/env_tpl", entry="qrun conf.yaml")
|
||||
```
|
||||
---
|
||||
### 📊 Expected Output
|
||||
|
||||
Upon successful execution, the test script will produce analysis results of benchmark returns and various risk metrics. The expected output should be similar to:
|
||||
|
||||
```
|
||||
'The following are analysis results of benchmark return (1 day).'
|
||||
risk
|
||||
mean 0.000477
|
||||
std 0.012295
|
||||
annualized_return 0.113561
|
||||
information_ratio 0.598699
|
||||
max_drawdown -0.370479
|
||||
|
||||
'The following are analysis results of the excess return without cost (1 day).'
|
||||
risk
|
||||
mean 0.000530
|
||||
std 0.005718
|
||||
annualized_return 0.126029
|
||||
information_ratio 1.428574
|
||||
max_drawdown -0.072310
|
||||
|
||||
'The following are analysis results of the excess return with cost (1 day).'
|
||||
risk
|
||||
mean 0.000339
|
||||
std 0.005717
|
||||
annualized_return 0.080654
|
||||
information_ratio 0.914486
|
||||
max_drawdown -0.086083
|
||||
|
||||
'The following are analysis results of indicators (1 day).'
|
||||
value
|
||||
ffr 1.0
|
||||
pa 0.0
|
||||
pos 0.0
|
||||
```
|
||||
|
||||
By following these steps and using the provided functions, you should be able to run the Qlib Docker tests and obtain the expected analysis results.
|
||||
+16
-17
@@ -2,10 +2,11 @@ import os
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.append(str(Path(__file__).resolve().parent.parent))
|
||||
from rdagent.utils.env import QTDockerEnv, LocalEnv, LocalConf
|
||||
import shutil
|
||||
|
||||
from rdagent.utils.env import LocalConf, LocalEnv, QTDockerEnv
|
||||
|
||||
DIRNAME = Path(__file__).absolute().resolve().parent
|
||||
|
||||
@@ -23,18 +24,17 @@ class EnvUtils(unittest.TestCase):
|
||||
|
||||
# NOTE: Since I don't know the exact environment in which it will be used, here's just an example.
|
||||
# NOTE: Because you need to download the data during the prepare process. So you need to have pyqlib in your environment.
|
||||
# def test_local(self):
|
||||
# local_conf = LocalConf(
|
||||
# py_bin="/home/v-linlanglv/miniconda3/envs/RD-Agent-310/bin",
|
||||
# default_entry="qrun conf.yaml",
|
||||
# )
|
||||
# qle = LocalEnv(conf=local_conf)
|
||||
# qle.prepare()
|
||||
# exe_path = str(DIRNAME / "env_tpl")
|
||||
# conf_path = str(DIRNAME / "env_tpl" / "conf.yaml")
|
||||
# qle.run(entry="qrun " + conf_path, local_path=exe_path)
|
||||
# mlrun_p = DIRNAME / "env_tpl" / "mlruns"
|
||||
# self.assertTrue(mlrun_p.exists(), f"Expected output file {mlrun_p} not found")
|
||||
def test_local(self):
|
||||
local_conf = LocalConf(
|
||||
py_bin="/home/v-linlanglv/miniconda3/envs/RD-Agent-310/bin",
|
||||
default_entry="qrun conf.yaml",
|
||||
)
|
||||
qle = LocalEnv(conf=local_conf)
|
||||
qle.prepare()
|
||||
conf_path = str(DIRNAME / "env_tpl" / "conf.yaml")
|
||||
qle.run(entry="qrun " + conf_path)
|
||||
mlrun_p = DIRNAME / "env_tpl" / "mlruns"
|
||||
self.assertTrue(mlrun_p.exists(), f"Expected output file {mlrun_p} not found")
|
||||
|
||||
def test_docker(self):
|
||||
"""
|
||||
@@ -42,16 +42,15 @@ class EnvUtils(unittest.TestCase):
|
||||
And run the docker image with `qrun conf.yaml`
|
||||
"""
|
||||
qtde = QTDockerEnv()
|
||||
qtde.prepare()
|
||||
qtde.prepare() # you can prepare for multiple times. It is expected to handle it correctly
|
||||
# the stdout are returned as result
|
||||
result = qtde.run(local_path=str(DIRNAME / "env_tpl"), entry="qrun conf.yaml")
|
||||
|
||||
mlrun_p = DIRNAME / "env_tpl" / "mlruns"
|
||||
|
||||
mlrun_p = DIRNAME / "env_tpl" / "mlruns"
|
||||
self.assertTrue(mlrun_p.exists(), f"Expected output file {mlrun_p} not found")
|
||||
|
||||
# read experiment
|
||||
result = qtde.run(local_path=str(DIRNAME / "env_tpl"), entry="python read_exp.py")
|
||||
result = qtde.run(local_path=str(DIRNAME / "env_tpl"), entry="python read_exp_res.py")
|
||||
print(result)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user