gavindiaz/NexQuant
1
0
Fork 0
mirror of https://github.com/NicolasBohn/NexQuant.git synced 2026-07-27 23:47:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

help yuante on the final version of data code

Browse Source
This commit is contained in:
Xu Yang
2024-07-15 08:28:34 +00:00
parent 57ae5c93ea
commit 1ed2f655b9
19 changed files with 267 additions and 214 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rdagent/app/qlib_rd_loop/model.py
+1 -1
View File
@@ -25,7 +25,7 @@ hypothesis2experiment: Hypothesis2Experiment = import_class(PROP_SETTING.qlib_mo
qlib_model_coder: TaskGenerator = import_class(PROP_SETTING.qlib_model_coder)(scen)
qlib_model_runner: TaskGenerator = import_class(PROP_SETTING.qlib_model_runner)(scen)
qlib_model_summarizer: HypothesisExperiment2Feedback = import_class(PROP_SETTING.qlib_model_hypothesis2experiment)()
qlib_model_summarizer: HypothesisExperiment2Feedback = import_class(PROP_SETTING.qlib_model_summarizer)()
trace = Trace(scen=scen)
for _ in range(PROP_SETTING.evolving_n):
rdagent/components/coder/factor_coder/CoSTEER/evaluators.py
+28 -3
View File
@@ -84,7 +84,7 @@ class FactorCodeEvaluator(FactorEvaluator):
gt_implementation: Implementation = None,
**kwargs,
):
factor_information = target_task.get_factor_information()
factor_information = target_task.get_task_information()
code = implementation.code
system_prompt = (
@@ -181,6 +181,28 @@ class FactorOutputFormatEvaluator(FactorEvaluator):
)
class FactorDatetimeDailyEvaluator(FactorEvaluator):
def evaluate(
self,
implementation: Implementation,
gt_implementation: Implementation,
) -> Tuple[str | object]:
_, gen_df = self._get_df(gt_implementation, implementation)
if gen_df is None:
return "The source dataframe is None. Skip the evaluation of the datetime format.", False
if "datetime" not in gen_df.index.names:
return "The source dataframe does not have a datetime index. Please check the implementation.", False
time_diff = gen_df.index.get_level_values("datetime").to_series().diff().dropna().unique()
if pd.Timedelta(minutes=1) in time_diff:
return (
"The generated dataframe is not daily. The implementation is definitely wrong. Please check the implementation.",
False,
)
return "The generated dataframe is daily.", True
class FactorRowCountEvaluator(FactorEvaluator):
def evaluate(
self,
@@ -314,6 +336,9 @@ class FactorValueEvaluator(FactorEvaluator):
feedback_str, _ = FactorOutputFormatEvaluator(self.scen).evaluate(implementation, gt_implementation)
conclusions.append(feedback_str)
feedback_str, _ = FactorDatetimeDailyEvaluator(self.scen).evaluate(implementation, gt_implementation)
conclusions.append(feedback_str)
# Check if both dataframe have the same rows count
if gt_implementation is not None:
feedback_str, _ = FactorRowCountEvaluator(self.scen).evaluate(implementation, gt_implementation)
@@ -373,7 +398,7 @@ class FactorFinalDecisionEvaluator(Evaluator):
evaluate_prompts["evaluator_final_decision_v1_user"],
)
.render(
factor_information=target_task.get_factor_information(),
factor_information=target_task.get_task_information(),
execution_feedback=execution_feedback_to_render,
code_feedback=code_feedback,
factor_value_feedback=(
@@ -475,7 +500,7 @@ class FactorEvaluatorForCoder(FactorEvaluator):
if implementation is None:
return None
target_task_information = target_task.get_factor_information()
target_task_information = target_task.get_task_information()
if (
queried_knowledge is not None
and target_task_information in queried_knowledge.success_task_to_knowledge_dict
rdagent/components/coder/factor_coder/CoSTEER/evolving_strategy.py
+3 -3
View File
@@ -59,7 +59,7 @@ class MultiProcessEvolvingStrategy(EvolvingStrategy):
# 1.找出需要evolve的factor
to_be_finished_task_index = []
for index, target_factor_task in enumerate(new_evo.sub_tasks):
target_factor_task_desc = target_factor_task.get_factor_information()
target_factor_task_desc = target_factor_task.get_task_information()
if target_factor_task_desc in queried_knowledge.success_task_to_knowledge_dict:
new_evo.sub_implementations[index] = queried_knowledge.success_task_to_knowledge_dict[
target_factor_task_desc
@@ -119,7 +119,7 @@ class FactorEvolvingStrategy(MultiProcessEvolvingStrategy):
target_task: FactorTask,
queried_knowledge: FactorQueriedKnowledgeV1 = None,
) -> Implementation:
factor_information_str = target_task.get_factor_information()
factor_information_str = target_task.get_task_information()
if queried_knowledge is not None and factor_information_str in queried_knowledge.success_task_to_knowledge_dict:
return queried_knowledge.success_task_to_knowledge_dict[factor_information_str].implementation
@@ -208,7 +208,7 @@ class FactorEvolvingStrategyWithGraph(MultiProcessEvolvingStrategy):
) -> Implementation:
error_summary = FACTOR_IMPLEMENT_SETTINGS.v2_error_summary
# 1. 提取因子的背景信息
target_factor_task_information = target_task.get_factor_information()
target_factor_task_information = target_task.get_task_information()
# 2. 检查该因子是否需要继续做（是否已经作对，是否做错太多）
if (
rdagent/components/coder/factor_coder/CoSTEER/knowledge_management.py
+6 -6
View File
@@ -114,7 +114,7 @@ class FactorRAGStrategyV1(RAGStrategy):
feedback = evo_step.feedback
for task_index in range(len(implementations.sub_tasks)):
target_task = implementations.sub_tasks[task_index]
target_task_information = target_task.get_factor_information()
target_task_information = target_task.get_task_information()
implementation = implementations.sub_implementations[task_index]
single_feedback = feedback[task_index]
if single_feedback is None:
@@ -147,7 +147,7 @@ class FactorRAGStrategyV1(RAGStrategy):
queried_knowledge = FactorQueriedKnowledgeV1()
for target_factor_task in evo.sub_tasks:
target_factor_task_information = target_factor_task.get_factor_information()
target_factor_task_information = target_factor_task.get_task_information()
if target_factor_task_information in self.knowledgebase.success_task_info_set:
queried_knowledge.success_task_to_knowledge_dict[target_factor_task_information] = (
self.knowledgebase.implementation_trace[target_factor_task_information][-1]
@@ -233,7 +233,7 @@ class FactorGraphRAGStrategy(RAGStrategy):
for task_index in range(len(implementations.sub_tasks)):
single_feedback = feedback[task_index]
target_task = implementations.sub_tasks[task_index]
target_task_information = target_task.get_factor_information()
target_task_information = target_task.get_task_information()
implementation = implementations.sub_implementations[task_index]
single_feedback = feedback[task_index]
if single_feedback is None:
@@ -395,7 +395,7 @@ class FactorGraphRAGStrategy(RAGStrategy):
fail_task_trial_limit = FACTOR_IMPLEMENT_SETTINGS.fail_task_trial_limit
for target_factor_task in evo.sub_tasks:
target_factor_task_information = target_factor_task.get_factor_information()
target_factor_task_information = target_factor_task.get_task_information()
if (
target_factor_task_information not in self.knowledgebase.success_task_to_knowledge_dict
and target_factor_task_information in self.knowledgebase.working_trace_knowledge
@@ -442,7 +442,7 @@ class FactorGraphRAGStrategy(RAGStrategy):
) -> QueriedKnowledge | None:
# queried_component_knowledge = FactorQueriedGraphComponentKnowledge()
for target_factor_task in evo.sub_tasks:
target_factor_task_information = target_factor_task.get_factor_information()
target_factor_task_information = target_factor_task.get_task_information()
if (
target_factor_task_information in self.knowledgebase.success_task_to_knowledge_dict
or target_factor_task_information in factor_implementation_queried_graph_knowledge.failed_task_info_set
@@ -582,7 +582,7 @@ class FactorGraphRAGStrategy(RAGStrategy):
) -> QueriedKnowledge | None:
# queried_error_knowledge = FactorQueriedGraphErrorKnowledge()
for task_index, target_factor_task in enumerate(evo.sub_tasks):
target_factor_task_information = target_factor_task.get_factor_information()
target_factor_task_information = target_factor_task.get_task_information()
factor_implementation_queried_graph_knowledge.error_with_success_task[target_factor_task_information] = {}
if (
target_factor_task_information in self.knowledgebase.success_task_to_knowledge_dict
rdagent/components/coder/factor_coder/CoSTEER/scheduler.py
+1 -1
View File
@@ -39,7 +39,7 @@ def LLMSelect(
tasks = []
for i in to_be_finished_task_index:
# find corresponding former trace for each task
target_factor_task_information = evo.sub_tasks[i].get_factor_information()
target_factor_task_information = evo.sub_tasks[i].get_task_information()
if target_factor_task_information in former_trace:
tasks.append((i, evo.sub_tasks[i], former_trace[target_factor_task_information]))
rdagent/components/coder/factor_coder/factor.py
+1 -1
View File
@@ -37,7 +37,7 @@ class FactorTask(Task):
self.variables = variables
self.factor_resources = resource
def get_factor_information(self):
def get_task_information(self):
return f"""factor_name: {self.factor_name}
factor_description: {self.factor_description}
factor_formulation: {self.factor_formulation}
rdagent/components/coder/factor_coder/prompts.yaml
+4 -4
View File
@@ -68,7 +68,7 @@ evolving_strategy_factor_implementation_v1_user: |-
--------------Correct code to similar factors:---------------
{% for similar_successful_knowledge in queried_similar_successful_knowledge %}
=====Factor {{loop.index}}:=====
{{ similar_successful_knowledge.target_task.get_factor_information() }}
{{ similar_successful_knowledge.target_task.get_task_information() }}
=====Code:=====
{{ similar_successful_knowledge.implementation.code }}
{% endfor %}
@@ -94,7 +94,7 @@ evolving_strategy_factor_implementation_v2_user: |-
When doing other tasks, you met some similar errors but you finally solve them. Here are some examples:
{% for error_content, similar_error_knowledge in queried_similar_error_knowledge %}
--------------Factor information to similar error ({{error_content}}):---------------
{{ similar_error_knowledge[0].target_task.get_factor_information() }}
{{ similar_error_knowledge[0].target_task.get_task_information() }}
=====Code with similar error ({{error_content}}):=====
{{ similar_error_knowledge[0].implementation.code }}
=====Success code to former code with similar error ({{error_content}}):=====
@@ -111,7 +111,7 @@ evolving_strategy_factor_implementation_v2_user: |-
--------------Correct code to similar factors:---------------
{% for similar_component_knowledge in queried_similar_component_knowledge %}
=====Factor {{loop.index}}:=====
{{ similar_component_knowledge.target_task.get_factor_information() }}
{{ similar_component_knowledge.target_task.get_task_information() }}
=====Code:=====
{{ similar_component_knowledge.implementation.code }}
{% endfor %}
@@ -137,7 +137,7 @@ evolving_strategy_error_summary_v2_user: |-
{% if queried_similar_error_knowledge|length != 0 %}
{% for error_content, similar_error_knowledge in queried_similar_error_knowledge %}
--------------Factor information to similar error ({{error_content}}):---------------
{{ similar_error_knowledge[0].target_task.get_factor_information() }}
{{ similar_error_knowledge[0].target_task.get_task_information() }}
=====Code with similar error ({{error_content}}):=====
{{ similar_error_knowledge[0].implementation.code }}
=====Success code to former code with similar error ({{error_content}}):=====
rdagent/components/coder/model_coder/CoSTEER/evaluators.py
+3 -3
View File
@@ -72,7 +72,7 @@ class ModelCodeEvaluator(Evaluator):
if gt_implementation is not None:
assert isinstance(gt_implementation, ModelImplementation)
model_task_information = target_task.get_information()
model_task_information = target_task.get_task_information()
code = implementation.code
system_prompt = (
@@ -146,7 +146,7 @@ class ModelFinalEvaluator(Evaluator):
evaluate_prompts["evaluator_final_feedback"]["user"],
)
.render(
model_information=target_task.get_information(),
model_information=target_task.get_task_information(),
model_execution_feedback=execution_feedback_to_render,
model_code_feedback=model_code_feedback,
model_value_feedback=model_value_feedback,
@@ -224,7 +224,7 @@ class ModelCoderEvaluator(Evaluator):
queried_knowledge: QueriedKnowledge = None,
**kwargs,
) -> ModelCoderFeedback:
target_task_information = target_task.get_information()
target_task_information = target_task.get_task_information()
if (
queried_knowledge is not None
and target_task_information in queried_knowledge.success_task_to_knowledge_dict
rdagent/components/coder/model_coder/CoSTEER/evolving_strategy.py
+2 -2
View File
@@ -27,7 +27,7 @@ class ModelCoderEvolvingStrategy(EvolvingStrategy):
target_task: ModelTask,
queried_knowledge: ModelQueriedKnowledge = None,
) -> ModelImplementation:
model_information_str = target_task.get_information()
model_information_str = target_task.get_task_information()
if queried_knowledge is not None and model_information_str in queried_knowledge.success_task_to_knowledge_dict:
return queried_knowledge.success_task_to_knowledge_dict[model_information_str].implementation
@@ -113,7 +113,7 @@ class ModelCoderEvolvingStrategy(EvolvingStrategy):
# 1.找出需要evolve的model
to_be_finished_task_index = []
for index, target_model_task in enumerate(new_evo.sub_tasks):
target_model_task_desc = target_model_task.get_information()
target_model_task_desc = target_model_task.get_task_information()
if target_model_task_desc in queried_knowledge.success_task_to_knowledge_dict:
new_evo.sub_implementations[index] = queried_knowledge.success_task_to_knowledge_dict[
target_model_task_desc
rdagent/components/coder/model_coder/CoSTEER/knowledge_management.py
+2 -2
View File
@@ -86,7 +86,7 @@ class ModelRAGStrategy(RAGStrategy):
feedback = evo_step.feedback
for task_index in range(len(implementations.sub_tasks)):
target_task = implementations.sub_tasks[task_index]
target_task_information = target_task.get_information()
target_task_information = target_task.get_task_information()
implementation = implementations.sub_implementations[task_index]
single_feedback = feedback[task_index]
if single_feedback is None:
@@ -119,7 +119,7 @@ class ModelRAGStrategy(RAGStrategy):
queried_knowledge = ModelQueriedKnowledge()
for target_model_task in evo.sub_tasks:
target_model_task_information = target_model_task.get_information()
target_model_task_information = target_model_task.get_task_information()
if target_model_task_information in self.knowledgebase.success_task_info_set:
queried_knowledge.success_task_to_knowledge_dict[target_model_task_information] = (
self.knowledgebase.implementation_trace[target_model_task_information][-1]
rdagent/components/coder/model_coder/model.py
+1 -1
View File
@@ -24,7 +24,7 @@ class ModelTask(Task):
self.variables: str = variables
self.model_type: str = model_type # Tabular for tabular model, TimesSeries for time series model
def get_information(self):
def get_task_information(self):
return f"""name: {self.name}
description: {self.description}
formulation: {self.formulation}
rdagent/core/experiment.py
+9 -2
View File
@@ -7,11 +7,17 @@ This file contains the all the class about organizing the task in RD-Agent.
"""
class Task:
class Task(ABC):
# TODO: 把name放在这里作为主键
# Please refer to rdagent/model_implementation/task.py for the implementation
# I think the task version applies to the base class.
pass
@abstractmethod
def get_task_information(self):
"""
Get the task information string to build the unique key
"""
pass
ASpecificTask = TypeVar("ASpecificTask", bound=Task)
@@ -117,6 +123,7 @@ class Experiment(ABC, Generic[ASpecificTask, ASpecificImp]):
"""
The experiment is a sequence of tasks and the implementations of the tasks after generated by the TaskGenerator.
"""
result_ws: Optional[FBImplementation]
def __init__(self, sub_tasks: Sequence[ASpecificTask]) -> None:
rdagent/scenarios/qlib/conf.py
+19
View File
@@ -0,0 +1,19 @@
from __future__ import annotations
from pathlib import Path
from dotenv import load_dotenv
from pydantic_settings import BaseSettings
# make sure that env variable is loaded while calling Config()
load_dotenv(verbose=True, override=True)
from pydantic_settings import BaseSettings
class QlibRDAgentSettings(BaseSettings):
runner_cache_result: bool = True # whether to cache the result of the docker execution
runner_cache_path: str = str(Path.cwd() / "runner_cache/") # the path to store the cache
Qlib_RD_AGENT_SETTINGS = QlibRDAgentSettings()
rdagent/scenarios/qlib/docker/Dockerfile
+21
View File
@@ -0,0 +1,21 @@
FROM pytorch/pytorch:latest
RUN apt-get clean && apt-get update && apt-get install -y \
curl \
vim \
git \
build-essential \
&& rm -rf /var/lib/apt/lists/*
RUN git clone https://github.com/microsoft/qlib.git
WORKDIR /workspace/qlib
RUN git reset c9ed050ef034fe6519c14b59f3d207abcb693282 --hard
RUN python -m pip install --upgrade numpy
RUN python -m pip install --upgrade cython
RUN python -m pip install -e .
RUN pip install catboost
RUN pip install xgboost
rdagent/scenarios/qlib/task_generator/data.py
+80 -126
View File
@@ -1,20 +1,23 @@
from pathlib import Path
import shutil
from typing import List
import pandas as pd
import pickle
from rdagent.app.qlib_rd_loop.conf import PROP_SETTING
from rdagent.core.task_generator import TaskGenerator
from rdagent.utils.env import QTDockerEnv, LocalConf, LocalEnv
from rdagent.scenarios.qlib.experiment.factor_experiment import QlibFactorExperiment
import shutil
from pathlib import Path
from typing import List, Tuple
import pandas as pd
from rdagent.core.log import RDAgentLog
from rdagent.core.task_generator import TaskGenerator
from rdagent.oai.llm_utils import md5_hash
from rdagent.scenarios.qlib.conf import Qlib_RD_AGENT_SETTINGS
from rdagent.scenarios.qlib.experiment.factor_experiment import QlibFactorExperiment
from rdagent.utils.env import QTDockerEnv
DIRNAME = Path(__file__).absolute().resolve().parent
DIRNAME_local = Path.cwd()
logger = RDAgentLog()
# class QlibFactorExpWorkspace:
# def prepare():
# # create a folder;
# # copy template
@@ -26,6 +29,7 @@ logger = RDAgentLog()
# TODO: supporting multiprocessing and keep previous results
class QlibFactorRunner(TaskGenerator[QlibFactorExperiment]):
"""
Docker run
@@ -35,141 +39,95 @@ class QlibFactorRunner(TaskGenerator[QlibFactorExperiment]):
- `data.py` + Adaptor to Factor implementation
- results in `mlflow`
"""
def FetchAlpha158ResultFromDocker(self):
"""
Run Docker to get alpha158 result.
This method prepares the Qlib Docker environment, executes the necessary commands to
run the backtest, and fetches the results stored in a pickle file.
def get_cache_key(self, exp: QlibFactorExperiment) -> str:
all_tasks = []
for based_exp in exp.based_experiments:
all_tasks.extend(based_exp.sub_tasks)
all_tasks.extend(exp.sub_tasks)
task_info_list = [task.get_task_information() for task in all_tasks]
task_info_str = "\n".join(task_info_list)
return md5_hash(task_info_str)
Returns:
Any: The alpha158 result. If successful, returns a pandas DataFrame. Otherwise, returns None.
"""
# Initialize and prepare the Qlib Docker environment
qtde = QTDockerEnv()
qtde.prepare()
# Clean up any previous run artifacts by deleting the mlruns directory
result = qtde.run(local_path=str(DIRNAME / "env_factor"), entry="rm -r mlruns", env={"PYTHONPATH": "./"})
# Run the Qlib backtest using the configuration file conf.yaml
result = qtde.run(local_path=str(DIRNAME / "env_factor"), entry="qrun conf.yaml", env={"PYTHONPATH": "./"})
# Execute a Python script to extract the experiment results
result = qtde.run(local_path=str(DIRNAME / "env_factor"), entry="python read_exp_res.py")
pkl_path = DIRNAME / 'env_factor/qlib_res.pkl'
if not pkl_path.exists():
logger.error(f"File {pkl_path} does not exist.")
return None
with open(pkl_path, 'rb') as f:
result = pickle.load(f)
# Check if the loaded result is a pandas DataFrame and not empty
if isinstance(result, pd.DataFrame):
if not result.empty:
logger.info("Successfully retrieved alpha158 result.")
return result
else:
logger.error("Result DataFrame is empty.")
return None
def get_cache_result(self, exp: QlibFactorExperiment) -> Tuple[bool, object]:
task_info_key = self.get_cache_key(exp)
Path(Qlib_RD_AGENT_SETTINGS.runner_cache_path).mkdir(parents=True, exist_ok=True)
cache_path = Path(Qlib_RD_AGENT_SETTINGS.runner_cache_path) / f"{task_info_key}.pkl"
if cache_path.exists():
return True, pickle.load(open(cache_path, "rb"))
else:
logger.error("Data format error.")
return None
return False, None
def dump_cache_result(self, exp: QlibFactorExperiment, result: object):
task_info_key = self.get_cache_key(exp)
cache_path = Path(Qlib_RD_AGENT_SETTINGS.runner_cache_path) / f"{task_info_key}.pkl"
pickle.dump(result, open(cache_path, "wb"))
def generate(self, exp: QlibFactorExperiment) -> QlibFactorExperiment:
"""
Generate the experiment by processing and combining factor data,
then passing the combined data to Docker for backtest results.
"""
SOTA_factor = None
if exp.based_experiments.__len__() != 1:
SOTA_factor = self.process_factor_data(exp.based_experiments)
if exp.based_experiments[-1].result is None:
exp.based_experiments[-1].result = self.FetchAlpha158ResultFromDocker()
# Process the new factors data
new_factors = self.process_factor_data(exp)
# Combine the SOTA factor and new factors if SOTA factor exists
if SOTA_factor is not None:
combined_factors = pd.concat([SOTA_factor, new_factors], axis=1).dropna()
else:
combined_factors = new_factors
# Sort and nest the combined factors under 'feature'
combined_factors = combined_factors.sort_index()
new_columns = pd.MultiIndex.from_product([['feature'], combined_factors.columns])
combined_factors.columns = new_columns
# logger.info(combined_factors)
# Save the combined factors to a pickle file
combined_factors_path = DIRNAME / 'env_factor/combined_factors_df.pkl'
with open(combined_factors_path, 'wb') as f:
pickle.dump(combined_factors, f)
if exp.based_experiments and exp.based_experiments[-1].result is None:
exp.based_experiments[-1] = self.generate(exp.based_experiments[-1])
if Qlib_RD_AGENT_SETTINGS.runner_cache_result:
cache_hit, result = self.get_cache_result(exp)
if cache_hit:
exp.result = result
return exp
if exp.based_experiments:
SOTA_factor = None
if exp.based_experiments.__len__() != 1:
SOTA_factor = self.process_factor_data(exp.based_experiments)
# Process the new factors data
new_factors = self.process_factor_data(exp)
# Combine the SOTA factor and new factors if SOTA factor exists
if SOTA_factor is not None and not SOTA_factor.empty:
combined_factors = pd.concat([SOTA_factor, new_factors], axis=1).dropna()
else:
combined_factors = new_factors
# Sort and nest the combined factors under 'feature'
combined_factors = combined_factors.sort_index()
new_columns = pd.MultiIndex.from_product([["feature"], combined_factors.columns])
combined_factors.columns = new_columns
# Save the combined factors to a pickle file
combined_factors_path = DIRNAME / "env_factor/combined_factors_df.pkl"
with open(combined_factors_path, "wb") as f:
pickle.dump(combined_factors, f)
# Docker run
# Call Docker, pass the combined factors to Docker, and generate backtest results
qtde = QTDockerEnv()
qtde.prepare()
# Run the Docker command
result = qtde.run(local_path=str(DIRNAME / "env_factor"), entry="rm -r mlruns", env={"PYTHONPATH": "./"})
execute_log = qtde.run(local_path=str(DIRNAME / "env_factor"), entry="rm -r mlruns")
# Run the Qlib backtest
result = qtde.run(local_path=str(DIRNAME / "env_factor"), entry="qrun conf_combined.yaml", env={"PYTHONPATH": "./"})
execute_log = qtde.run(
local_path=str(DIRNAME / "env_factor"),
entry=f"qrun conf.yaml" if len(exp.based_experiments) == 0 else "qrun conf_combined.yaml",
)
result = qtde.run(local_path=str(DIRNAME / "env_factor"), entry="python read_exp_res.py")
execute_log = qtde.run(local_path=str(DIRNAME / "env_factor"), entry="python read_exp_res.py")
pkl_path = DIRNAME / 'env_factor/qlib_res.pkl'
pkl_path = DIRNAME / "env_factor/qlib_res.pkl"
if not pkl_path.exists():
logger.error(f"File {pkl_path} does not exist.")
return None
with open(pkl_path, 'rb') as f:
with open(pkl_path, "rb") as f:
result = pickle.load(f)
"""
# TODO: Implement the Docker run in the following way
# Local run
# Clean up any previous run artifacts by deleting the mlruns directory
mlruns_path = DIRNAME_local / 'mlruns' / '1'
if mlruns_path.exists() and mlruns_path.is_dir():
shutil.rmtree(mlruns_path)
# Prepare local Qlib environment
local_conf = LocalConf(
py_bin=PROP_SETTING.py_bin,
default_entry="qrun conf_combined.yaml",
)
qle = LocalEnv(conf=local_conf)
qle.prepare()
conf_path = str(DIRNAME / "env_factor" / "conf_combined.yaml")
qle.run(entry="qrun " + conf_path, local_path=PROP_SETTING.local_qlib_folder)
# Verify if the new folder is created
mlrun_p = DIRNAME_local / 'mlruns' / '1'
assert mlrun_p.exists(), f"Expected output file {mlrun_p} not found"
# Locate the newly generated folder in mlruns/1/
new_folders = [folder for folder in mlrun_p.iterdir() if folder.is_dir()]
if not new_folders:
raise FileNotFoundError("No new folders found in 'mlruns/1/'.")
new_folder = new_folders[0] # Assuming there's only one new folder
pickle_file = new_folder / 'artifacts' / 'portfolio_analysis' / 'port_analysis_1day.pkl'
assert pickle_file.exists(), f"Expected pickle file {pickle_file} not found"
with open(pickle_file, 'rb') as f:
result = pickle.load(f)
"""
exp.result = result
if Qlib_RD_AGENT_SETTINGS.runner_cache_result:
self.dump_cache_result(exp, result)
# Check if the result is valid and is a DataFrame
if isinstance(result, pd.DataFrame):
@@ -204,18 +162,14 @@ class QlibFactorRunner(TaskGenerator[QlibFactorExperiment]):
message, df = implementation.execute()
# Check if factor generation was successful
if 'Execution succeeded without error.\nExpected output file found.' in message:
factor_dfs.append(df)
if df is not None:
time_diff = df.index.get_level_values("datetime").to_series().diff().dropna().unique()
if pd.Timedelta(minutes=1) not in time_diff:
factor_dfs.append(df)
# Combine all successful factor data
if factor_dfs:
combined_factors = pd.concat(factor_dfs, axis=1)
# Remove rows with NaN values
combined_factors = combined_factors.dropna()
# print(combined_factors)
return combined_factors
return pd.concat(factor_dfs, axis=1)
else:
logger.error("No valid factor data found to merge.")
return pd.DataFrame() # Return an empty DataFrame if no valid data
rdagent/scenarios/qlib/task_generator/env_factor/read_exp_res.py
+13 -12
View File
@@ -1,17 +1,19 @@
from pathlib import Path
import qlib
from mlflow.tracking import MlflowClient
from mlflow.entities import ViewType
import pandas as pd
import pickle
import os
import pickle
from pathlib import Path
import pandas as pd
import qlib
from mlflow.entities import ViewType
from mlflow.tracking import MlflowClient
qlib.init()
from qlib.workflow import R
# here is the documents of the https://qlib.readthedocs.io/en/latest/component/recorder.html
# TODO: list all the recorder and metrics
# TODO: list all the recorder and metrics
# Assuming you have already listed the experiments
experiments = R.list_experiments()
@@ -26,8 +28,8 @@ for experiment in experiments:
if recorder_id is not None:
experiment_name = experiment
recorder = R.get_recorder(recorder_id=recorder_id, experiment_name=experiment)
end_time = recorder.info['end_time']
if latest_recorder is None or end_time > latest_recorder.info['end_time']:
end_time = recorder.info["end_time"]
if latest_recorder is None or end_time > latest_recorder.info["end_time"]:
latest_recorder = recorder
# Check if the latest recorder is found
@@ -43,10 +45,9 @@ else:
# Optionally convert to DataFrame if not already in DataFrame format
if not isinstance(indicator_analysis_df, pd.DataFrame):
indicator_analysis_df = pd.DataFrame(indicator_analysis_df)
output_path = os.path.join(str(Path(__file__).resolve().parent), "qlib_res.pkl")
with open(output_path, "wb") as f:
pickle.dump(indicator_analysis_df, f)
print("here2")
print(output_path)
print(f"Output has been saved to {output_path}")
rdagent/scenarios/qlib/task_generator/feedback.py
+32 -18
View File
@@ -1,27 +1,33 @@
# TODO:
# Implement to feedback.
import json
import pickle
from pathlib import Path
import pandas as pd
from jinja2 import Environment, StrictUndefined
from rdagent.core.prompts import Prompts
from rdagent.core.proposal import HypothesisExperiment2Feedback
from rdagent.core.proposal import Trace
from rdagent.core.experiment import Experiment
from rdagent.core.proposal import Hypothesis, HypothesisFeedback
from rdagent.core.log import RDAgentLog
from rdagent.core.prompts import Prompts
from rdagent.core.proposal import (
Hypothesis,
HypothesisExperiment2Feedback,
HypothesisFeedback,
Trace,
)
from rdagent.oai.llm_utils import APIBackend
from rdagent.utils.env import QTDockerEnv
from rdagent.core.log import RDAgentLog
import json
import pandas as pd
import pickle
feedback_prompts = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")
DIRNAME = Path(__file__).absolute().resolve().parent
logger = RDAgentLog()
class QlibModelHypothesisExperiment2Feedback(HypothesisExperiment2Feedback): ...
class QlibFactorHypothesisExperiment2Feedback(HypothesisExperiment2Feedback):
def generateFeedback(self, exp: Experiment, hypothesis: Hypothesis, trace: Trace) -> HypothesisFeedback:
"""
@@ -38,18 +44,26 @@ class QlibFactorHypothesisExperiment2Feedback(HypothesisExperiment2Feedback):
logger.info("Generating feedback...")
hypothesis_text = hypothesis.hypothesis
current_result = exp.result
tasks_factors = [task.get_factor_information() for task in exp.sub_tasks]
tasks_factors = [task.get_task_information() for task in exp.sub_tasks]
sota_result = exp.based_experiments[-1].result
# Generate the system prompt
sys_prompt = Environment(undefined=StrictUndefined).from_string(feedback_prompts["data_feedback_generation"]["system"]).render(scenario=self.scen.get_scenario_all_desc())
sys_prompt = (
Environment(undefined=StrictUndefined)
.from_string(feedback_prompts["data_feedback_generation"]["system"])
.render(scenario=self.scen.get_scenario_all_desc())
)
# Generate the user prompt
usr_prompt = Environment(undefined=StrictUndefined).from_string(feedback_prompts["data_feedback_generation"]["user"]).render(
hypothesis_text=hypothesis_text,
task_details=tasks_factors,
current_result=current_result,
sota_result=sota_result
usr_prompt = (
Environment(undefined=StrictUndefined)
.from_string(feedback_prompts["data_feedback_generation"]["user"])
.render(
hypothesis_text=hypothesis_text,
task_details=tasks_factors,
current_result=current_result,
sota_result=sota_result,
)
)
# Call the APIBackend to generate the response for hypothesis feedback
@@ -61,21 +75,21 @@ class QlibFactorHypothesisExperiment2Feedback(HypothesisExperiment2Feedback):
# Parse the JSON response to extract the feedback
response_json = json.loads(response)
# Extract fields from JSON response
observations = response_json.get("Observations", "No observations provided")
hypothesis_evaluation = response_json.get("Feedback for Hypothesis", "No feedback provided")
new_hypothesis = response_json.get("New Hypothesis", "No new hypothesis provided")
reason = response_json.get("Reasoning", "No reasoning provided")
decision = response_json.get("Replace Best Result", "no").lower() == "yes"
# Create HypothesisFeedback object
hypothesis_feedback = HypothesisFeedback(
observations=observations,
hypothesis_evaluation=hypothesis_evaluation,
new_hypothesis=new_hypothesis,
reason=reason,
decision=decision
decision=decision,
)
logger.info(
rdagent/utils/env.py
+40 -29
View File
@@ -5,14 +5,21 @@ Tries to create uniform environment for the agent to run;
- All the code and data is expected included in one folder
"""
import os
import sys
import docker
import subprocess
import sys
from abc import abstractmethod
from pydantic import BaseModel
from typing import Generic, TypeVar, Optional, Dict
from pathlib import Path
from typing import Dict, Generic, Optional, TypeVar
import docker
import docker.models
import docker.models.containers
from pydantic import BaseModel
from pydantic_settings import BaseSettings
from rdagent.core.log import RDAgentLog
ASpecificBaseModel = TypeVar("ASpecificBaseModel", bound=BaseModel)
@@ -71,6 +78,7 @@ class LocalEnv(Env[LocalConf]):
"""
Sometimes local environment may be more convinient for testing
"""
def prepare(self):
if not (Path("~/.qlib/qlib_data/cn_data").expanduser().resolve().exists()):
self.run(
@@ -79,10 +87,7 @@ class LocalEnv(Env[LocalConf]):
else:
print("Data already exists. Download skipped.")
def run(self,
entry: str | None = None,
local_path: Optional[str] = None,
env: dict | None = None) -> str:
def run(self, entry: str | None = None, local_path: Optional[str] = None, env: dict | None = None) -> str:
if env is None:
env = {}
@@ -94,15 +99,7 @@ class LocalEnv(Env[LocalConf]):
cwd = None
if local_path:
cwd = Path(local_path).resolve()
print(f"CWD: {cwd}")
result = subprocess.run(
command,
cwd=cwd,
env={**os.environ, **env},
capture_output=True,
text=True
)
result = subprocess.run(command, cwd=cwd, env={**os.environ, **env}, capture_output=True, text=True)
if result.returncode != 0:
raise RuntimeError(f"Error while running the command: {result.stderr}")
@@ -113,8 +110,10 @@ class LocalEnv(Env[LocalConf]):
## Docker Environment -----
class DockerConf(BaseModel):
image: str # the image you want to run
class DockerConf(BaseSettings):
build_from_dockerfile: bool = False
dockerfile_folder_path: Path # the path to the dockerfile
image: str # the image you want to build
mount_path: str # the path in the docker image to mount the folder
default_entry: str # the entry point of the image
@@ -122,14 +121,16 @@ class DockerConf(BaseModel):
# Sometime, we need maintain some extra data for the workspace.
# And the extra data may be shared and the downloading can be time consuming.
# So we just want to download it once.
network: str | None = "bridge" # the network mode for the docker
QLIB_TORCH_IMAGE = DockerConf(
image="linlanglv/qlib_image_nightly_pytorch:240711",
mount_path="/workspace",
default_entry="qrun conf.yaml",
extra_volumes={Path("~/.qlib/").expanduser().resolve(): "/root/.qlib/"},
)
class QlibDockerConf(DockerConf):
build_from_dockerfile: bool = True
dockerfile_folder_path: Path = Path(__file__).parent.parent / "scenarios" / "qlib" / "docker"
image: str = "local_qlib:latest"
mount_path: str = "/workspace/qlib_workspace/"
default_entry: str = "qrun conf.yaml"
extra_volumes: dict = {Path("~/.qlib/").expanduser().resolve(): "/root/.qlib/"}
class DockerEnv(Env[DockerConf]):
@@ -140,6 +141,12 @@ class DockerEnv(Env[DockerConf]):
Download image if it doesn't exist
"""
client = docker.from_env()
if self.conf.build_from_dockerfile is not None and self.conf.dockerfile_folder_path.exists():
RDAgentLog().info(f"Building the image from dockerfile: {self.conf.dockerfile_folder_path}")
image, logs = client.images.build(
path=str(self.conf.dockerfile_folder_path), tag=self.conf.image, network_mode=self.conf.network
)
RDAgentLog().info(f"Finished building the image from dockerfile: {self.conf.dockerfile_folder_path}")
try:
client.images.get(self.conf.image)
except docker.errors.ImageNotFound:
@@ -164,14 +171,15 @@ class DockerEnv(Env[DockerConf]):
log_output = ""
try:
container = client.containers.run(
container: docker.models.containers.Container = client.containers.run(
image=self.conf.image,
command=entry,
volumes=volumns,
environment=env,
detach=True,
working_dir=self.conf.mount_path,
auto_remove=True,
# auto_remove=True, # remove too fast might cause the logs not to be get
network=self.conf.network,
)
logs = container.logs(stream=True)
for log in logs:
@@ -179,6 +187,8 @@ class DockerEnv(Env[DockerConf]):
print(decoded_log)
log_output += decoded_log + "\n"
container.wait()
container.stop()
container.remove()
return log_output
except docker.errors.ContainerError as e:
raise RuntimeError(f"Error while running the container: {e}")
@@ -191,7 +201,7 @@ class DockerEnv(Env[DockerConf]):
class QTDockerEnv(DockerEnv):
"""Qlib Torch Docker"""
def __init__(self, conf: DockerConf = QLIB_TORCH_IMAGE):
def __init__(self, conf: DockerConf = QlibDockerConf()):
super().__init__(conf)
def prepare(self):
@@ -201,7 +211,8 @@ class QTDockerEnv(DockerEnv):
super().prepare()
qlib_data_path = next(iter(self.conf.extra_volumes.keys()))
if not (Path(qlib_data_path) / "qlib_data" / "cn_data").exists():
RDAgentLog().info("We are downloading!")
cmd = "python -m qlib.run.get_data qlib_data --target_dir ~/.qlib/qlib_data/cn_data --region cn --interval 1d --delete_old False"
self.run(entry=cmd)
else:
print("Data already exists. Download skipped.")
RDAgentLog().info("Data already exists. Download skipped.")
requirements/package.txt
+1
View File
@@ -18,6 +18,7 @@ matplotlib
langchain
tiktoken
scikit-learn
docker
# azure identity related
azure.identity