gavindiaz/NexQuant
1
0
Fork 0
mirror of https://github.com/NicolasBohn/NexQuant.git synced 2026-07-27 23:47:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

CI checks that can be automatically repaired (#119)

Browse Source 
* fix isort & black & toml-sort & sphinx error

* fix ci error

* fix ci error

* add comments

* Update Makefile

* change sphinx build command

* add auto-lint

* add black args

* format with black

* Auto Linting document

* fix ci error

---------

Co-authored-by: you-n-g <you-n-g@users.noreply.github.com>
Co-authored-by: Young <afe.young@gmail.com>
This commit is contained in:
Linlang
2024-07-26 12:12:16 +08:00
committed by GitHub
parent a4fa000862
commit faa2fb03ad
56 changed files with 604 additions and 475 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/ci.yml
+4 -3
View File
@@ -20,16 +20,17 @@ jobs:
- run: env | sort
- run: make dev
- env:
# Two environment variables here can cause sphinx to fail to build in ci.
# CHAT_MAX_TOKENS: ${{ secrets.CHAT_MAX_TOKENS }}
# CHAT_TEMPERATURE: ${{ secrets.CHAT_TEMPERATURE }}
CHAT_AZURE_API_BASE: ${{ secrets.CHAT_AZURE_API_BASE }}
CHAT_AZURE_API_VERSION: ${{ secrets.CHAT_AZURE_API_VERSION }}
CHAT_MAX_TOKENS: ${{ secrets.CHAT_MAX_TOKENS }}
CHAT_MODEL: ${{ secrets.CHAT_MODEL }}
CHAT_TEMPERATURE: ${{ secrets.CHAT_TEMPERATURE }}
EMBEDDING_AZURE_API_BASE: ${{ secrets.CHAT_AZURE_API_BASE }}
EMBEDDING_AZURE_API_VERSION: ${{ secrets.CHAT_AZURE_API_VERSION }}
EMBEDDING_MODEL: ${{ secrets.EMBEDDING_MODEL }}
name: lint test docs and build
run: make lint # test docs build sphinx
run: make lint docs-gen # test docs build
strategy:
matrix:
python-version:
Makefile
+23 -6
View File
@@ -81,10 +81,7 @@ constraints: deepclean
# Check lint with black.
black:
$(PIPRUN) python -m black --check . --extend-exclude test/scripts --extend-exclude git_ignore_folder -l 120
sphinx:
$(PIPRUN) sphinx-build -W --keep-going -b html ./docs _build
$(PIPRUN) python -m black --check --diff . --extend-exclude test/scripts --extend-exclude git_ignore_folder -l 120
# Check lint with isort.
isort:
@@ -107,13 +104,33 @@ toml-sort:
$(PIPRUN) toml-sort --check pyproject.toml
# Check lint with all linters.
# lint: black isort mypy ruff toml-sort
lint: mypy ruff
# Prioritize fixing isort, then black, otherwise you'll get weird and unfixable black errors.
# lint: mypy ruff
lint: mypy ruff isort black toml-sort
# Run pre-commit with autofix against all files.
pre-commit:
pre-commit run --all-files
########################################################################################
# Auto Lint
########################################################################################
# Auto lint with black.
auto-black:
$(PIPRUN) python -m black . --extend-exclude test/scripts --extend-exclude git_ignore_folder -l 120
# Auto lint with isort.
auto-isort:
$(PIPRUN) python -m isort . -s git_ignore_folder -s test/scripts
# Auto lint with toml-sort.
auto-toml-sort:
$(PIPRUN) toml-sort pyproject.toml
# Auto lint with all linters.
auto-lint: auto-isort auto-black auto-toml-sort
########################################################################################
# Test
########################################################################################
docs/conf.py
+1 -1
View File
@@ -16,7 +16,7 @@ author = "Microsoft"
# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration
extensions = [
'sphinx.ext.autodoc',
"sphinx.ext.autodoc",
]
autodoc_member_order = "bysource"
docs/development.rst
+7 -1
View File
@@ -11,12 +11,18 @@ For Development
make dev
```
- Run linting and formatting.
- Run linting and checking.
```bash
make lint
```
- Some linting issues can be fixed automatically. We have added a command in the Makefile for easy use.
```bash
make auto-lint
```
Code Structure
=========================
docs/index.rst
+1
View File
@@ -17,6 +17,7 @@ Welcome to RDAgent's documentation!
development
api_reference
policy
test_dropdown/test1
.. test_dropdown/test1
docs/installation.rst
+9 -6
View File
@@ -27,13 +27,16 @@ USE_AZURE_TOKEN_PROVIDER
### ☁️ Azure Configuration
- Install Azure CLI:
```sh
curl -L https://aka.ms/InstallAzureCli | bash
```
```sh
curl -L https://aka.ms/InstallAzureCli | bash
```
- Log in to Azure:
```sh
az login --use-device-code
```
```sh
az login --use-device-code
```
- `exit` and re-login to your environment (this step may not be necessary).
docs/scenarios_and_quickstart.rst
+2 -2
View File
@@ -23,7 +23,7 @@ Scenario lists
-
Scnarios' demo & quick start
=========================
============================
Scen1
-----
@@ -69,7 +69,7 @@ TODO: Show some examples:
Scen2:
-----
------
📄 Research Report-Based Factor Extraction
Scen2 Intro
pyproject.toml
+3 -4
View File
@@ -4,7 +4,7 @@ requires = [
"setuptools",
"setuptools-scm",
]
root= "rdagent"
root = "rdagent"
[project]
authors = [
@@ -27,14 +27,13 @@ dynamic = [
]
keywords = [
"Autonomous Agents",
"Research and Development",
"Large Language Models",
"Research and Development",
]
name = "rdagent"
readme = "README.md"
requires-python = ">=3.8"
[project.urls]
homepage = "https://github.com/microsoft/RD-Agent/"
issue = "https://github.com/microsoft/RD-Agent/issues"
@@ -52,13 +51,13 @@ color_output = true
profile = "black"
[tool.mypy]
explicit_package_bases = true
check_untyped_defs = true
disallow_any_unimported = true
disallow_untyped_defs = true
enable_error_code = [
"ignore-without-code",
]
explicit_package_bases = true
warn_return_any = true
warn_unused_ignores = true
rdagent/app/data_mining/conf.py
+3 -4
View File
@@ -1,6 +1,5 @@
from pathlib import Path
from rdagent.components.workflow.conf import BasePropSetting
@@ -19,11 +18,11 @@ class PropSetting(BasePropSetting):
evolving_n: int = 10
# 2) Extra config for the scenario
# 2) Extra config for the scenario
# physionet account
# NOTE: You should apply the account in https://physionet.org/
username: str = ''
password: str = ''
username: str = ""
password: str = ""
PROP_SETTING = PropSetting()
rdagent/app/data_mining/model.py
+2
View File
@@ -1,8 +1,10 @@
import fire
from rdagent.app.data_mining.conf import PROP_SETTING
from rdagent.components.workflow.rd_loop import RDLoop
from rdagent.core.exception import ModelEmptyError
class ModelRDLoop(RDLoop):
skip_loop_error = (ModelEmptyError,)
rdagent/app/qlib_rd_loop/RDAgent.py
+19 -11
View File
@@ -1,4 +1,5 @@
import pickle
from rdagent.app.qlib_rd_loop.conf import PROP_SETTING
from rdagent.core.developer import Developer
from rdagent.core.exception import ModelEmptyError
@@ -12,6 +13,7 @@ from rdagent.core.scenario import Scenario
from rdagent.core.utils import import_class
from rdagent.log import rdagent_logger as logger
# TODO: we can design a workflow that can automatically save session and traceback in the future
class Model_RD_Agent:
def __init__(self):
@@ -20,50 +22,55 @@ class Model_RD_Agent:
self.hypothesis2experiment: Hypothesis2Experiment = import_class(PROP_SETTING.model_hypothesis2experiment)()
self.qlib_model_coder: Developer = import_class(PROP_SETTING.model_coder)(self.scen)
self.qlib_model_runner: Developer = import_class(PROP_SETTING.model_runner)(self.scen)
self.qlib_model_summarizer: HypothesisExperiment2Feedback = import_class(PROP_SETTING.model_summarizer)(self.scen)
self.qlib_model_summarizer: HypothesisExperiment2Feedback = import_class(PROP_SETTING.model_summarizer)(
self.scen
)
self.trace = Trace(scen=self.scen)
def generate_hypothesis(self):
hypothesis = self.hypothesis_gen.gen(self.trace)
self.dump_objects(hypothesis=hypothesis, trace=self.trace, filename='step_hypothesis.pkl')
self.dump_objects(hypothesis=hypothesis, trace=self.trace, filename="step_hypothesis.pkl")
return hypothesis
def convert_hypothesis(self, hypothesis):
exp = self.hypothesis2experiment.convert(hypothesis, self.trace)
self.dump_objects(exp=exp, hypothesis=hypothesis, trace=self.trace, filename='step_experiment.pkl')
self.dump_objects(exp=exp, hypothesis=hypothesis, trace=self.trace, filename="step_experiment.pkl")
return exp
def generate_code(self, exp):
exp = self.qlib_model_coder.develop(exp)
self.dump_objects(exp=exp, trace=self.trace, filename='step_code.pkl')
self.dump_objects(exp=exp, trace=self.trace, filename="step_code.pkl")
return exp
def run_experiment(self, exp):
exp = self.qlib_model_runner.develop(exp)
self.dump_objects(exp=exp, trace=self.trace, filename='step_run.pkl')
self.dump_objects(exp=exp, trace=self.trace, filename="step_run.pkl")
return exp
def generate_feedback(self, exp, hypothesis):
feedback = self.qlib_model_summarizer.generate_feedback(exp, hypothesis, self.trace)
self.dump_objects(exp=exp, hypothesis=hypothesis, feedback=feedback, trace=self.trace, filename='step_feedback.pkl')
self.dump_objects(
exp=exp, hypothesis=hypothesis, feedback=feedback, trace=self.trace, filename="step_feedback.pkl"
)
return feedback
def append_to_trace(self, hypothesis, exp, feedback):
self.trace.hist.append((hypothesis, exp, feedback))
self.dump_objects(trace=self.trace, filename='step_trace.pkl')
self.dump_objects(trace=self.trace, filename="step_trace.pkl")
def dump_objects(self, exp=None, hypothesis=None, feedback=None, trace=None, filename='dumped_objects.pkl'):
with open(filename, 'wb') as f:
def dump_objects(self, exp=None, hypothesis=None, feedback=None, trace=None, filename="dumped_objects.pkl"):
with open(filename, "wb") as f:
pickle.dump((exp, hypothesis, feedback, trace or self.trace), f)
def load_objects(self, filename):
with open(filename, 'rb') as f:
with open(filename, "rb") as f:
return pickle.load(f)
def process_steps(agent):
# Load trace if available
try:
_, _, _, trace = agent.load_objects('step_trace.pkl')
_, _, _, trace = agent.load_objects("step_trace.pkl")
agent.trace = trace
print(trace.get_sota_hypothesis_and_experiment())
except FileNotFoundError:
@@ -99,6 +106,7 @@ def process_steps(agent):
# # Step 6: Append to trace
# agent.append_to_trace(hypothesis, exp, feedback)
if __name__ == "__main__":
agent = Model_RD_Agent()
process_steps(agent)
rdagent/app/qlib_rd_loop/factor_from_report.py
+31 -22
View File
@@ -1,34 +1,38 @@
import json
from pathlib import Path
import pickle
from pathlib import Path
import pandas as pd
from dotenv import load_dotenv
from jinja2 import Environment, StrictUndefined
import pandas as pd
from rdagent.app.qlib_rd_loop.conf import PROP_SETTING
from rdagent.components.document_reader.document_reader import load_and_process_pdfs_by_langchain
from rdagent.components.document_reader.document_reader import (
load_and_process_pdfs_by_langchain,
)
from rdagent.core.developer import Developer
from rdagent.core.prompts import Prompts
from rdagent.core.proposal import (
Hypothesis,
Hypothesis2Experiment,
HypothesisExperiment2Feedback,
HypothesisGen,
Trace,
)
from rdagent.core.scenario import Scenario
from rdagent.core.utils import import_class
from rdagent.log import rdagent_logger as logger
from rdagent.oai.llm_utils import APIBackend
from rdagent.scenarios.qlib.developer.factor_coder import QlibFactorCoSTEER
from rdagent.scenarios.qlib.experiment.factor_experiment import QlibFactorScenario, QlibFactorExperiment
from rdagent.scenarios.qlib.experiment.factor_experiment import (
QlibFactorExperiment,
QlibFactorScenario,
)
from rdagent.scenarios.qlib.factor_experiment_loader.pdf_loader import (
FactorExperimentLoaderFromPDFfiles,
classify_report_from_dict,
)
from rdagent.core.proposal import (
Hypothesis2Experiment,
HypothesisExperiment2Feedback,
HypothesisGen,
Hypothesis,
Trace,
)
from rdagent.core.developer import Developer
assert load_dotenv()
scen: Scenario = import_class(PROP_SETTING.factor_scen)()
@@ -43,17 +47,21 @@ qlib_factor_runner: Developer = import_class(PROP_SETTING.factor_runner)(scen)
qlib_factor_summarizer: HypothesisExperiment2Feedback = import_class(PROP_SETTING.factor_summarizer)(scen)
with open(PROP_SETTING.report_result_json_file_path, 'r') as f:
with open(PROP_SETTING.report_result_json_file_path, "r") as f:
judge_pdf_data = json.load(f)
prompts_path = Path(__file__).parent / "prompts.yaml"
prompts = Prompts(file_path=prompts_path)
def generate_hypothesis(factor_result: dict, report_content: str) -> str:
system_prompt = Environment(undefined=StrictUndefined).from_string(prompts["hypothesis_generation"]["system"]).render()
user_prompt = Environment(undefined=StrictUndefined).from_string(prompts["hypothesis_generation"]["user"]).render(
factor_descriptions=json.dumps(factor_result),
report_content=report_content
system_prompt = (
Environment(undefined=StrictUndefined).from_string(prompts["hypothesis_generation"]["system"]).render()
)
user_prompt = (
Environment(undefined=StrictUndefined)
.from_string(prompts["hypothesis_generation"]["user"])
.render(factor_descriptions=json.dumps(factor_result), report_content=report_content)
)
response = APIBackend().build_messages_and_create_chat_completion(
@@ -68,16 +76,16 @@ def generate_hypothesis(factor_result: dict, report_content: str) -> str:
return Hypothesis(hypothesis=hypothesis_text, reason=reason_text)
def extract_factors_and_implement(report_file_path: str) -> tuple:
scenario = QlibFactorScenario()
with logger.tag("extract_factors_and_implement"):
with logger.tag("load_factor_tasks"):
exp = FactorExperimentLoaderFromPDFfiles().load(report_file_path)
if exp is None or exp.sub_tasks == []:
return None, None
docs_dict = load_and_process_pdfs_by_langchain(Path(report_file_path))
factor_result = {
@@ -85,7 +93,7 @@ def extract_factors_and_implement(report_file_path: str) -> tuple:
"description": task.factor_description,
"formulation": task.factor_formulation,
"variables": task.variables,
"resources": task.factor_resources
"resources": task.factor_resources,
}
for task in exp.sub_tasks
}
@@ -95,6 +103,7 @@ def extract_factors_and_implement(report_file_path: str) -> tuple:
return exp, hypothesis
trace = Trace(scen=scen)
for file_path, attributes in judge_pdf_data.items():
rdagent/app/qlib_rd_loop/factor_from_report_sh.py
+37 -25
View File
@@ -1,35 +1,40 @@
# TODO: we should have more advanced mechanism to handle such requirements for saving sessions.
import json
from pathlib import Path
import pickle
from pathlib import Path
import pandas as pd
from dotenv import load_dotenv
from jinja2 import Environment, StrictUndefined
import pandas as pd
from rdagent.app.qlib_rd_loop.conf import PROP_SETTING
from rdagent.components.document_reader.document_reader import extract_first_page_screenshot_from_pdf, load_and_process_pdfs_by_langchain
from rdagent.components.document_reader.document_reader import (
extract_first_page_screenshot_from_pdf,
load_and_process_pdfs_by_langchain,
)
from rdagent.core.developer import Developer
from rdagent.core.prompts import Prompts
from rdagent.core.proposal import (
Hypothesis,
Hypothesis2Experiment,
HypothesisExperiment2Feedback,
HypothesisGen,
Trace,
)
from rdagent.core.scenario import Scenario
from rdagent.core.utils import import_class
from rdagent.log import rdagent_logger as logger
from rdagent.oai.llm_utils import APIBackend
from rdagent.scenarios.qlib.developer.factor_coder import QlibFactorCoSTEER
from rdagent.scenarios.qlib.experiment.factor_experiment import QlibFactorScenario, QlibFactorExperiment
from rdagent.scenarios.qlib.experiment.factor_experiment import (
QlibFactorExperiment,
QlibFactorScenario,
)
from rdagent.scenarios.qlib.factor_experiment_loader.pdf_loader import (
FactorExperimentLoaderFromPDFfiles,
classify_report_from_dict,
)
from rdagent.core.proposal import (
Hypothesis2Experiment,
HypothesisExperiment2Feedback,
HypothesisGen,
Hypothesis,
Trace,
)
from rdagent.core.developer import Developer
assert load_dotenv()
scen: Scenario = import_class(PROP_SETTING.factor_scen)()
@@ -44,27 +49,33 @@ qlib_factor_runner: Developer = import_class(PROP_SETTING.factor_runner)(scen)
qlib_factor_summarizer: HypothesisExperiment2Feedback = import_class(PROP_SETTING.factor_summarizer)(scen)
with open(PROP_SETTING.report_result_json_file_path, 'r') as f:
with open(PROP_SETTING.report_result_json_file_path, "r") as f:
judge_pdf_data = json.load(f)
prompts_path = Path(__file__).parent / "prompts.yaml"
prompts = Prompts(file_path=prompts_path)
def save_progress(trace, current_index):
with open(PROP_SETTING.progress_file_path, "wb") as f:
pickle.dump((trace, current_index), f)
def load_progress():
if Path(PROP_SETTING.progress_file_path).exists():
with open(PROP_SETTING.progress_file_path, "rb") as f:
return pickle.load(f)
return Trace(scen=scen), 0
def generate_hypothesis(factor_result: dict, report_content: str) -> str:
system_prompt = Environment(undefined=StrictUndefined).from_string(prompts["hypothesis_generation"]["system"]).render()
user_prompt = Environment(undefined=StrictUndefined).from_string(prompts["hypothesis_generation"]["user"]).render(
factor_descriptions=json.dumps(factor_result),
report_content=report_content
system_prompt = (
Environment(undefined=StrictUndefined).from_string(prompts["hypothesis_generation"]["system"]).render()
)
user_prompt = (
Environment(undefined=StrictUndefined)
.from_string(prompts["hypothesis_generation"]["user"])
.render(factor_descriptions=json.dumps(factor_result), report_content=report_content)
)
response = APIBackend().build_messages_and_create_chat_completion(
@@ -79,12 +90,12 @@ def generate_hypothesis(factor_result: dict, report_content: str) -> str:
return Hypothesis(hypothesis=hypothesis_text, reason=reason_text)
def extract_factors_and_implement(report_file_path: str) -> tuple:
scenario = QlibFactorScenario()
with logger.tag("extract_factors_and_implement"):
with logger.tag("load_factor_tasks"):
exp = FactorExperimentLoaderFromPDFfiles().load(report_file_path)
if exp is None or exp.sub_tasks == []:
return None, None
@@ -100,7 +111,7 @@ def extract_factors_and_implement(report_file_path: str) -> tuple:
"description": task.factor_description,
"formulation": task.factor_formulation,
"variables": task.variables,
"resources": task.factor_resources
"resources": task.factor_resources,
}
for task in exp.sub_tasks
}
@@ -110,6 +121,7 @@ def extract_factors_and_implement(report_file_path: str) -> tuple:
return exp, hypothesis
trace, start_index = load_progress()
try:
@@ -122,7 +134,7 @@ try:
report_file_path = Path(file_path.replace(PROP_SETTING.origin_report_path, PROP_SETTING.local_report_path))
if report_file_path.exists():
logger.info(f"Processing {report_file_path}")
with logger.tag("r"):
exp, hypothesis = extract_factors_and_implement(str(report_file_path))
if exp is None:
@@ -132,7 +144,7 @@ try:
exp.based_experiments.append(QlibFactorExperiment(sub_tasks=[]))
logger.log_object(hypothesis, tag="hypothesis generation")
logger.log_object(exp.sub_tasks, tag="experiment generation")
with logger.tag("d"):
exp = qlib_factor_coder.develop(exp)
logger.log_object(exp.sub_workspace_list)
@@ -145,10 +157,10 @@ try:
logger.log_object(exp, tag="factor runner result")
feedback = qlib_factor_summarizer.generate_feedback(exp, hypothesis, trace)
logger.log_object(feedback, tag="feedback")
trace.hist.append((hypothesis, exp, feedback))
logger.info(f"Processed {report_file_path}: Result: {exp}")
# Save progress after processing each report
save_progress(trace, index + 1)
else:
rdagent/app/qlib_rd_loop/factor_w_sc.py
+1 -1
View File
@@ -30,4 +30,4 @@ def main(path=None, step_n=None):
if __name__ == "__main__":
fire.Fire(main)
fire.Fire(main)
rdagent/app/quant_factor_benchmark/analysis.py
+14 -15
View File
@@ -1,12 +1,14 @@
import json
import pickle
import pandas as pd
from pathlib import Path
import matplotlib.pyplot as plt
import pandas as pd
import seaborn as sns
from rdagent.components.benchmark.eval_method import FactorImplementEval
from rdagent.components.benchmark.conf import BenchmarkSettings
from rdagent.components.benchmark.eval_method import FactorImplementEval
class BenchmarkAnalyzer:
def __init__(self, settings):
@@ -25,10 +27,10 @@ class BenchmarkAnalyzer:
file_path = Path(file_path)
if not (file_path.is_file() and file_path.suffix == ".pkl"):
raise ValueError("Invalid file path")
with file_path.open("rb") as f:
res = pickle.load(f)
return res
def process_results(self, results):
@@ -39,7 +41,7 @@ class BenchmarkAnalyzer:
processed_data = self.analyze_data(summarized_data)
final_res[experiment] = processed_data.iloc[-1, :]
return final_res
def reformat_succ_rate(self, display_df):
new_idx = []
display_df = display_df[display_df.index.isin(self.index_map.keys())]
@@ -52,8 +54,10 @@ class BenchmarkAnalyzer:
)
display_df = display_df.swaplevel(0, 2).swaplevel(0, 1).sort_index(axis=0)
return display_df.sort_index(key=lambda x: [{"Easy": 0, "Medium": 1, "Hard": 2, "New Discovery": 3}.get(i, i) for i in x])
return display_df.sort_index(
key=lambda x: [{"Easy": 0, "Medium": 1, "Hard": 2, "New Discovery": 3}.get(i, i) for i in x]
)
def result_all_key_order(self, x):
order_v = []
for i in x:
@@ -92,9 +96,7 @@ class BenchmarkAnalyzer:
sum_df_clean["FactorRowCountEvaluator"]
format_issue = (
sum_df_clean["FactorRowCountEvaluator"] & sum_df_clean["FactorIndexEvaluator"]
)
format_issue = sum_df_clean["FactorRowCountEvaluator"] & sum_df_clean["FactorIndexEvaluator"]
eval_series = format_issue.unstack()
succ_rate = eval_series.T.fillna(False).astype(bool) # false indicate failure
format_succ_rate = succ_rate.mean(axis=0).to_frame("success rate")
@@ -113,10 +115,7 @@ class BenchmarkAnalyzer:
value_max_res = self.reformat_succ_rate(value_max)
value_avg = (
(sum_df_clean["FactorMissingValuesEvaluator"] * format_issue)
.unstack()
.T.mean(axis=0)
.to_frame("avg_value")
(sum_df_clean["FactorMissingValuesEvaluator"] * format_issue).unstack().T.mean(axis=0).to_frame("avg_value")
)
value_avg_res = self.reformat_succ_rate(value_avg)
@@ -148,7 +147,6 @@ class BenchmarkAnalyzer:
return df_w_mean
class Plotter:
@staticmethod
def change_fs(font_size):
@@ -169,6 +167,7 @@ class Plotter:
plt.title("Comparison of Different Methods")
plt.savefig(file_name)
if __name__ == "__main__":
settings = BenchmarkSettings()
benchmark = BenchmarkAnalyzer(settings)
rdagent/app/quant_factor_benchmark/eval.py
+9 -12
View File
@@ -1,23 +1,20 @@
import os
from pathlib import Path
import pickle
import time
from pathlib import Path
from pprint import pprint
from rdagent.app.qlib_rd_loop.conf import PROP_SETTING
from rdagent.components.benchmark.conf import BenchmarkSettings
from rdagent.components.benchmark.eval_method import FactorImplementEval
from rdagent.core.scenario import Scenario
from rdagent.core.utils import import_class
from rdagent.log import rdagent_logger as logger
from rdagent.scenarios.qlib.experiment.factor_experiment import QlibFactorScenario
from rdagent.scenarios.qlib.factor_experiment_loader.json_loader import (
FactorTestCaseLoaderFromJsonFile,
)
from rdagent.components.benchmark.conf import BenchmarkSettings
from rdagent.components.benchmark.eval_method import FactorImplementEval
from rdagent.core.utils import import_class
from rdagent.core.utils import import_class
from rdagent.core.scenario import Scenario
from rdagent.scenarios.qlib.experiment.factor_experiment import QlibFactorScenario
from pprint import pprint
# 1.read the settings
bs = BenchmarkSettings()
@@ -28,7 +25,7 @@ test_cases = FactorTestCaseLoaderFromJsonFile().load(bs.bench_data_path)
scen: Scenario = import_class(PROP_SETTING.factor_scen)()
generate_method = import_class(bs.bench_method_cls)(scen=scen)
# 4.declare the eval method and pass the arguments.
eval_method = FactorImplementEval(
method=generate_method,
rdagent/components/benchmark/eval_method.py
+1 -2
View File
@@ -19,18 +19,17 @@ from rdagent.components.coder.factor_coder.CoSTEER.evaluators import (
from rdagent.components.coder.factor_coder.factor import FactorFBWorkspace
from rdagent.core.conf import RD_AGENT_SETTINGS
from rdagent.core.developer import Developer
from rdagent.core.exception import CoderException, RunnerException
from rdagent.core.experiment import Task, Workspace
from rdagent.core.scenario import Scenario
from rdagent.core.utils import multiprocessing_wrapper
EVAL_RES = Dict[
str,
List[Tuple[FactorEvaluator, Union[object, RunnerException]]],
]
class TestCase:
def __init__(
self,
rdagent/components/coder/factor_coder/CoSTEER/evaluators.py
+10 -4
View File
@@ -356,7 +356,9 @@ class FactorValueEvaluator(FactorEvaluator):
# Check if both dataframe have the same rows count
if gt_implementation is not None:
feedback_str, single_column_result = FactorRowCountEvaluator(self.scen).evaluate(implementation, gt_implementation)
feedback_str, single_column_result = FactorRowCountEvaluator(self.scen).evaluate(
implementation, gt_implementation
)
conclusions.append(feedback_str)
feedback_str, same_index_result = FactorIndexEvaluator(self.scen).evaluate(
@@ -364,7 +366,9 @@ class FactorValueEvaluator(FactorEvaluator):
)
conclusions.append(feedback_str)
feedback_str, output_format_result = FactorMissingValuesEvaluator(self.scen).evaluate(implementation, gt_implementation)
feedback_str, output_format_result = FactorMissingValuesEvaluator(self.scen).evaluate(
implementation, gt_implementation
)
conclusions.append(feedback_str)
feedback_str, equal_value_ratio_result = FactorEqualValueCountEvaluator(self.scen).evaluate(
@@ -464,8 +468,10 @@ class FactorFinalDecisionEvaluator(Evaluator):
except KeyError as e:
attempts += 1
if attempts >= max_attempts:
raise KeyError("Response from API is missing 'final_decision' or 'final_feedback' key after multiple attempts.") from e
raise KeyError(
"Response from API is missing 'final_decision' or 'final_feedback' key after multiple attempts."
) from e
return None, None
rdagent/components/coder/factor_coder/CoSTEER/evolving_strategy.py
+6 -2
View File
@@ -68,8 +68,12 @@ class MultiProcessEvolvingStrategy(EvolvingStrategy):
# if the number of factors to be implemented is larger than the limit, we need to select some of them
if FACTOR_IMPLEMENT_SETTINGS.select_ratio < 1:
# if the number of loops is equal to the select_loop, we need to select some of them
implementation_factors_per_round = round(FACTOR_IMPLEMENT_SETTINGS.select_ratio * len(to_be_finished_task_index) + 0.5) # ceilling
implementation_factors_per_round = min(implementation_factors_per_round, len(to_be_finished_task_index)) # but not exceed the total number of tasks
implementation_factors_per_round = round(
FACTOR_IMPLEMENT_SETTINGS.select_ratio * len(to_be_finished_task_index) + 0.5
) # ceilling
implementation_factors_per_round = min(
implementation_factors_per_round, len(to_be_finished_task_index)
) # but not exceed the total number of tasks
if FACTOR_IMPLEMENT_SETTINGS.select_method == "random":
to_be_finished_task_index = RandomSelect(
rdagent/components/coder/factor_coder/factor.py
+1 -5
View File
@@ -10,11 +10,7 @@ import pandas as pd
from filelock import FileLock
from rdagent.components.coder.factor_coder.config import FACTOR_IMPLEMENT_SETTINGS
from rdagent.core.exception import (
CodeFormatError,
NoOutputError,
CustomRuntimeError,
)
from rdagent.core.exception import CodeFormatError, CustomRuntimeError, NoOutputError
from rdagent.core.experiment import Experiment, FBWorkspace, Task
from rdagent.log import rdagent_logger as logger
from rdagent.oai.llm_utils import md5_hash
rdagent/components/coder/model_coder/model.py
+16 -7
View File
@@ -16,7 +16,14 @@ from rdagent.utils import get_module_by_module_path
class ModelTask(Task):
def __init__(
self, name: str, description: str, formulation: str, architecture: str, variables: Dict[str, str], hyperparameters: Dict[str, str], model_type: Optional[str] = None
self,
name: str,
description: str,
formulation: str,
architecture: str,
variables: Dict[str, str],
hyperparameters: Dict[str, str],
model_type: Optional[str] = None,
) -> None:
self.name: str = name
self.description: str = description
@@ -24,7 +31,9 @@ class ModelTask(Task):
self.architecture: str = architecture
self.variables: str = variables
self.hyperparameters: str = hyperparameters
self.model_type: str = model_type # Tabular for tabular model, TimesSeries for time series model, Graph for graph model
self.model_type: str = (
model_type # Tabular for tabular model, TimesSeries for time series model, Graph for graph model
)
def get_task_information(self):
return f"""name: {self.name}
@@ -107,21 +116,21 @@ class ModelFBWorkspace(FBWorkspace):
# Initialize all parameters of `m` to `param_init_value`
for _, param in m.named_parameters():
param.data.fill_(param_init_value)
# Execute the model
if self.target_task.model_type == "Graph":
out = m(*data)
else:
out = m(data)
execution_model_output = out.cpu().detach()
execution_feedback_str = f"Execution successful, output tensor shape: {execution_model_output.shape}"
if MODEL_IMPL_SETTINGS.enable_execution_cache:
pickle.dump((execution_feedback_str, execution_model_output), open(cache_file_path, "wb"))
return execution_feedback_str, execution_model_output
except Exception as e:
return f"Execution error: {e}", None
rdagent/components/document_reader/document_reader.py
+6 -4
View File
@@ -1,16 +1,17 @@
from __future__ import annotations
from pathlib import Path
import fitz
from PIL import Image
from typing import TYPE_CHECKING
import fitz
from azure.ai.formrecognizer import DocumentAnalysisClient
from azure.core.credentials import AzureKeyCredential
from langchain.document_loaders import PyPDFDirectoryLoader, PyPDFLoader
from PIL import Image
if TYPE_CHECKING:
from langchain_core.documents import Document
from rdagent.core.conf import RD_AGENT_SETTINGS
@@ -99,10 +100,11 @@ def load_and_process_pdfs_by_azure_document_intelligence(path: Path) -> dict[str
)
return content_dict
def extract_first_page_screenshot_from_pdf(pdf_path: Path) -> Image:
doc = fitz.open(pdf_path)
page = doc.load_page(0)
pix = page.get_pixmap()
image = Image.frombytes("RGB", [pix.width, pix.height], pix.samples)
return image
return image
rdagent/components/proposal/model_proposal.py
+1 -1
View File
@@ -15,11 +15,11 @@ from rdagent.core.proposal import (
)
from rdagent.oai.llm_utils import APIBackend
ModelHypothesis = Hypothesis
prompt_dict = Prompts(file_path=Path(__file__).parent / "prompts.yaml")
class ModelHypothesisGen(HypothesisGen):
prompts: Prompts = prompt_dict
rdagent/components/workflow/conf.py
+2
View File
@@ -1,5 +1,6 @@
from pydantic_settings import BaseSettings
class BasePropSetting(BaseSettings):
"""
The common part of the config for RD Loop to propose and developement
@@ -11,6 +12,7 @@ class BasePropSetting(BaseSettings):
env_prefix = "DM_MODEL_" # Use MODEL_CODER_ as prefix for environment variables
protected_namespaces = () # Add 'model_' to the protected namespaces
"""
scen: str = ""
hypothesis_gen: str = ""
hypothesis2experiment: str = ""
rdagent/components/workflow/rd_loop.py
+3 -3
View File
@@ -4,6 +4,7 @@ It is from `rdagent/app/qlib_rd_loop/model.py` and try to replace `rdagent/app/q
"""
from typing import Any
from rdagent.components.workflow.conf import BasePropSetting
from rdagent.core.developer import Developer
from rdagent.core.proposal import (
@@ -15,11 +16,10 @@ from rdagent.core.proposal import (
from rdagent.core.scenario import Scenario
from rdagent.core.utils import import_class
from rdagent.log import rdagent_logger as logger
from rdagent.utils.workflow import LoopBase, LoopMeta
from rdagent.utils.workflow import LoopMeta, LoopBase
class RDLoop(LoopBase, metaclass=LoopMeta):
def __init__(self, PROP_SETTING: BasePropSetting):
scen: Scenario = import_class(PROP_SETTING.scen)()
@@ -62,4 +62,4 @@ class RDLoop(LoopBase, metaclass=LoopMeta):
feedback = self.summarizer.generate_feedback(prev_out["running"], prev_out["propose"], self.trace)
with logger.tag("ef"): # evaluate and feedback
logger.log_object(feedback, tag="feedback")
self.trace.hist.append((prev_out["propose"],prev_out["running"] , feedback))
self.trace.hist.append((prev_out["propose"], prev_out["running"], feedback))
rdagent/core/evolving_agent.py
+7 -4
View File
@@ -25,14 +25,16 @@ class EvoAgent(ABC):
evo: EvolvableSubjects,
eva: Evaluator | Feedback,
filter_final_evo: bool = False,
) -> EvolvableSubjects: ...
) -> EvolvableSubjects:
...
@abstractmethod
def filter_evolvable_subjects_by_feedback(
self,
evo: EvolvableSubjects,
feedback: Feedback | None,
) -> EvolvableSubjects: ...
) -> EvolvableSubjects:
...
class RAGEvoAgent(EvoAgent):
@@ -52,7 +54,6 @@ class RAGEvoAgent(EvoAgent):
self.with_feedback = with_feedback
self.knowledge_self_gen = knowledge_self_gen
def multistep_evolve(
self,
evo: EvolvableSubjects,
@@ -86,7 +87,9 @@ class RAGEvoAgent(EvoAgent):
es.feedback = (
# TODO: Due to the irregular design of rdagent.core.evaluation.Evaluator,
# it fails mypy's test here, so we'll ignore this error for now.
eva if isinstance(eva, Feedback) else eva.evaluate(evo, queried_knowledge=queried_knowledge) # type: ignore[arg-type, call-arg]
eva
if isinstance(eva, Feedback)
else eva.evaluate(evo, queried_knowledge=queried_knowledge) # type: ignore[arg-type, call-arg]
)
logger.log_object(es.feedback, tag="evolving feedback")
rdagent/core/proposal.py
+5 -5
View File
@@ -99,12 +99,12 @@ class HypothesisGen(ABC):
# def gen(self, scenario_desc: str, ) -> Hypothesis:
"""
Motivation of the variable `scenario_desc`:
- Mocking a data-scientist is observing the scenario.
- Mocking a data-scientist is observing the scenario.
scenario_desc may conclude:
- data observation:
- Original or derivative
- Task information:
scenario_desc may include:
- data observation:
- Original or derivative
- Task information:
"""
rdagent/core/utils.py
+1 -1
View File
@@ -18,6 +18,7 @@ class SingletonBaseClass:
Because we try to support defining Singleton with `class A(SingletonBaseClass)`
instead of `A(metaclass=SingletonMeta)` this class becomes necessary.
"""
_instance_dict: ClassVar[dict] = {}
def __new__(cls, *args: Any, **kwargs: Any) -> Any:
@@ -34,7 +35,6 @@ class SingletonBaseClass:
return cls._instance_dict[kwargs_hash]
def parse_json(response: str) -> Any:
try:
return json.loads(response)
rdagent/log/base.py
+8 -4
View File
@@ -2,19 +2,22 @@ from __future__ import annotations
from abc import abstractmethod
from collections.abc import Generator
from dataclasses import dataclass
from datetime import datetime
from pathlib import Path
from typing import Literal, Optional, Union, Literal
from dataclasses import dataclass
from typing import Literal, Optional, Union
@dataclass
class Message:
"""The info unit of the storage"""
tag: str # namespace like like a.b.c
level: Literal["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"] # The level of the logging
timestamp: datetime # The time when the message is generated
caller: Optional[str] # The caller of the logging like `rdagent.oai.llm_utils:_create_chat_completion_inner_function:55`(file:func:line)
caller: Optional[
str
] # The caller of the logging like `rdagent.oai.llm_utils:_create_chat_completion_inner_function:55`(file:func:line)
pid_trace: Optional[str] # The process id trace; A-B-C represents A create B, B create C
content: object # The content
@@ -39,7 +42,8 @@ class Storage:
@abstractmethod
def log(
self,
obj: object, name: str = "",
obj: object,
name: str = "",
save_type: Literal["json", "text", "pkl"] = "text",
timestamp: datetime | None = None,
**kwargs: dict,
rdagent/log/logger.py
+4 -2
View File
@@ -3,15 +3,17 @@ import sys
from contextlib import contextmanager
from datetime import datetime, timezone
from functools import partial
from logging import LogRecord
from multiprocessing import Pipe
from multiprocessing.connection import Connection
from pathlib import Path
from typing import TYPE_CHECKING, Union, Generator, Dict, Any
from logging import LogRecord
from typing import TYPE_CHECKING, Any, Dict, Generator, Union
from loguru import logger
if TYPE_CHECKING:
from loguru import Record
from psutil import Process
from rdagent.core.conf import RD_AGENT_SETTINGS
rdagent/log/storage.py
+5 -12
View File
@@ -1,9 +1,9 @@
import re
import json
import pickle
import re
from datetime import datetime, timezone
from pathlib import Path
from typing import Literal, Generator, Union, Any, cast
from typing import Any, Generator, Literal, Union, cast
from .base import Message, Storage
@@ -17,7 +17,6 @@ class FileStorage(Storage):
TODO: describe the storage format
"""
def __init__(self, path: str | Path = "./log/") -> None:
self.path = Path(path)
self.path.mkdir(parents=True, exist_ok=True)
@@ -69,7 +68,7 @@ class FileStorage(Storage):
def iter_msg(self, watch: bool = False) -> Generator[Message, None, None]:
msg_l = []
for file in self.path.glob("**/*.log"):
tag = '.'.join(str(file.relative_to(self.path)).replace("/", ".").split(".")[:-3])
tag = ".".join(str(file.relative_to(self.path)).replace("/", ".").split(".")[:-3])
pid = file.parent.name
with file.open("r") as f:
@@ -92,12 +91,7 @@ class FileStorage(Storage):
message_content = content[message_start:message_end].strip()
m = Message(
tag=tag,
level=level,
timestamp=timestamp,
caller=caller,
pid_trace=pid,
content=message_content
tag=tag, level=level, timestamp=timestamp, caller=caller, pid_trace=pid, content=message_content
)
if isinstance(m.content, str) and "Logging object in" in m.content:
@@ -119,7 +113,6 @@ class FileStorage(Storage):
def truncate(self, time: datetime) -> None:
# any message later than `time` will be removed
for file in self.path.glob("**/*.log"):
with file.open("r") as f:
content = f.read()
@@ -135,7 +128,7 @@ class FileStorage(Storage):
log_start = match.start()
log_end = next_match.start() if next_match else len(content)
msg = content[match.end():log_end].strip()
msg = content[match.end() : log_end].strip()
if timestamp > time:
if "Logging object in" in msg:
rdagent/log/ui/app.py
+10 -4
View File
@@ -1,9 +1,15 @@
from rdagent.log.ui.web import WebView, SimpleTraceWindow, TraceObjWindow, mock_msg, TraceWindow
from rdagent.log.storage import FileStorage, Message
from rdagent.core.proposal import Trace
from pathlib import Path
import pickle
from pathlib import Path
from rdagent.core.proposal import Trace
from rdagent.log.storage import FileStorage, Message
from rdagent.log.ui.web import (
SimpleTraceWindow,
TraceObjWindow,
TraceWindow,
WebView,
mock_msg,
)
# show logs folder
WebView(TraceWindow()).display(FileStorage("/data/home/bowen/workspace/RD-Agent/log/yuante/2024-07-24_04-03-33-691119"))
rdagent/log/ui/st_fixed_container.py
+1 -1
View File
@@ -123,4 +123,4 @@ if __name__ == "__main__":
st.container(border=True).write("This is a regular container.")
for i in range(30):
st.write(f"Line {i}")
st.write(f"Line {i}")
rdagent/log/ui/web.py
+218 -192
View File
@@ -1,51 +1,45 @@
import pandas as pd
import streamlit as st
import plotly.express as px
import time
from rdagent.log.base import Storage, View
from rdagent.log.base import Message
from datetime import timezone, datetime
from collections import defaultdict
from copy import deepcopy
from rdagent.core.proposal import Trace
from datetime import datetime, timezone
from typing import Callable, Type
from streamlit.delta_generator import DeltaGenerator
from rdagent.core.proposal import Hypothesis, HypothesisFeedback
import pandas as pd
import plotly.express as px
import streamlit as st
from streamlit.delta_generator import DeltaGenerator
from rdagent.components.coder.factor_coder.CoSTEER.evaluators import (
FactorSingleFeedback,
)
from rdagent.components.coder.factor_coder.factor import FactorFBWorkspace, FactorTask
from rdagent.components.coder.model_coder.CoSTEER.evaluators import ModelCoderFeedback
from rdagent.components.coder.model_coder.model import ModelFBWorkspace, ModelTask
from rdagent.core.proposal import Hypothesis, HypothesisFeedback, Trace
from rdagent.log.base import Message, Storage, View
from rdagent.scenarios.qlib.experiment.factor_experiment import QlibFactorExperiment
from rdagent.scenarios.qlib.experiment.model_experiment import QlibModelExperiment
from rdagent.components.coder.factor_coder.factor import FactorTask, FactorFBWorkspace
from rdagent.components.coder.factor_coder.CoSTEER.evaluators import FactorSingleFeedback
from rdagent.components.coder.model_coder.CoSTEER.evaluators import ModelCoderFeedback
from rdagent.components.coder.model_coder.model import ModelTask, ModelFBWorkspace
st.set_page_config(layout="wide")
TIME_DELAY = 0.001
class WebView(View):
def __init__(self, ui: 'StWindow'):
class WebView(View):
def __init__(self, ui: "StWindow"):
self.ui = ui
# Save logs to your desired data structure
# ...
def display(self, s: Storage, watch: bool = False):
for msg in s.iter_msg(): # iterate overtime
# NOTE: iter_msg will correctly seperate the information.
# TODO: msg may support streaming mode.
self.ui.consume_msg(msg)
class StWindow:
def __init__(self, container: 'DeltaGenerator'):
def __init__(self, container: "DeltaGenerator"):
self.container = container
def consume_msg(self, msg: Message):
@@ -54,31 +48,32 @@ class StWindow:
class LLMWindow(StWindow):
def __init__(self, container: 'DeltaGenerator', session_name: str="common"):
def __init__(self, container: "DeltaGenerator", session_name: str = "common"):
self.session_name = session_name
self.container = container.expander(f"{self.session_name} message")
def consume_msg(self, msg: Message):
self.container.chat_message('user').markdown(f"{msg.content}")
self.container.chat_message("user").markdown(f"{msg.content}")
class ProgressTabsWindow(StWindow):
'''
"""
For windows with stream messages, will refresh when a new tab is created.
'''
def __init__(self,
container: 'DeltaGenerator',
inner_class: Type[StWindow] = StWindow,
mapper: Callable[[Message], str] = lambda x: x.pid_trace):
"""
def __init__(
self,
container: "DeltaGenerator",
inner_class: Type[StWindow] = StWindow,
mapper: Callable[[Message], str] = lambda x: x.pid_trace,
):
self.inner_class = inner_class
self.mapper = mapper
self.container = container.empty()
self.tab_windows: dict[str, StWindow] = defaultdict(None)
self.tab_caches: dict[str, list[Message]] = defaultdict(list)
def consume_msg(self, msg: Message):
name = self.mapper(msg)
@@ -93,61 +88,67 @@ class ProgressTabsWindow(StWindow):
for id, name in enumerate(names):
self.tab_windows[name] = self.inner_class(tabs[id])
# consume the cache
for name in self.tab_caches:
for msg in self.tab_caches[name]:
self.tab_windows[name].consume_msg(msg)
self.tab_caches[name].append(msg)
self.tab_windows[name].consume_msg(msg)
class ObjectsTabsWindow(StWindow):
def __init__(self,
container: 'DeltaGenerator',
inner_class: Type[StWindow] = StWindow,
mapper: Callable[[object], str] = lambda x: str(x),
tab_names: list[str] | None = None):
def __init__(
self,
container: "DeltaGenerator",
inner_class: Type[StWindow] = StWindow,
mapper: Callable[[object], str] = lambda x: str(x),
tab_names: list[str] | None = None,
):
self.inner_class = inner_class
self.mapper = mapper
self.container = container
self.tab_names = tab_names
def consume_msg(self, msg: Message):
if isinstance(msg.content, list):
if self.tab_names:
assert len(self.tab_names) == len(msg.content), "List of objects should have the same length as provided tab names."
assert len(self.tab_names) == len(
msg.content
), "List of objects should have the same length as provided tab names."
objs_dict = {self.tab_names[id]: obj for id, obj in enumerate(msg.content)}
else:
objs_dict = {self.mapper(obj): obj for obj in msg.content}
elif not isinstance(msg.content, dict):
raise ValueError("Message content should be a list or a dict of objects.")
# two many tabs may cause display problem
tab_names = list(objs_dict.keys())
tabs = []
for i in range(0, len(tab_names), 10):
tabs.extend(self.container.tabs(tab_names[i:i+10]))
tabs.extend(self.container.tabs(tab_names[i : i + 10]))
for id, obj in enumerate(objs_dict.values()):
splited_msg = Message(tag=msg.tag,
level=msg.level,
timestamp=msg.timestamp,
caller=msg.caller,
pid_trace=msg.pid_trace,
content=obj)
splited_msg = Message(
tag=msg.tag,
level=msg.level,
timestamp=msg.timestamp,
caller=msg.caller,
pid_trace=msg.pid_trace,
content=obj,
)
self.inner_class(tabs[id]).consume_msg(splited_msg)
class RoundTabsWindow(StWindow):
def __init__(self,
container: 'DeltaGenerator',
new_tab_func: Callable[[Message], bool],
inner_class: Type[StWindow] = StWindow,
title: str = 'Round tabs'):
def __init__(
self,
container: "DeltaGenerator",
new_tab_func: Callable[[Message], bool],
inner_class: Type[StWindow] = StWindow,
title: str = "Round tabs",
):
container.markdown(f"### **{title}**")
self.inner_class = inner_class
self.new_tab_func = new_tab_func
@@ -156,42 +157,42 @@ class RoundTabsWindow(StWindow):
self.current_win = StWindow(container)
self.tabs_c = container.empty()
def consume_msg(self, msg: Message):
if self.new_tab_func(msg):
self.round += 1
self.current_win = self.inner_class(self.tabs_c.tabs([str(i) for i in range(1, self.round+1)])[-1])
self.current_win = self.inner_class(self.tabs_c.tabs([str(i) for i in range(1, self.round + 1)])[-1])
self.current_win.consume_msg(msg)
class HypothesisWindow(StWindow):
def consume_msg(self, msg: Message | Hypothesis):
h: Hypothesis = msg.content if isinstance(msg, Message) else msg
self.container.markdown('#### **Hypothesis💡**')
self.container.markdown(f"""
self.container.markdown("#### **Hypothesis💡**")
self.container.markdown(
f"""
- **Hypothesis**: {h.hypothesis}
- **Reason**: {h.reason}""")
- **Reason**: {h.reason}"""
)
class HypothesisFeedbackWindow(StWindow):
def consume_msg(self, msg: Message | HypothesisFeedback):
h: HypothesisFeedback = msg.content if isinstance(msg, Message) else msg
self.container.markdown('#### **Hypothesis Feedback🔍**')
self.container.markdown(f"""
self.container.markdown("#### **Hypothesis Feedback🔍**")
self.container.markdown(
f"""
- **Observations**: {h.observations}
- **Hypothesis Evaluation**: {h.hypothesis_evaluation}
- **New Hypothesis**: {h.new_hypothesis}
- **Decision**: {h.decision}
- **Reason**: {h.reason}""")
- **Reason**: {h.reason}"""
)
class FactorTaskWindow(StWindow):
def consume_msg(self, msg: Message | FactorTask):
ft: FactorTask = msg.content if isinstance(msg, Message) else msg
@@ -199,14 +200,13 @@ class FactorTaskWindow(StWindow):
self.container.markdown(f"**Description**: {ft.factor_description}")
self.container.latex(f"Formulation: {ft.factor_formulation}")
variables_df = pd.DataFrame(ft.variables, index=['Description']).T
variables_df.index.name = 'Variable'
variables_df = pd.DataFrame(ft.variables, index=["Description"]).T
variables_df.index.name = "Variable"
self.container.table(variables_df)
self.container.text(f"Factor resources: {ft.factor_resources}")
class ModelTaskWindow(StWindow):
def consume_msg(self, msg: Message | ModelTask):
mt: ModelTask = msg.content if isinstance(msg, Message) else msg
@@ -214,18 +214,18 @@ class ModelTaskWindow(StWindow):
self.container.markdown(f"**Model Type**: {mt.model_type}")
self.container.markdown(f"**Description**: {mt.description}")
self.container.markdown(f"**Formulation**: {mt.formulation}")
variables_df = pd.DataFrame(mt.variables, index=['Value']).T
variables_df.index.name = 'Variable'
variables_df = pd.DataFrame(mt.variables, index=["Value"]).T
variables_df.index.name = "Variable"
self.container.table(variables_df)
class FactorFeedbackWindow(StWindow):
def consume_msg(self, msg: Message | FactorSingleFeedback):
fb: FactorSingleFeedback = msg.content if isinstance(msg, Message) else msg
self.container.markdown(f"""### :blue[Factor Execution Feedback]
self.container.markdown(
f"""### :blue[Factor Execution Feedback]
{fb.execution_feedback}
### :blue[Factor Code Feedback]
{fb.code_feedback}
@@ -235,15 +235,16 @@ class FactorFeedbackWindow(StWindow):
{fb.final_feedback}
### :blue[Factor Final Decision]
This implementation is {'SUCCESS' if fb.final_decision else 'FAIL'}.
""")
"""
)
class ModelFeedbackWindow(StWindow):
def consume_msg(self, msg: Message | ModelCoderFeedback):
mb: ModelCoderFeedback = msg.content if isinstance(msg, Message) else msg
self.container.markdown(f"""### :blue[Model Execution Feedback]
self.container.markdown(
f"""### :blue[Model Execution Feedback]
{mb.execution_feedback}
### :blue[Model Shape Feedback]
{mb.shape_feedback}
@@ -255,11 +256,12 @@ class ModelFeedbackWindow(StWindow):
{mb.final_feedback}
### :blue[Model Final Decision]
This implementation is {'SUCCESS' if mb.final_decision else 'FAIL'}.
""")
"""
)
class WorkspaceWindow(StWindow):
def __init__(self, container: 'DeltaGenerator', show_task_info: bool = False):
def __init__(self, container: "DeltaGenerator", show_task_info: bool = False):
self.container = container
self.show_task_info = show_task_info
@@ -267,21 +269,22 @@ class WorkspaceWindow(StWindow):
ws: FactorFBWorkspace | ModelFBWorkspace = msg.content if isinstance(msg, Message) else msg
# no workspace
if ws is None: return
if ws is None:
return
# task info
if self.show_task_info:
task_msg = deepcopy(msg)
task_msg.content = ws.target_task
if isinstance(ws, FactorFBWorkspace):
self.container.subheader('Factor Info')
self.container.subheader("Factor Info")
FactorTaskWindow(self.container.container()).consume_msg(task_msg)
else:
self.container.subheader('Model Info')
self.container.subheader("Model Info")
ModelTaskWindow(self.container.container()).consume_msg(task_msg)
# task codes
for k,v in ws.code_dict.items():
for k, v in ws.code_dict.items():
self.container.markdown(f"`{k}`")
self.container.code(v, language="python")
@@ -302,24 +305,25 @@ class QlibFactorExpWindow(StWindow):
if self.show_task_info:
ftm_msg = deepcopy(msg)
ftm_msg.content = [ws for ws in exp.sub_workspace_list if ws]
self.container.markdown('**Factor Tasks**')
ObjectsTabsWindow(self.container.container(),
inner_class=WorkspaceWindow,
mapper=lambda x: x.target_task.factor_name,
).consume_msg(ftm_msg)
self.container.markdown("**Factor Tasks**")
ObjectsTabsWindow(
self.container.container(),
inner_class=WorkspaceWindow,
mapper=lambda x: x.target_task.factor_name,
).consume_msg(ftm_msg)
# result
self.container.markdown('**Results**')
results = pd.DataFrame({f'base_exp_{id}':e.result for id, e in enumerate(exp.based_experiments)})
results['now'] = exp.result
self.container.markdown("**Results**")
results = pd.DataFrame({f"base_exp_{id}": e.result for id, e in enumerate(exp.based_experiments)})
results["now"] = exp.result
self.container.expander('results table').table(results)
self.container.expander("results table").table(results)
try:
bar_chart = px.bar(results, orientation='h', barmode='group')
self.container.expander('results chart').plotly_chart(bar_chart)
bar_chart = px.bar(results, orientation="h", barmode="group")
self.container.expander("results chart").plotly_chart(bar_chart)
except:
self.container.text('Results are incomplete.')
self.container.text("Results are incomplete.")
class QlibModelExpWindow(StWindow):
@@ -334,44 +338,45 @@ class QlibModelExpWindow(StWindow):
if self.show_task_info:
_msg = deepcopy(msg)
_msg.content = [ws for ws in exp.sub_workspace_list if ws]
self.container.markdown('**Model Tasks**')
ObjectsTabsWindow(self.container.container(),
inner_class=WorkspaceWindow,
mapper=lambda x: x.target_task.name,
).consume_msg(_msg)
self.container.markdown("**Model Tasks**")
ObjectsTabsWindow(
self.container.container(),
inner_class=WorkspaceWindow,
mapper=lambda x: x.target_task.name,
).consume_msg(_msg)
# result
self.container.subheader('Results', divider=True)
results = pd.DataFrame({f'base_exp_{id}':e.result for id, e in enumerate(exp.based_experiments)})
results['now'] = exp.result
self.container.subheader("Results", divider=True)
results = pd.DataFrame({f"base_exp_{id}": e.result for id, e in enumerate(exp.based_experiments)})
results["now"] = exp.result
self.container.expander('results table').table(results)
self.container.expander("results table").table(results)
class SimpleTraceWindow(StWindow):
def __init__(self, container: 'DeltaGenerator' = st.container(), show_llm: bool = False, show_common_logs: bool = False):
def __init__(
self, container: "DeltaGenerator" = st.container(), show_llm: bool = False, show_common_logs: bool = False
):
super().__init__(container)
self.show_llm = show_llm
self.show_common_logs = show_common_logs
self.pid_trace = ''
self.current_tag = ''
self.pid_trace = ""
self.current_tag = ""
self.current_win = StWindow(self.container)
self.evolving_tasks: list[str] = []
def consume_msg(self, msg: Message):
# divide tag levels
if len(msg.tag) > len(self.current_tag):
# write a header about current task, if it is llm message, not write.
if not msg.tag.endswith('llm_messages'):
self.container.header(msg.tag.replace('.', ' ➡ '), divider=True)
if not msg.tag.endswith("llm_messages"):
self.container.header(msg.tag.replace(".", " ➡ "), divider=True)
self.current_tag = msg.tag
# set log writer (window) according to msg
if msg.tag.endswith('llm_messages'):
if msg.tag.endswith("llm_messages"):
# llm messages logs
if not self.show_llm:
return
@@ -392,29 +397,41 @@ class SimpleTraceWindow(StWindow):
if len(msg.content) == 0:
return
if isinstance(msg.content[0], FactorTask):
self.current_win = ObjectsTabsWindow(self.container.expander('Factor Tasks'), FactorTaskWindow, lambda x: x.factor_name)
self.current_win = ObjectsTabsWindow(
self.container.expander("Factor Tasks"), FactorTaskWindow, lambda x: x.factor_name
)
elif isinstance(msg.content[0], ModelTask):
self.current_win = ObjectsTabsWindow(self.container.expander('Model Tasks'), ModelTaskWindow, lambda x: x.name)
self.current_win = ObjectsTabsWindow(
self.container.expander("Model Tasks"), ModelTaskWindow, lambda x: x.name
)
elif isinstance(msg.content[0], FactorFBWorkspace):
self.current_win = ObjectsTabsWindow(self.container.expander('Factor Workspaces'),
inner_class=WorkspaceWindow,
mapper=lambda x: x.target_task.factor_name)
self.current_win = ObjectsTabsWindow(
self.container.expander("Factor Workspaces"),
inner_class=WorkspaceWindow,
mapper=lambda x: x.target_task.factor_name,
)
self.evolving_tasks = [m.target_task.factor_name for m in msg.content]
elif isinstance(msg.content[0], ModelFBWorkspace):
self.current_win = ObjectsTabsWindow(self.container.expander('Model Workspaces'),
inner_class=WorkspaceWindow,
mapper=lambda x: x.target_task.name)
self.current_win = ObjectsTabsWindow(
self.container.expander("Model Workspaces"),
inner_class=WorkspaceWindow,
mapper=lambda x: x.target_task.name,
)
self.evolving_tasks = [m.target_task.name for m in msg.content]
elif isinstance(msg.content[0], FactorSingleFeedback):
self.current_win = ObjectsTabsWindow(self.container.expander('Factor Feedbacks'),
inner_class=FactorFeedbackWindow,
tab_names=self.evolving_tasks)
self.current_win = ObjectsTabsWindow(
self.container.expander("Factor Feedbacks"),
inner_class=FactorFeedbackWindow,
tab_names=self.evolving_tasks,
)
elif isinstance(msg.content[0], ModelCoderFeedback):
self.current_win = ObjectsTabsWindow(self.container.expander('Model Feedbacks'),
inner_class=ModelFeedbackWindow,
tab_names=self.evolving_tasks)
self.current_win = ObjectsTabsWindow(
self.container.expander("Model Feedbacks"),
inner_class=ModelFeedbackWindow,
tab_names=self.evolving_tasks,
)
else:
# common logs
if not self.show_common_logs:
@@ -425,12 +442,11 @@ class SimpleTraceWindow(StWindow):
def mock_msg(obj) -> Message:
return Message(tag='mock', level='INFO', timestamp=datetime.now(), pid_trace='000', caller='mock',content=obj)
return Message(tag="mock", level="INFO", timestamp=datetime.now(), pid_trace="000", caller="mock", content=obj)
class TraceObjWindow(StWindow):
def __init__(self, container: 'DeltaGenerator' = st.container()):
def __init__(self, container: "DeltaGenerator" = st.container()):
self.container = container
def consume_msg(self, msg: Message | Trace):
@@ -440,7 +456,7 @@ class TraceObjWindow(StWindow):
trace = msg
for id, (h, e, hf) in enumerate(trace.hist):
self.container.header(f'Trace History {id}', divider=True)
self.container.header(f"Trace History {id}", divider=True)
HypothesisWindow(self.container).consume_msg(mock_msg(h))
if isinstance(e, QlibFactorExperiment):
QlibFactorExpWindow(self.container).consume_msg(mock_msg(e))
@@ -450,70 +466,74 @@ class TraceObjWindow(StWindow):
class ResearchWindow(StWindow):
def consume_msg(self, msg: Message):
if msg.tag.endswith('hypothesis generation'):
if msg.tag.endswith("hypothesis generation"):
HypothesisWindow(self.container.container()).consume_msg(msg)
elif msg.tag.endswith('experiment generation'):
elif msg.tag.endswith("experiment generation"):
if isinstance(msg.content, list):
if isinstance(msg.content[0], FactorTask):
self.container.markdown('**Factor Tasks**')
ObjectsTabsWindow(self.container.container(), FactorTaskWindow, lambda x: x.factor_name).consume_msg(msg)
self.container.markdown("**Factor Tasks**")
ObjectsTabsWindow(
self.container.container(), FactorTaskWindow, lambda x: x.factor_name
).consume_msg(msg)
elif isinstance(msg.content[0], ModelTask):
self.container.markdown('**Model Tasks**')
self.container.markdown("**Model Tasks**")
ObjectsTabsWindow(self.container.container(), ModelTaskWindow, lambda x: x.name).consume_msg(msg)
class EvolvingWindow(StWindow):
def __init__(self, container: 'DeltaGenerator'):
def __init__(self, container: "DeltaGenerator"):
self.container = container
self.evolving_tasks: list[str] = []
def consume_msg(self, msg: Message):
if msg.tag.endswith('evolving code'):
if msg.tag.endswith("evolving code"):
if isinstance(msg.content, list):
msg.content = [m for m in msg.content if m]
if len(msg.content) == 0:
return
if isinstance(msg.content[0], FactorFBWorkspace):
self.container.markdown('**Factor Codes**')
ObjectsTabsWindow(self.container.container(),
inner_class=WorkspaceWindow,
mapper=lambda x: x.target_task.factor_name).consume_msg(msg)
self.container.markdown("**Factor Codes**")
ObjectsTabsWindow(
self.container.container(),
inner_class=WorkspaceWindow,
mapper=lambda x: x.target_task.factor_name,
).consume_msg(msg)
self.evolving_tasks = [m.target_task.factor_name for m in msg.content]
elif isinstance(msg.content[0], ModelFBWorkspace):
self.container.markdown('**Model Codes**')
ObjectsTabsWindow(self.container.container(),
inner_class=WorkspaceWindow,
mapper=lambda x: x.target_task.name).consume_msg(msg)
self.container.markdown("**Model Codes**")
ObjectsTabsWindow(
self.container.container(), inner_class=WorkspaceWindow, mapper=lambda x: x.target_task.name
).consume_msg(msg)
self.evolving_tasks = [m.target_task.name for m in msg.content]
elif msg.tag.endswith('evolving feedback'):
elif msg.tag.endswith("evolving feedback"):
if isinstance(msg.content, list):
msg.content = [m for m in msg.content if m]
if len(msg.content) == 0:
return
if isinstance(msg.content[0], FactorSingleFeedback):
self.container.markdown('**Factor Feedbacks🔍**')
ObjectsTabsWindow(self.container.container(),
inner_class=FactorFeedbackWindow,
tab_names=self.evolving_tasks).consume_msg(msg)
self.container.markdown("**Factor Feedbacks🔍**")
ObjectsTabsWindow(
self.container.container(), inner_class=FactorFeedbackWindow, tab_names=self.evolving_tasks
).consume_msg(msg)
elif isinstance(msg.content[0], ModelCoderFeedback):
self.container.markdown('**Model Feedbacks🔍**')
ObjectsTabsWindow(self.container.container(),
inner_class=ModelFeedbackWindow,
tab_names=self.evolving_tasks).consume_msg(msg)
self.container.markdown("**Model Feedbacks🔍**")
ObjectsTabsWindow(
self.container.container(), inner_class=ModelFeedbackWindow, tab_names=self.evolving_tasks
).consume_msg(msg)
class DevelopmentWindow(StWindow):
def __init__(self, container: 'DeltaGenerator'):
self.E_win = RoundTabsWindow(container.container(),
new_tab_func=lambda x: x.tag.endswith('evolving code'),
inner_class=EvolvingWindow,
title='Evolving Loops🔧')
def __init__(self, container: "DeltaGenerator"):
self.E_win = RoundTabsWindow(
container.container(),
new_tab_func=lambda x: x.tag.endswith("evolving code"),
inner_class=EvolvingWindow,
title="Evolving Loops🔧",
)
def consume_msg(self, msg: Message):
if 'evolving' in msg.tag:
if "evolving" in msg.tag:
self.E_win.consume_msg(msg)
# elif msg.tag.endswith('result'):
# self.container.subheader('Results')
@@ -528,8 +548,7 @@ class DevelopmentWindow(StWindow):
class FeedbackWindow(StWindow):
def __init__(self, container: 'DeltaGenerator'):
def __init__(self, container: "DeltaGenerator"):
self.container = container
def consume_msg(self, msg: Message):
@@ -542,8 +561,7 @@ class FeedbackWindow(StWindow):
class SingleRDLoopWindow(StWindow):
def __init__(self, container: 'DeltaGenerator'):
def __init__(self, container: "DeltaGenerator"):
self.container = container
col1, col2 = self.container.columns([2, 3])
self.R_win = ResearchWindow(col1.container(border=True))
@@ -551,58 +569,66 @@ class SingleRDLoopWindow(StWindow):
self.D_win = DevelopmentWindow(col2.container(border=True))
def consume_msg(self, msg: Message):
tags = msg.tag.split('.')
if 'r' in tags:
tags = msg.tag.split(".")
if "r" in tags:
self.R_win.consume_msg(msg)
elif 'd' in tags:
elif "d" in tags:
self.D_win.consume_msg(msg)
elif 'ef' in tags:
elif "ef" in tags:
self.F_win.consume_msg(msg)
class TraceWindow(StWindow):
def __init__(self, container: 'DeltaGenerator' = st.container(), show_llm: bool = False, show_common_logs: bool = False):
def __init__(
self, container: "DeltaGenerator" = st.container(), show_llm: bool = False, show_common_logs: bool = False
):
self.show_llm = show_llm
self.show_common_logs = show_common_logs
top_container = container.container()
col1, col2 = top_container.columns([2,3])
col1, col2 = top_container.columns([2, 3])
chart_c = col2.container(border=True, height=300)
chart_c.markdown('**Metrics📈**')
chart_c.markdown("**Metrics📈**")
self.chart_c = chart_c.empty()
hypothesis_status_c = col1.container(border=True, height=300)
hypothesis_status_c.markdown('**Hypotheses🏅**')
hypothesis_status_c.markdown("**Hypotheses🏅**")
self.summary_c = hypothesis_status_c.empty()
self.RDL_win = RoundTabsWindow(container.container(),
new_tab_func=lambda x: x.tag.endswith('hypothesis generation'),
inner_class=SingleRDLoopWindow,
title='R&D Loops♾️')
self.RDL_win = RoundTabsWindow(
container.container(),
new_tab_func=lambda x: x.tag.endswith("hypothesis generation"),
inner_class=SingleRDLoopWindow,
title="R&D Loops♾️",
)
self.hypothesis_decisions = defaultdict(bool)
self.current_hypothesis = None
self.results = []
def consume_msg(self, msg: Message):
if not self.show_llm and 'llm_messages' in msg.tag:
if not self.show_llm and "llm_messages" in msg.tag:
return
if not self.show_common_logs and isinstance(msg.content, str):
return
if isinstance(msg.content, dict):
return
if msg.tag.endswith('hypothesis generation'):
if msg.tag.endswith("hypothesis generation"):
self.current_hypothesis = msg.content.hypothesis
elif msg.tag.endswith('ef.feedback'):
elif msg.tag.endswith("ef.feedback"):
self.hypothesis_decisions[self.current_hypothesis] = msg.content.decision
self.summary_c.markdown('\n'.join(f"{id+1}. :green[{h}]\n" if d else f"{id+1}. {h}\n" for id,(h,d) in enumerate(self.hypothesis_decisions.items())))
elif msg.tag.endswith('ef.model runner result') or msg.tag.endswith('ef.factor runner result'):
self.summary_c.markdown(
"\n".join(
f"{id+1}. :green[{h}]\n" if d else f"{id+1}. {h}\n"
for id, (h, d) in enumerate(self.hypothesis_decisions.items())
)
)
elif msg.tag.endswith("ef.model runner result") or msg.tag.endswith("ef.factor runner result"):
self.results.append(msg.content.result)
if len(self.results) == 1:
self.chart_c.table(self.results[0])
else:
df = pd.DataFrame(self.results, index=range(1, len(self.results)+1))
df = pd.DataFrame(self.results, index=range(1, len(self.results) + 1))
fig = px.line(df, x=df.index, y=df.columns, markers=True)
self.chart_c.plotly_chart(fig)
rdagent/log/utils.py
+1 -2
View File
@@ -1,7 +1,6 @@
import inspect
import re
from typing import Union, Dict, TypedDict, Optional
from typing import Dict, Optional, TypedDict, Union
class LogColors:
rdagent/scenarios/data_mining/developer/feedback.py
+1 -1
View File
@@ -18,7 +18,7 @@ from rdagent.log import rdagent_logger as logger
from rdagent.oai.llm_utils import APIBackend
from rdagent.utils import convert2bool
feedback_prompts = Prompts(file_path=Path(__file__).parent.parent.parent/ "qlib" / "prompts.yaml")
feedback_prompts = Prompts(file_path=Path(__file__).parent.parent.parent / "qlib" / "prompts.yaml")
DIRNAME = Path(__file__).absolute().resolve().parent
rdagent/scenarios/data_mining/developer/model_runner.py
-1
View File
@@ -15,7 +15,6 @@ from rdagent.utils.env import DMDockerEnv
class DMModelRunner(CachedRunner[DMModelExperiment]):
def develop(self, exp: DMModelExperiment) -> DMModelExperiment:
if RUNNER_SETTINGS.cache_result:
cache_hit, result = self.get_cache_result(exp)
rdagent/scenarios/data_mining/experiment/model_experiment.py
+4 -4
View File
@@ -22,11 +22,11 @@ class DMModelScenario(Scenario):
@property
def background(self) -> str:
return prompt_dict["dm_model_background"]
@property
def source_data(self) -> str:
raise NotImplementedError("source_data is not implemented")
@property
def output_format(self) -> str:
return prompt_dict["dm_model_output_format"]
@@ -38,9 +38,9 @@ class DMModelScenario(Scenario):
@property
def simulator(self) -> str:
return prompt_dict["dm_model_simulator"]
@property
def rich_style_description(self)->str:
def rich_style_description(self) -> str:
return "Below is MIMIC Model Evolving Automatic R&D Demo."
def get_scenario_all_desc(self) -> str:
rdagent/scenarios/data_mining/experiment/model_template/train.py
+28 -20
View File
@@ -1,21 +1,23 @@
from pathlib import Path
import pandas as pd
import torch
import torch.nn.functional as F
from torchvision import transforms, datasets
from torch.utils.data import DataLoader, Dataset
import torch.nn as nn
import sparse
import random
import os
import random
from pathlib import Path
import numpy as np
import pandas as pd
import sparse
import torch
import torch.nn as nn
import torch.nn.functional as F
from model import model_cls
from sklearn.metrics import accuracy_score, roc_auc_score
import numpy as np
from torch.utils.data import DataLoader, Dataset
from torchvision import datasets, transforms
# Set device for training
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
# device = torch.device("cpu")
class MyDataset(Dataset):
def __init__(self, x, label, device):
self.x1 = x
@@ -28,9 +30,11 @@ class MyDataset(Dataset):
def __getitem__(self, idx):
if torch.is_tensor(idx):
idx = idx.tolist()
return torch.FloatTensor(self.x1[idx]).to(self.device), \
torch.tensor(self.label[idx], dtype=torch.float).to(self.device)
return torch.FloatTensor(self.x1[idx]).to(self.device), torch.tensor(self.label[idx], dtype=torch.float).to(
self.device
)
def collate_fn(batch):
x, label = [], []
for data in batch:
@@ -39,12 +43,12 @@ def collate_fn(batch):
return torch.stack(x, 0), torch.stack(label, 0)
datapath = '/root/.data'
datapath = "/root/.data"
# datapath = '/home/v-suhancui/RD-Agent/physionet.org/files/mimic-eicu-fiddle-feature/1.0.0/FIDDLE_mimic3'
X = sparse.load_npz(datapath+'/features/ARF_12h/X.npz').todense()
df_pop = pd.read_csv(datapath+'/population/ARF_12h.csv')['ARF_LABEL']
X = sparse.load_npz(datapath + "/features/ARF_12h/X.npz").todense()
df_pop = pd.read_csv(datapath + "/population/ARF_12h.csv")["ARF_LABEL"]
X = X.transpose(0, 2, 1)
@@ -56,8 +60,12 @@ X_train, y_train = X[indices[:split_point]], np.array(df_pop[indices[:split_poin
X_test, y_test = X[indices[split_point:]], np.array(df_pop[indices[split_point:]])
train_dataloader = DataLoader(MyDataset(X_train, y_train, device), collate_fn=collate_fn, shuffle=True, drop_last=True, batch_size=64)
test_dataloader = DataLoader(MyDataset(X_test, y_test, device), collate_fn=collate_fn, shuffle=False, drop_last=False, batch_size=64)
train_dataloader = DataLoader(
MyDataset(X_train, y_train, device), collate_fn=collate_fn, shuffle=True, drop_last=True, batch_size=64
)
test_dataloader = DataLoader(
MyDataset(X_test, y_test, device), collate_fn=collate_fn, shuffle=False, drop_last=False, batch_size=64
)
num_features = 4816
num_timesteps = 12
@@ -88,5 +96,5 @@ acc = roc_auc_score(y_test, np.concatenate(y_pred))
print(acc)
# Save the predictions to submission.csv
with open('./submission.txt', 'w') as f:
f.write(str(acc))
with open("./submission.txt", "w") as f:
f.write(str(acc))
rdagent/scenarios/data_mining/experiment/workspace.py
+3 -2
View File
@@ -2,10 +2,11 @@ from pathlib import Path
import pandas as pd
from rdagent.app.data_mining.conf import PROP_SETTING
from rdagent.core.experiment import FBWorkspace
from rdagent.log import rdagent_logger as logger
from rdagent.utils.env import DMDockerEnv
from rdagent.app.data_mining.conf import PROP_SETTING
class DMFBWorkspace(FBWorkspace):
def __init__(self, template_folder_path: Path, *args, **kwargs) -> None:
@@ -27,5 +28,5 @@ class DMFBWorkspace(FBWorkspace):
if not csv_path.exists():
logger.error(f"File {csv_path} does not exist.")
return None
with open(self.workspace_path / "submission.txt", 'r') as f:
with open(self.workspace_path / "submission.txt", "r") as f:
return f.read()
rdagent/scenarios/data_mining/proposal/model_proposal.py
+5 -2
View File
@@ -30,6 +30,7 @@ class DMModelHypothesisGen(ModelHypothesisGen):
class XXXDMModelHypothesisGen(DMModelHypothesisGen):
prompts: Prompts = a_specifc_prompt_dict
"""
def __init__(self, scen: Scenario) -> Tuple[dict, bool]:
super().__init__(scen)
@@ -43,7 +44,7 @@ class DMModelHypothesisGen(ModelHypothesisGen):
"hypothesis_and_feedback": hypothesis_feedback,
"RAG": "",
"hypothesis_output_format": prompt_dict["hypothesis_output_format"],
"hypothesis_specification": prompt_dict["model_hypothesis_specification"]
"hypothesis_specification": prompt_dict["model_hypothesis_specification"],
}
return context_dict, True
@@ -89,7 +90,9 @@ class DMModelHypothesis2Experiment(ModelHypothesis2Experiment):
variables = response_dict[model_name]["variables"]
hyperparameters = response_dict[model_name]["hyperparameters"]
model_type = response_dict[model_name]["model_type"]
tasks.append(ModelTask(model_name, description, formulation, architecture, variables, hyperparameters, model_type))
tasks.append(
ModelTask(model_name, description, formulation, architecture, variables, hyperparameters, model_type)
)
exp = DMModelExperiment(tasks)
exp.based_experiments = [t[1] for t in trace.hist if t[2]]
return exp
rdagent/scenarios/qlib/developer/feedback.py
+20 -19
View File
@@ -1,8 +1,8 @@
import json
from pathlib import Path
from jinja2 import Environment, StrictUndefined
import pandas as pd
from jinja2 import Environment, StrictUndefined
from rdagent.core.experiment import Experiment
from rdagent.core.prompts import Prompts
@@ -19,37 +19,38 @@ from rdagent.utils import convert2bool
feedback_prompts = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")
DIRNAME = Path(__file__).absolute().resolve().parent
def process_results(current_result, sota_result):
# Convert the results to dataframes
current_df = pd.DataFrame(current_result)
sota_df = pd.DataFrame(sota_result)
# Set the metric as the index
current_df.index.name = 'metric'
sota_df.index.name = 'metric'
current_df.index.name = "metric"
sota_df.index.name = "metric"
# Rename the value column to reflect the result type
current_df.rename(columns={'0': 'Current Result'}, inplace=True)
sota_df.rename(columns={'0': 'SOTA Result'}, inplace=True)
current_df.rename(columns={"0": "Current Result"}, inplace=True)
sota_df.rename(columns={"0": "SOTA Result"}, inplace=True)
# Combine the dataframes on the Metric index
combined_df = pd.concat([current_df, sota_df], axis=1)
# Select important metrics for comparison
important_metrics = [
'Rank ICIR',
'1day.excess_return_without_cost.max_drawdown',
'1day.excess_return_without_cost.information_ratio',
'1day.excess_return_without_cost.annualized_return',
'1day.excess_return_with_cost.max_drawdown',
'1day.excess_return_with_cost.information_ratio',
'1day.excess_return_with_cost.annualized_return',
'IC',
"Rank ICIR",
"1day.excess_return_without_cost.max_drawdown",
"1day.excess_return_without_cost.information_ratio",
"1day.excess_return_without_cost.annualized_return",
"1day.excess_return_with_cost.max_drawdown",
"1day.excess_return_with_cost.information_ratio",
"1day.excess_return_with_cost.annualized_return",
"IC",
]
# Filter the combined DataFrame to retain only the important metrics
filtered_combined_df = combined_df.loc[important_metrics]
return filtered_combined_df.to_dict()
rdagent/scenarios/qlib/experiment/factor_experiment.py
+2 -2
View File
@@ -39,9 +39,9 @@ class QlibFactorScenario(Scenario):
@property
def simulator(self) -> str:
return prompt_dict["qlib_factor_simulator"]
@property
def rich_style_description(self)->str:
def rich_style_description(self) -> str:
return "Below is QlibFactor Evolving Automatic R&D Demo."
def get_scenario_all_desc(self) -> str:
rdagent/scenarios/qlib/experiment/factor_template/read_exp_res.py
+1 -1
View File
@@ -45,4 +45,4 @@ else:
print(f"Output has been saved to {output_path}")
ret_data_frame = latest_recorder.load_object("portfolio_analysis/report_normal_1day.pkl")
ret_data_frame.to_pickle("ret.pkl")
ret_data_frame.to_pickle("ret.pkl")
rdagent/scenarios/qlib/experiment/model_experiment.py
+4 -4
View File
@@ -38,10 +38,10 @@ class QlibModelScenario(Scenario):
@property
def simulator(self) -> str:
return prompt_dict["qlib_model_simulator"]
@property
def rich_style_description(self)->str:
return '''
def rich_style_description(self) -> str:
return """
### Qlib Model Evolving Automatic R&D Demo
#### Overview
@@ -75,7 +75,7 @@ The demo showcases the iterative process of hypothesis generation, knowledge con
To demonstrate the dynamic evolution of models through the Qlib platform, emphasizing how each iteration enhances the accuracy and reliability of the resulting models.
'''
"""
def get_scenario_all_desc(self) -> str:
return f"""Background of the scenario:
rdagent/scenarios/qlib/experiment/model_template/read_exp_res.py
-1
View File
@@ -39,7 +39,6 @@ else:
# Load the specified file from the latest recorder
metrics = pd.Series(latest_recorder.list_metrics())
output_path = Path(__file__).resolve().parent / "qlib_res.csv"
metrics.to_csv(output_path)
rdagent/scenarios/qlib/factor_experiment_loader/json_loader.py
+1 -3
View File
@@ -56,9 +56,7 @@ class FactorTestCaseLoaderFromJsonFile:
variables=factor_data["variables"],
)
gt = FactorFBWorkspace(task)
code = {
"factor.py": factor_data["gt_code"]
}
code = {"factor.py": factor_data["gt_code"]}
gt.inject_code(**code)
gt.execute()
TestData.target_task.sub_tasks.append(task)
rdagent/scenarios/qlib/factor_experiment_loader/pdf_loader.py
+8 -5
View File
@@ -196,7 +196,11 @@ def __extract_factor_and_formulation_from_one_report(
factor_dict,
)
for factor_name in factor_dict:
if factor_name not in factor_to_formulation or "formulation" not in factor_to_formulation[factor_name] or "variables" not in factor_to_formulation[factor_name]:
if (
factor_name not in factor_to_formulation
or "formulation" not in factor_to_formulation[factor_name]
or "variables" not in factor_to_formulation[factor_name]
):
continue
final_factor_dict_to_one_report.setdefault(factor_name, {})
@@ -272,7 +276,6 @@ def merge_file_to_factor_dict_to_factor_dict(
def __check_factor_dict_viability_simulate_json_mode(
factor_df_string: str,
) -> dict[str, dict[str, str]]:
extract_result_resp = APIBackend().build_messages_and_create_chat_completion(
system_prompt=document_process_prompts["factor_viability_system"],
user_prompt=factor_df_string,
@@ -512,11 +515,11 @@ class FactorExperimentLoaderFromPDFfiles(FactorExperimentLoader):
logger.log_object(docs_dict)
selected_report_dict = classify_report_from_dict(report_dict=docs_dict, vote_time=1)
with logger.tag("file_to_factor_result"):
file_to_factor_result = extract_factors_from_report_dict(docs_dict, selected_report_dict)
logger.log_object(file_to_factor_result)
with logger.tag("factor_dict"):
factor_dict = merge_file_to_factor_dict_to_factor_dict(file_to_factor_result)
logger.log_object(factor_dict)
@@ -526,5 +529,5 @@ class FactorExperimentLoaderFromPDFfiles(FactorExperimentLoader):
logger.log_object(filtered_factor_dict)
# factor_dict, duplication_names_list = deduplicate_factors_by_llm(factor_dict, factor_viability)
return FactorExperimentLoaderFromDict().load(filtered_factor_dict)
rdagent/scenarios/qlib/proposal/factor_proposal.py
+8 -4
View File
@@ -39,7 +39,11 @@ class QlibFactorHypothesisGen(FactorHypothesisGen):
def convert_response(self, response: str) -> FactorHypothesis:
response_dict = json.loads(response)
hypothesis = QlibFactorHypothesis(hypothesis=response_dict["hypothesis"], reason=response_dict["reason"], concise_reason=response_dict["concise_reason"])
hypothesis = QlibFactorHypothesis(
hypothesis=response_dict["hypothesis"],
reason=response_dict["reason"],
concise_reason=response_dict["concise_reason"],
)
return hypothesis
@@ -78,15 +82,15 @@ class QlibFactorHypothesis2Experiment(FactorHypothesis2Experiment):
formulation = response_dict[factor_name]["formulation"]
variables = response_dict[factor_name]["variables"]
tasks.append(FactorTask(factor_name, description, formulation, variables))
exp = QlibFactorExperiment(tasks)
exp.based_experiments = [t[1] for t in trace.hist if t[2]]
if len(exp.based_experiments) == 0:
exp.based_experiments.append(QlibFactorExperiment(sub_tasks=[]))
unique_tasks = []
for task in tasks:
duplicate = False
for based_exp in exp.based_experiments:
rdagent/scenarios/qlib/proposal/model_proposal.py
+9 -3
View File
@@ -33,13 +33,17 @@ class QlibModelHypothesisGen(ModelHypothesisGen):
"hypothesis_and_feedback": hypothesis_feedback,
"RAG": "In Quantitative Finance, market data could be time-series, and GRU model/LSTM model are suitable for them. Do not generate GNN model as for now.",
"hypothesis_output_format": prompt_dict["hypothesis_output_format"],
"hypothesis_specification": prompt_dict["model_hypothesis_specification"]
"hypothesis_specification": prompt_dict["model_hypothesis_specification"],
}
return context_dict, True
def convert_response(self, response: str) -> ModelHypothesis:
response_dict = json.loads(response)
hypothesis = QlibModelHypothesis(hypothesis=response_dict["hypothesis"], reason=response_dict["reason"], concise_reason=response_dict["concise_reason"])
hypothesis = QlibModelHypothesis(
hypothesis=response_dict["hypothesis"],
reason=response_dict["reason"],
concise_reason=response_dict["concise_reason"],
)
return hypothesis
@@ -79,7 +83,9 @@ class QlibModelHypothesis2Experiment(ModelHypothesis2Experiment):
variables = response_dict[model_name]["variables"]
hyperparameters = response_dict[model_name]["hyperparameters"]
model_type = response_dict[model_name]["model_type"]
tasks.append(ModelTask(model_name, description, formulation, architecture, variables, hyperparameters, model_type))
tasks.append(
ModelTask(model_name, description, formulation, architecture, variables, hyperparameters, model_type)
)
exp = QlibModelExperiment(tasks)
exp.based_experiments = [t[1] for t in trace.hist if t[2]]
return exp
rdagent/utils/agent/ret.py
+1 -1
View File
@@ -3,8 +3,8 @@ The output of a agent is very important.
We think this part can be shared.
"""
from abc import abstractclassmethod
import re
from abc import abstractclassmethod
from typing import Any
from rdagent.utils.agent.tpl import T
rdagent/utils/agent/tpl.py
+10 -10
View File
@@ -4,13 +4,12 @@ Here are some infrastruture to build a agent
The motivation of tempalte and AgentOutput Design
"""
from typing import Any
from jinja2 import Environment, StrictUndefined
import inspect
from pathlib import Path
from typing import Any
import yaml
import inspect
from jinja2 import Environment, StrictUndefined
from rdagent.core.utils import SingletonBaseClass
@@ -21,9 +20,10 @@ PROJ_PATH = DIRNAME.parent.parent
# class T(SingletonBaseClass): TODO: singleton does not support args now.
class T:
"""Use the simplest way to (C)reate a Template and (r)ender it!!"""
def __init__(self, uri: str):
"""
here are some uri usages
here are some uri usages
case 1) "a.b.c:x.y.z"
It will load DIRNAME/a/b/c.yaml as `yaml` and load yaml[x][y][z]
case 2) ".c:x.y.z"
@@ -38,16 +38,16 @@ class T:
caller_dir = Path(caller_module.__file__).parent
# Parse the URI
path_part, yaml_path = uri.split(':')
yaml_keys = yaml_path.split('.')
path_part, yaml_path = uri.split(":")
yaml_keys = yaml_path.split(".")
if path_part.startswith('.'):
if path_part.startswith("."):
yaml_file_path = caller_dir / f"{path_part[1:].replace('.', '/')}.yaml"
else:
yaml_file_path = (PROJ_PATH / path_part.replace('.', '/')).with_suffix('.yaml')
yaml_file_path = (PROJ_PATH / path_part.replace(".", "/")).with_suffix(".yaml")
# Load the YAML file
with open(yaml_file_path, 'r') as file:
with open(yaml_file_path, "r") as file:
yaml_content = yaml.safe_load(file)
# Traverse the YAML content to get the desired template
rdagent/utils/env.py
+14 -7
View File
@@ -144,16 +144,21 @@ class QlibDockerConf(DockerConf):
class DMDockerConf(DockerConf):
class Config:
env_prefix = "DM_DOCKER_"
env_prefix = "DM_DOCKER_"
build_from_dockerfile: bool = True
dockerfile_folder_path: Path = Path(__file__).parent.parent / "scenarios" / "data_mining" / "docker"
image: str = "local_dm:latest"
mount_path: str = "/workspace/dm_workspace/"
default_entry: str = "python train.py"
extra_volumes: dict = {Path("~/.rdagent/.data/physionet.org/files/mimic-eicu-fiddle-feature/1.0.0/FIDDLE_mimic3/").expanduser().resolve(): "/root/.data/"}
extra_volumes: dict = {
Path("~/.rdagent/.data/physionet.org/files/mimic-eicu-fiddle-feature/1.0.0/FIDDLE_mimic3/")
.expanduser()
.resolve(): "/root/.data/"
}
shm_size: str | None = "16g"
# physionet.org/files/mimic-eicu-fiddle-feature/1.0.0/FIDDLE_mimic3
class DockerEnv(Env[DockerConf]):
# TODO: Save the output into a specific file
@@ -181,9 +186,9 @@ class DockerEnv(Env[DockerConf]):
if not self.conf.enable_gpu:
return {}
gpu_kwargs = {
"device_requests": [
docker.types.DeviceRequest(count=-1, capabilities=[['gpu']])
] if self.conf.enable_gpu else None,
"device_requests": [docker.types.DeviceRequest(count=-1, capabilities=[["gpu"]])]
if self.conf.enable_gpu
else None,
}
try:
client.containers.run(self.conf.image, "nvidia-smi", **gpu_kwargs)
@@ -220,7 +225,7 @@ class DockerEnv(Env[DockerConf]):
# auto_remove=True, # remove too fast might cause the logs not to be get
network=self.conf.network,
shm_size=self.conf.shm_size,
**self._gpu_kwargs(client)
**self._gpu_kwargs(client),
)
logs = container.logs(stream=True)
for log in logs:
@@ -273,7 +278,9 @@ class DMDockerEnv(DockerEnv):
data_path = next(iter(self.conf.extra_volumes.keys()))
if not (Path(data_path)).exists():
logger.info("We are downloading!")
cmd = 'wget -r -N -c -np --user={} --password={} -P ~/.rdagent/.data/ https://physionet.org/files/mimic-eicu-fiddle-feature/1.0.0/'.format(username, password)
cmd = "wget -r -N -c -np --user={} --password={} -P ~/.rdagent/.data/ https://physionet.org/files/mimic-eicu-fiddle-feature/1.0.0/".format(
username, password
)
os.system(cmd)
else:
logger.info("Data already exists. Download skipped.")
rdagent/utils/workflow.py
+13 -12
View File
@@ -7,20 +7,19 @@ Postscripts:
However, Python generator is not picklable (dill does not support pickle as well)
"""
from pathlib import Path
import datetime
import pickle
from tqdm.auto import tqdm
from collections import defaultdict
from dataclasses import dataclass, field
import datetime
from pathlib import Path
from typing import Callable
from tqdm.auto import tqdm
from rdagent.log import rdagent_logger as logger
class LoopMeta(type):
@staticmethod
def _get_steps(bases):
"""
@@ -28,7 +27,7 @@ class LoopMeta(type):
"""
steps = []
for base in bases:
steps.extend(LoopMeta._get_steps(base.__bases__) + getattr(base,"steps", []))
steps.extend(LoopMeta._get_steps(base.__bases__) + getattr(base, "steps", []))
return steps
def __new__(cls, clsname, bases, attrs):
@@ -52,12 +51,14 @@ class LoopBase:
steps: list[Callable] # a list of steps to work on
loop_trace: dict[int, list[LoopTrace]]
skip_loop_error: tuple[Exception] = field(default_factory=tuple) # you can define a list of error that will skip current loop
skip_loop_error: tuple[Exception] = field(
default_factory=tuple
) # you can define a list of error that will skip current loop
def __init__(self):
self.loop_idx = 0 # current loop index
self.step_idx = 0 # the index of next step to be run
self.loop_prev_out = {} # the step results of current loop
self.loop_idx = 0 # current loop index
self.step_idx = 0 # the index of next step to be run
self.loop_prev_out = {} # the step results of current loop
self.loop_trace = defaultdict(list[LoopTrace]) # the key is the number of loop
self.session_folder = logger.log_trace_path / "__session__"
@@ -121,7 +122,7 @@ class LoopBase:
with path.open("rb") as f:
session = pickle.load(f)
logger.set_trace_path(session.session_folder.parent)
max_loop = max(session.loop_trace.keys())
logger.storage.truncate(time=session.loop_trace[max_loop][-1].end)
return session
test/utils/test_agent_infra.py
-4
View File
@@ -5,12 +5,8 @@ from rdagent.utils.agent.ret import PythonAgentOut
from rdagent.utils.agent.tpl import T
class TestAgentInfra(unittest.TestCase):
def test_agent_infra(self):
# NOTE: It is not serious. It is just for testing
sys_prompt = T("components.proposal.prompts:hypothesis_gen.system_prompt").r(
targets="targets",
test/utils/test_misc.py
-1
View File
@@ -9,7 +9,6 @@ class A(SingletonBaseClass):
class MiscTest(unittest.TestCase):
def test_singleton(self):
a1 = A()
a2 = A()