From 32a29a7479d4549c550c012b7635f2db6c3469b1 Mon Sep 17 00:00:00 2001 From: Linlang <30293408+SunsetWolf@users.noreply.github.com> Date: Wed, 18 Jun 2025 14:35:45 +0800 Subject: [PATCH] fix: main bug (#938) * feat: parameterize cache paths with USER to avoid conflicts * guide for missing training_hyperparameters * guidance for KeyError: 'concise_reason' * fixed three bugs in the test * fix general_model task bug * fixed some bugs in the med_model scenario * delete comments * format with black * fix mypy error * fix ruff error * fix isort error * sync code * revert cache_path code * revert cache_path code * delete data mining scenario * fix factor report loop * fix LiteLLMAPIBackend log_llm_chat_content setting * refine fin factor report scenario * remove unused LogColors * fix UI * remove medical scenario docs * change **kaggle** to **data_science** * remove default dataset_path in create_debug_data * remove KAGGLE_SETTINGS in kaggle_crawler * limit litellm versions * reformat with black * change README * fix_data_science_docs * make hypothesis observations string * Hiding old versions of kaggle docs * hidding kaggle agent docs --------- Co-authored-by: Young Co-authored-by: Bowen Xian Co-authored-by: yuanteli <1957922024@qq.com> --- README.md | 14 +- constraints/3.10.txt | 1 + constraints/3.11.txt | 1 + docs/scens/catalog.rst | 11 +- docs/scens/data_science.rst | 2 +- docs/scens/kaggle_agent.rst | 236 ------------------ docs/scens/model_agent_med.rst | 128 ---------- rdagent/app/cli.py | 6 +- rdagent/app/data_mining/conf.py | 45 ---- rdagent/app/data_mining/model.py | 33 --- rdagent/app/qlib_rd_loop/conf.py | 4 +- .../app/qlib_rd_loop/factor_from_report.py | 58 ++--- rdagent/app/qlib_rd_loop/quant.py | 2 +- rdagent/components/coder/CoSTEER/__init__.py | 2 +- .../components/coder/model_coder/prompts.yaml | 58 ++--- .../coder/model_coder/task_loader.py | 5 + rdagent/components/proposal/__init__.py | 2 + rdagent/components/workflow/rd_loop.py | 2 +- rdagent/log/logger.py | 8 +- rdagent/log/ui/app.py | 48 ++-- rdagent/oai/backend/litellm.py | 24 +- .../data_mining/developer/feedback.py | 62 ----- .../data_mining/developer/model_coder.py | 3 - .../data_mining/developer/model_runner.py | 26 -- .../scenarios/data_mining/docker/Dockerfile | 25 -- .../experiment/model_experiment.py | 75 ------ .../experiment/model_template/README.md | 3 - .../experiment/model_template/train.py | 116 --------- .../data_mining/experiment/prompts.yaml | 54 ---- .../data_mining/experiment/workspace.py | 31 --- .../data_mining/proposal/model_proposal.py | 122 --------- rdagent/scenarios/data_science/debug/data.py | 16 +- rdagent/scenarios/kaggle/kaggle_crawler.py | 17 +- .../factor_experiment_loader/json_loader.py | 2 +- .../factor_experiment_loader/pdf_loader.py | 3 +- rdagent/utils/env.py | 45 +--- rdagent/utils/workflow/loop.py | 5 +- requirements.txt | 2 +- test/utils/test_kaggle.py | 6 +- 39 files changed, 125 insertions(+), 1178 deletions(-) delete mode 100644 docs/scens/kaggle_agent.rst delete mode 100644 docs/scens/model_agent_med.rst delete mode 100644 rdagent/app/data_mining/conf.py delete mode 100644 rdagent/app/data_mining/model.py delete mode 100644 rdagent/scenarios/data_mining/developer/feedback.py delete mode 100644 rdagent/scenarios/data_mining/developer/model_coder.py delete mode 100644 rdagent/scenarios/data_mining/developer/model_runner.py delete mode 100644 rdagent/scenarios/data_mining/docker/Dockerfile delete mode 100644 rdagent/scenarios/data_mining/experiment/model_experiment.py delete mode 100644 rdagent/scenarios/data_mining/experiment/model_template/README.md delete mode 100644 rdagent/scenarios/data_mining/experiment/model_template/train.py delete mode 100644 rdagent/scenarios/data_mining/experiment/prompts.yaml delete mode 100644 rdagent/scenarios/data_mining/experiment/workspace.py delete mode 100644 rdagent/scenarios/data_mining/proposal/model_proposal.py diff --git a/README.md b/README.md index ba6801bc..2b8bd3eb 100644 --- a/README.md +++ b/README.md @@ -71,7 +71,7 @@ You can learn more details about **RD-Agent(Q)** through the [paper](https://arx | MLE-Bench Results Released | R&D-Agent currently leads as the [top-performing machine learning engineering agent](#-the-best-machine-learning-engineering-agent) on MLE-bench | | Support LiteLLM Backend | We now fully support **[LiteLLM](https://github.com/BerriAI/litellm)** as a backend for integration with multiple LLM providers. | | General Data Science Agent | [Data Science Agent](https://rdagent.readthedocs.io/en/latest/scens/data_science.html) | -| Kaggle Scenario release | We release **[Kaggle Agent](https://rdagent.readthedocs.io/en/latest/scens/kaggle_agent.html)**, try the new features! | +| Kaggle Scenario release | We release **[Kaggle Agent](https://rdagent.readthedocs.io/en/latest/scens/data_science.html)**, try the new features! | | Official WeChat group release | We created a WeChat group, welcome to join! (πŸ—ͺ[QR Code](https://github.com/microsoft/RD-Agent/issues/880)) | | Official Discord release | We launch our first chatting channel in Discord (πŸ—ͺ[![Chat](https://img.shields.io/badge/chat-discord-blue)](https://discord.gg/ybQ97B6Jjy)) | | First release | **R&D-Agent** is released on GitHub | @@ -249,7 +249,7 @@ The **[πŸ–₯️ Live Demo](https://rdagent.azurewebsites.net/)** is implemented b > 3. Join the competition: Click `Join the competition` -> `I Understand and Accept` at the bottom of the [competition details page](https://www.kaggle.com/competitions/sf-crime/data). ```bash # Generally, you can run the Kaggle competition program with the following command: - rdagent kaggle --competition + rdagent data_science --competition # Specifically, you need to create a folder for storing competition files (e.g., competition description file, competition datasets, etc.), and configure the path to the folder in your environment. In addition, you need to use chromedriver when you download the competition descriptors, which you can follow for this specific example: @@ -257,14 +257,12 @@ The **[πŸ–₯️ Live Demo](https://rdagent.azurewebsites.net/)** is implemented b # 2. Add the competition description file path to the `.env` file. mkdir -p ./git_ignore_folder/kaggle_data - dotenv set KG_LOCAL_DATA_PATH "$(pwd)/git_ignore_folder/kaggle_data" + dotenv set DS_LOCAL_DATA_PATH "$(pwd)/git_ignore_folder/kaggle_data" + dotenv set DS_IF_USING_MLE_DATA True # 3. run the application - rdagent kaggle --competition sf-crime + rdagent data_science --competition sf-crime ``` - > **Description of the above example:**
- > - Kaggle competition data is roughly divided into three sections: competition description file (json file) and complete dataset for the competition and simplified dataset for the competition.
- > - The Kaggle competition data will be downloaded automatically, the download process depends on `chromedriver`, installation instructions can be found in the [documentation](https://rdagent.readthedocs.io/en/latest/scens/kaggle_agent.html#example-guide).
### πŸ–₯️ Monitor the Application Results - You can run the following command for our demo program to see the run logs. @@ -310,7 +308,7 @@ The supported scenarios are listed below: | **🩺 Medical** | πŸ€– [Iteratively Proposing Ideas & Evolving](https://rdagent.azurewebsites.net/dmm)[▢️YouTube](https://www.youtube.com/watch?v=VIaSTZuoZg4) | - | | **🏭 General** | 🦾 [Auto paper reading & implementation](https://rdagent.azurewebsites.net/report_model)[▢️YouTube](https://www.youtube.com/watch?v=BiA2SfdKQ7o)
πŸ€– Auto Kaggle Model Tuning | πŸ€–Auto Kaggle feature Engineering | -- **[RoadMap](https://rdagent.readthedocs.io/en/latest/scens/kaggle_agent.html#roadmap)**: Currently, we are working hard to add new features to the Kaggle scenario. +- **[RoadMap](https://rdagent.readthedocs.io/en/latest/scens/data_science.html#roadmap)**: Currently, we are working hard to add new features to the Kaggle scenario. Different scenarios vary in entrance and configuration. Please check the detailed setup tutorial in the scenarios documents. diff --git a/constraints/3.10.txt b/constraints/3.10.txt index 864aac11..8ddd8a66 100644 --- a/constraints/3.10.txt +++ b/constraints/3.10.txt @@ -5,3 +5,4 @@ psutil==6.1.0 rich==13.9.2 scipy==1.14.1 tqdm==4.66.5 +litellm==1.72.4 diff --git a/constraints/3.11.txt b/constraints/3.11.txt index 864aac11..8ddd8a66 100644 --- a/constraints/3.11.txt +++ b/constraints/3.11.txt @@ -5,3 +5,4 @@ psutil==6.1.0 rich==13.9.2 scipy==1.14.1 tqdm==4.66.5 +litellm==1.72.4 diff --git a/docs/scens/catalog.rst b/docs/scens/catalog.rst index 81f54035..1c921635 100644 --- a/docs/scens/catalog.rst +++ b/docs/scens/catalog.rst @@ -26,16 +26,10 @@ The supported scenarios are listed below: :ref:`🦾Auto reports reading & implementation ` :ref:`πŸ€–Iteratively Proposing Ideas & Evolving ` - * - 🩺 Medical - - :ref:`πŸ€–Iteratively Proposing Ideas & Evolving ` - - *(no data yet)* * - 🏭 General - :ref:`🦾Auto paper reading & implementation ` - :ref:`πŸ€–Auto Kaggle Model Tuning ` - - :ref:`πŸ€–Auto Kaggle feature Engineering ` - - :ref:`πŸ€– Data Science ` + - :ref:`πŸ€– Data Science ` .. toctree:: @@ -47,8 +41,5 @@ The supported scenarios are listed below: data_agent_fin data_copilot_fin model_agent_fin - model_agent_med model_copilot_general - kaggle_agent data_science - diff --git a/docs/scens/data_science.rst b/docs/scens/data_science.rst index 1fab4265..8764243f 100644 --- a/docs/scens/data_science.rst +++ b/docs/scens/data_science.rst @@ -128,7 +128,7 @@ The Data Science Agent is an agent that can automatically perform feature engine .. code-block:: sh - rdagent kaggle --competition + rdagent data_science --competition - πŸ“₯ **Visualize the R&D Process** diff --git a/docs/scens/kaggle_agent.rst b/docs/scens/kaggle_agent.rst deleted file mode 100644 index 364dd714..00000000 --- a/docs/scens/kaggle_agent.rst +++ /dev/null @@ -1,236 +0,0 @@ -.. _kaggle_agent: - -=========================== -Data Science Agent - Kaggle -=========================== - -**πŸ€– Automated Feature Engineering & Model Tuning Evolution** ------------------------------------------------------------------------------------------- - -🎨 Design -~~~~~~~~~~~ - -.. image:: kaggle_design.png - :alt: Design of Kaggle Agent - :align: center - -πŸ“– Background -~~~~~~~~~~~~~~ -In the landscape of data science competitions, Kaggle serves as the ultimate arena where data enthusiasts harness the power of algorithms to tackle real-world challenges. -The Kaggle Agent stands as a pivotal tool, empowering participants to seamlessly integrate cutting-edge models and datasets, transforming raw data into actionable insights. - -By utilizing the **Kaggle Agent**, data scientists can craft innovative solutions that not only uncover hidden patterns but also drive significant advancements in predictive accuracy and model robustness. - - -🌟 Introduction -~~~~~~~~~~~~~~~~ - -In this scenario, our automated system proposes hypothesis, choose action, implements code, conducts validation, and utilizes feedback in a continuous, iterative process. - -The goal is to automatically optimize performance metrics within the validation set or Kaggle Leaderboard, ultimately discovering the most efficient features and models through autonomous research and development. - -Here's an enhanced outline of the steps: - -**Step 1 : Hypothesis Generation πŸ”** - -- Generate and propose initial hypotheses based on previous experiment analysis and domain expertise, with thorough reasoning and financial justification. - -**Step 2 : Experiment Creation ✨** - -- Transform the hypothesis into a task. -- Choose a specific action within feature engineering or model tuning. -- Develop, define, and implement a new feature or model, including its name, description, and formulation. - -**Step 3 : Model/Feature Implementation πŸ‘¨β€πŸ’»** - -- Implement the model code based on the detailed description. -- Evolve the model iteratively as a developer would, ensuring accuracy and efficiency. - -**Step 4 : Validation on Test Set or Kaggle πŸ“‰** - -- Validate the newly developed model using the test set or Kaggle dataset. -- Assess the model's effectiveness and performance based on the validation results. - -**Step 5: Feedback Analysis πŸ”** - -- Analyze validation results to assess performance. -- Use insights to refine hypotheses and enhance the model. - -**Step 6: Hypothesis Refinement ♻️** - -- Adjust hypotheses based on validation feedback. -- Iterate the process to continuously improve the model. - -🧭 Example Guide -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -- πŸ”§ **Set up RD-Agent Environment** - - - Before you start, please make sure you have installed RD-Agent and configured the environment for RD-Agent correctly. If you want to know how to install and configure the RD-Agent, please refer to the `documentation <../installation_and_configuration.html>`_. - -- πŸ”¨ **Configuring the Kaggle API** - - - Register and login on the `Kaggle `_ website. - - Click on the avatar (usually in the top right corner of the page) -> ``Settings`` -> ``Create New Token``, A file called ``kaggle.json`` will be downloaded. - - Move ``kaggle.json`` to ``~/.config/kaggle/`` - - Modify the permissions of the ``kaggle.json`` file. - - .. code-block:: sh - - chmod 600 ~/.config/kaggle/kaggle.json - - - For more information about Kaggle API Settings, refer to the `Kaggle API `_. - -- πŸ”© **Setting the Environment variables at .env file** - - - Determine the path where the data will be stored and add it to the ``.env`` file. - - .. code-block:: sh - - mkdir -p /kaggle_data - dotenv set KG_LOCAL_DATA_PATH /kaggle_data - -- πŸ“₯ **Download Competition Data** - - - Kaggle competition data, contains two parts: competition description file (json file) and competition dataset (zip file). - - - **How to get the competition description file** - - - The competition description file is downloaded automatically when the programme is run, and the download process relies on ``chromedriver``, which can be installed as follows: - - .. code-block:: sh - - # install chrome - wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb - sudo apt install ./google-chrome-stable_current_amd64.deb - google-chrome --version - - # install chromedriver - wget "https://storage.googleapis.com/chrome-for-testing-public/$(google-chrome --version | grep -oP '\d+\.\d+\.\d+\.\d+')/linux64/chromedriver-linux64.zip" - unzip chromedriver-linux64.zip - cd chromedriver-linux64 - sudo mv chromedriver /usr/local/bin - sudo chmod +x /usr/local/bin/chromedriver - chromedriver --version - - - **How to get the competition dataset** - - - The competition dataset is downloaded and extracted automatically when the program is run. If the zip file exists, the download will be skipped, if the unzip folder exists, the unzip will be skipped. - - - **Correct directory structure (Here is an example of competition data with id sf-crime)** - - .. code-block:: text - - kaggle_data - └── zip_files - | └── sf-crime.zip - β”œβ”€β”€ sample - | └── sf-crime - | └── ... - β”œβ”€β”€ sf-crime.json - └── sf-crime - └── ... - - - ``kaggle_data/zip_files/sf-crime.zip:`` Competition dataset zip files downloaded from the Kaggle website. - - - ``kaggle_data/sf-crime.json:`` Competition description file. - - - ``kaggle_data/sf-crime:`` The target folder for unzipping the competition dataset. Complete dataset. - - - ``kaggle_data/sample/sf-crime:`` Simplified dataset based on the complete dataset. Used to quickly verify that the code works. - -- πŸ—³οΈ **Join the competition** - - - If your Kaggle API account has not joined a competition, you will need to join the competition before running the program. - - - At the bottom of the competition details page, you can find the ``Join the competition`` button, click on it and select ``I Understand and Accept`` to join the competition. - - - In the **Competition List Available** below, you can jump to the competition details page. - -- πŸš€ **Run the Application** - - - You can directly run the application by using the following command: - - .. code-block:: sh - - rdagent kaggle --competition - -- πŸ“€ **Submit the Result Automatically or Manually** - - - If Auto: You need to set ``KG_AUTO_SUBMIT`` to ``true`` in the ``.env`` file. - - .. code-block:: sh - - dotenv set KG_AUTO_SUBMIT true - - - Else: You can download the prediction results from the UI interface and submit them manually. For more details, refer to the :doc:`UI guide <../ui>`. - - - -🎨 Customize one template for a new competition -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -In order to facilitate RD-Agent to generate competition codes, we have specified a competition code structure: - -.. image:: kaggle_template.png - :alt: Design of Kaggle Code Template - :align: center - -- **feature directory** contains the feature engineering code. Generally no modification is required. -- **model directory** contains the model codes. - select_xx.py is used to select different features according to different models. - model_xx.py is the basic code of different models. Generally, only some initial parameters need to be adjusted. -- **fea_share_preprocess.py** is some basic preprocessing code shared by different models. The degree of customization here is high, but the preprocess_script() function needs to be retained, which will be called by train.py -- **train.py** is the main code, which connects all the codes and is also the code called during the final execution. - -**We will soon provide a tool for automatic/semi-automatic template generation.** -If you want to try a different competition now, you can refer to our current template structure and content to write a new template. - - -🎯 Roadmap -~~~~~~~~~~~ - -**Completed:** - -- **Kaggle Project Schema Design** βœ… - -- **RD-Agent Integration with kaggle schema** βœ… - -**Ongoing:** - -- **Template auto generation** - -- **Bench Optimization** - - - **Online Bench** - - - **RealMLBench** - - - Ongoing integration - - - Auto online submission - - - Batch Evaluation - - - **Offline Bench** - - - MLE-Bench - - -πŸ› οΈ Usage of modules -~~~~~~~~~~~~~~~~~~~~~ - -.. _Env Config: - -- **Env Config** - -The following environment variables can be set in the `.env` file to customize the application's behavior: - -.. autopydantic_settings:: rdagent.app.kaggle.conf.KaggleBasePropSetting - :settings-show-field-summary: False - :exclude-members: Config - -.. autopydantic_settings:: rdagent.components.coder.factor_coder.config.FactorCoSTEERSettings - :settings-show-field-summary: False - :members: coder_use_cache, file_based_execution_timeout, select_method, max_loop - :exclude-members: Config, fail_task_trial_limit, v1_query_former_trace_limit, v1_query_similar_success_limit, v2_query_component_limit, v2_query_error_limit, v2_query_former_trace_limit, v2_error_summary, v2_knowledge_sampler, v2_add_fail_attempt_to_latest_successful_execution, new_knowledge_base_path, knowledge_base_path, data_folder, data_folder_debug - :no-index: diff --git a/docs/scens/model_agent_med.rst b/docs/scens/model_agent_med.rst deleted file mode 100644 index 294830f0..00000000 --- a/docs/scens/model_agent_med.rst +++ /dev/null @@ -1,128 +0,0 @@ -.. _model_agent_med: - -======================= -Medical Model Agent -======================= - -**πŸ€– Automated Medical Predtion Model Evolution** ------------------------------------------------------------------------------------------- - -πŸ“– Background -~~~~~~~~~~~~~~ -In this scenario, we consider the problem of risk prediction from patients' ICU monitoring data. We use the a public EHR dataset - MIMIC-III and extract a binary classification task for evaluating the framework. -In this task, we aim at predicting the whether the patients will suffer from Acute Respiratory Failure (ARF) based their first 12 hours ICU monitoring data. - -πŸŽ₯ `Demo `_ -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. raw:: html - -
- -
- - -🌟 Introduction -~~~~~~~~~~~~~~~~ - -In this scenario, our automated system proposes hypothesis, constructs model, implements code, receives back-testing, and uses feedbacks. -Hypothesis is iterated in this continuous process. -The system aims to automatically optimise performance metrics of medical prediction thereby finding the optimised code through autonomous research and development. - -Here's an enhanced outline of the steps: - -**Step 1 : Hypothesis Generation πŸ”** - -- Generate and propose initial hypotheses based on previous experiment analysis and domain expertise, with thorough reasoning and justification. - -**Step 2 : Model Creation ✨** - -- Transform the hypothesis into a model. -- Develop, define, and implement a machine learning model, including its name, description, and formulation. - -**Step 3 : Model Implementation πŸ‘¨β€πŸ’»** - -- Implement the model code based on the detailed description. -- Evolve the model iteratively as a developer would, ensuring accuracy and efficiency. - -**Step 4 : Backtesting with MIMIC-III πŸ“‰** - -- Conduct backtesting using the newly developed model on the extracted task from MIMIC-III. -- Evaluate the model's effectiveness and performance in terms of AUROC score. - -**Step 5 : Feedback Analysis πŸ”** - -- Analyze backtest results to assess performance. -- Incorporate feedback to refine hypotheses and improve the model. - -**Step 6 :Hypothesis Refinement ♻️** - -- Refine hypotheses based on feedback from backtesting. -- Repeat the process to continuously improve the model. - -⚑ Quick Start -~~~~~~~~~~~~~~~~~ - -Please refer to the installation part in :doc:`../installation_and_configuration` to prepare your system dependency. - -You can try our demo by running the following command: - -- 🐍 Create a Conda Environment - - - Create a new conda environment with Python (3.10 and 3.11 are well tested in our CI): - - .. code-block:: sh - - conda create -n rdagent python=3.10 - - - Activate the environment: - - .. code-block:: sh - - conda activate rdagent - -- πŸ“¦ Install the RDAgent - - - You can install the RDAgent package from PyPI: - - .. code-block:: sh - - pip install rdagent - -- πŸ“¦ Request PhysioNet Account - - - Apply for an account at `PhysioNet `_. - - Request access to FIDDLE preprocessed data: `FIDDLE Dataset `_. - - Place your username and password in `.env`. - - .. code-block:: bash - - cat << EOF >> .env - DM_USERNAME= - DM_PASSWORD= - EOF - - -- πŸš€ Run the Application - - - You can directly run the application by using the following command: - - .. code-block:: sh - - rdagent med_model - -πŸ› οΈ Usage of modules -~~~~~~~~~~~~~~~~~~~~~ - -.. _Env Config: - -- **Env Config** - -The following environment variables can be set in the `.env` file to customize the application's behavior: - -.. autopydantic_settings:: rdagent.app.data_mining.conf.MedBasePropSetting - :settings-show-field-summary: False - :exclude-members: Config diff --git a/rdagent/app/cli.py b/rdagent/app/cli.py index 5d6567a2..4b6536af 100644 --- a/rdagent/app/cli.py +++ b/rdagent/app/cli.py @@ -17,8 +17,7 @@ from importlib.resources import path as rpath import fire -from rdagent.app.data_mining.model import main as med_model -from rdagent.app.data_science.loop import main as kaggle +from rdagent.app.data_science.loop import main as data_science from rdagent.app.general_model.general_model import ( extract_models_and_implement as general_model, ) @@ -59,12 +58,11 @@ def app(): "fin_factor_report": fin_factor_report, "fin_model": fin_model, "fin_quant": fin_quant, - "med_model": med_model, "general_model": general_model, "ui": ui, "health_check": health_check, "collect_info": collect_info, - "kaggle": kaggle, + "data_science": data_science, "server_ui": server_ui, } ) diff --git a/rdagent/app/data_mining/conf.py b/rdagent/app/data_mining/conf.py deleted file mode 100644 index 9493648b..00000000 --- a/rdagent/app/data_mining/conf.py +++ /dev/null @@ -1,45 +0,0 @@ -from pathlib import Path - -from pydantic_settings import SettingsConfigDict - -from rdagent.components.workflow.conf import BasePropSetting - - -class MedBasePropSetting(BasePropSetting): - model_config = SettingsConfigDict(env_prefix="DM_", protected_namespaces=()) - - # 1) overriding the default - scen: str = "rdagent.scenarios.data_mining.experiment.model_experiment.DMModelScenario" - """Scenario class for data mining model""" - - hypothesis_gen: str = "rdagent.scenarios.data_mining.proposal.model_proposal.DMModelHypothesisGen" - """Hypothesis generation class""" - - hypothesis2experiment: str = "rdagent.scenarios.data_mining.proposal.model_proposal.DMModelHypothesis2Experiment" - """Hypothesis to experiment class""" - - coder: str = "rdagent.scenarios.data_mining.developer.model_coder.DMModelCoSTEER" - """Coder class""" - - runner: str = "rdagent.scenarios.data_mining.developer.model_runner.DMModelRunner" - """Runner class""" - - summarizer: str = "rdagent.scenarios.data_mining.developer.feedback.DMModelExperiment2Feedback" - """Summarizer class""" - - evolving_n: int = 10 - """Number of evolutions""" - - evolving_n: int = 10 - - # 2) Extra config for the scenario - # physionet account - # NOTE: You should apply the account in https://physionet.org/ - username: str = "" - """Physionet account username""" - - password: str = "" - """Physionet account password""" - - -MED_PROP_SETTING = MedBasePropSetting() diff --git a/rdagent/app/data_mining/model.py b/rdagent/app/data_mining/model.py deleted file mode 100644 index ba83b553..00000000 --- a/rdagent/app/data_mining/model.py +++ /dev/null @@ -1,33 +0,0 @@ -import asyncio - -import fire - -from rdagent.app.data_mining.conf import MED_PROP_SETTING -from rdagent.components.workflow.rd_loop import RDLoop -from rdagent.core.exception import ModelEmptyError - - -class ModelRDLoop(RDLoop): - skip_loop_error = (ModelEmptyError,) - - -def main(path=None, step_n=None, loop_n=None, all_duration=None, checkout=True): - """ - Auto R&D Evolving loop for models in a medical scenario. - - You can continue running session by - - .. code-block:: python - - dotenv run -- python rdagent/app/data_mining/model.py $LOG_PATH/__session__/1/0_propose --step_n 1 # `step_n` is a optional paramter - - """ - if path is None: - model_loop = ModelRDLoop(MED_PROP_SETTING) - else: - model_loop = ModelRDLoop.load(path, checkout=checkout) - asyncio.run(model_loop.run(step_n=step_n, loop_n=loop_n, all_duration=all_duration)) - - -if __name__ == "__main__": - fire.Fire(main) diff --git a/rdagent/app/qlib_rd_loop/conf.py b/rdagent/app/qlib_rd_loop/conf.py index 42debfa3..7c889bfe 100644 --- a/rdagent/app/qlib_rd_loop/conf.py +++ b/rdagent/app/qlib_rd_loop/conf.py @@ -67,8 +67,8 @@ class FactorFromReportPropSetting(FactorBasePropSetting): max_factors_per_exp: int = 10000 """Maximum number of factors implemented per experiment""" - is_report_limit_enabled: bool = False - """Limits report processing count if True; processes all if False""" + report_limit: int = 10000 + """Maximum number of reports to process""" class QuantBasePropSetting(BasePropSetting): diff --git a/rdagent/app/qlib_rd_loop/factor_from_report.py b/rdagent/app/qlib_rd_loop/factor_from_report.py index e5938af6..5f596986 100644 --- a/rdagent/app/qlib_rd_loop/factor_from_report.py +++ b/rdagent/app/qlib_rd_loop/factor_from_report.py @@ -57,7 +57,7 @@ def generate_hypothesis(factor_result: dict, report_content: str) -> str: ) -def extract_hypothesis_and_exp_from_reports(report_file_path: str) -> Tuple[QlibFactorExperiment, Hypothesis]: +def extract_hypothesis_and_exp_from_reports(report_file_path: str) -> QlibFactorExperiment | None: """ Extract hypothesis and experiment details from report files. @@ -65,17 +65,15 @@ def extract_hypothesis_and_exp_from_reports(report_file_path: str) -> Tuple[Qlib report_file_path (str): Path to the report file. Returns: - Tuple[QlibFactorExperiment, Hypothesis]: The extracted experiment and generated hypothesis. + QlibFactorExperiment: An instance of QlibFactorExperiment containing the extracted details. + None: If no valid experiment is found in the report. """ - with logger.tag("extract_factors_and_implement"): - with logger.tag("load_factor_tasks"): - exp = FactorExperimentLoaderFromPDFfiles().load(report_file_path) - if exp is None or exp.sub_tasks == []: - return None, None + exp = FactorExperimentLoaderFromPDFfiles().load(report_file_path) + if exp is None or exp.sub_tasks == []: + return None - with logger.tag("load_pdf_screenshot"): - pdf_screenshot = extract_first_page_screenshot_from_pdf(report_file_path) - logger.log_object(pdf_screenshot) + pdf_screenshot = extract_first_page_screenshot_from_pdf(report_file_path) + logger.log_object(pdf_screenshot, tag="load_pdf_screenshot") docs_dict = load_and_process_pdfs_by_langchain(report_file_path) @@ -92,7 +90,7 @@ def extract_hypothesis_and_exp_from_reports(report_file_path: str) -> Tuple[Qlib report_content = "\n".join(docs_dict.values()) hypothesis = generate_hypothesis(factor_result, report_content) exp.hypothesis = hypothesis - return exp, hypothesis + return exp class FactorReportLoop(FactorRDLoop, metaclass=LoopMeta): @@ -105,47 +103,31 @@ class FactorReportLoop(FactorRDLoop, metaclass=LoopMeta): else: self.judge_pdf_data_items = [i for i in Path(report_folder).rglob("*.pdf")] - self.pdf_file_index = 0 - self.valid_pdf_file_count = 0 - self.current_loop_hypothesis = None - self.current_loop_exp = None - self.steps = ["propose_hypo_exp", "propose", "direct_exp_gen", "coding", "running", "feedback"] + self.loop_n = min(len(self.judge_pdf_data_items), FACTOR_FROM_REPORT_PROP_SETTING.report_limit) - def propose_hypo_exp(self, prev_out: dict[str, Any]): + def direct_exp_gen(self, prev_out: dict[str, Any]): while True: - if FACTOR_FROM_REPORT_PROP_SETTING.is_report_limit_enabled and self.valid_pdf_file_count > 15: - break - report_file_path = self.judge_pdf_data_items[self.pdf_file_index] - logger.info(f"Processing number {self.pdf_file_index} report: {report_file_path}") - self.pdf_file_index += 1 - exp, hypothesis = extract_hypothesis_and_exp_from_reports(str(report_file_path)) + report_file_path = self.judge_pdf_data_items[self.loop_idx] + logger.info(f"Processing number {self.loop_idx} report: {report_file_path}") + exp = extract_hypothesis_and_exp_from_reports(str(report_file_path)) if exp is None: continue - self.valid_pdf_file_count += 1 - exp.based_experiments = [QlibFactorExperiment(sub_tasks=[], hypothesis=hypothesis)] + [ + exp.based_experiments = [QlibFactorExperiment(sub_tasks=[], hypothesis=exp.hypothesis)] + [ t[0] for t in self.trace.hist if t[1] ] exp.sub_workspace_list = exp.sub_workspace_list[: FACTOR_FROM_REPORT_PROP_SETTING.max_factors_per_exp] exp.sub_tasks = exp.sub_tasks[: FACTOR_FROM_REPORT_PROP_SETTING.max_factors_per_exp] - logger.log_object(hypothesis, tag="hypothesis generation") + logger.log_object(exp.hypothesis, tag="hypothesis generation") logger.log_object(exp.sub_tasks, tag="experiment generation") - self.current_loop_hypothesis = hypothesis - self.current_loop_exp = exp - return None - - def propose(self, prev_out: dict[str, Any]): - return self.current_loop_hypothesis - - def direct_exp_gen(self, prev_out: dict[str, Any]): - return {"propose": self.current_loop_hypothesis, "exp_gen": self.current_loop_exp} + return exp def coding(self, prev_out: dict[str, Any]): - exp = self.coder.develop(prev_out["direct_exp_gen"]["exp_gen"]) + exp = self.coder.develop(prev_out["direct_exp_gen"]) logger.log_object(exp.sub_workspace_list, tag="coder result") return exp -def main(report_folder=None, path=None, step_n=None, loop_n=None, all_duration=None, checkout=True): +def main(report_folder=None, path=None, all_duration=None, checkout=True): """ Auto R&D Evolving loop for fintech factors (the factors are extracted from finance reports). @@ -161,7 +143,7 @@ def main(report_folder=None, path=None, step_n=None, loop_n=None, all_duration=N else: model_loop = FactorReportLoop(report_folder=report_folder) - asyncio.run(model_loop.run(step_n=step_n, loop_n=loop_n, all_duration=all_duration)) + asyncio.run(model_loop.run(all_duration=all_duration)) if __name__ == "__main__": diff --git a/rdagent/app/qlib_rd_loop/quant.py b/rdagent/app/qlib_rd_loop/quant.py index 3a3a8b34..c956c940 100644 --- a/rdagent/app/qlib_rd_loop/quant.py +++ b/rdagent/app/qlib_rd_loop/quant.py @@ -97,7 +97,7 @@ class QuantRDLoop(RDLoop): e = prev_out.get(self.EXCEPTION_KEY, None) if e is not None: feedback = HypothesisFeedback( - observations=e, + observations=str(e), hypothesis_evaluation="", new_hypothesis="", reason="", diff --git a/rdagent/components/coder/CoSTEER/__init__.py b/rdagent/components/coder/CoSTEER/__init__.py index 3c99e549..51d30a72 100644 --- a/rdagent/components/coder/CoSTEER/__init__.py +++ b/rdagent/components/coder/CoSTEER/__init__.py @@ -103,7 +103,7 @@ class CoSTEER(Developer[Experiment]): assert isinstance(evo_exp, Experiment) # multiple inheritance logger.log_object(evo_exp.sub_workspace_list, tag="evolving code") for sw in evo_exp.sub_workspace_list: - logger.info(f"evolving code workspace: {sw}") + logger.info(f"evolving workspace: {sw}") if (datetime.now() - start_datetime).seconds > self.max_seconds: logger.info(f"Reached max time limit {self.max_seconds} seconds, stop evolving") break diff --git a/rdagent/components/coder/model_coder/prompts.yaml b/rdagent/components/coder/model_coder/prompts.yaml index 126c986a..20e5462a 100644 --- a/rdagent/components/coder/model_coder/prompts.yaml +++ b/rdagent/components/coder/model_coder/prompts.yaml @@ -1,41 +1,29 @@ extract_model_formulation_system: |- offer description of the proposed model in this paper, write a latex formula with variable as well as the architecture of the model. the format should be like { - "model_name (The name of the model)": { - "description": "A detailed description of the model", - "formulation": "A LaTeX formula representing the model's formulation", - "architecture": "A detailed description of the model's architecture, e.g., neural network layers or tree structures", - "variables": { - "\\hat{y}_u": "The predicted output for node u", - "variable_name_2": "Description of variable 2", - "variable_name_3": "Description of variable 3" - }, - "hyperparameters": { - "hyperparameter_name_1": "value of hyperparameter 1", - "hyperparameter_name_2": "value of hyperparameter 2", - "hyperparameter_name_3": "value of hyperparameter 3" - }, - "model_type": "Tabular or TimeSeries or Graph or XGBoost" # Should be one of "Tabular", "TimeSeries", "Graph", or "XGBoost" - } - } - Eg. - { - "ABC Model": { - "description": "A detailed description of the model", - "formulation": "A LaTeX formula representing the model's formulation", - "architecture": "A detailed description of the model's architecture, e.g., neural network layers or tree structures", - "variables": { - "\\hat{y}_u": "The predicted output for node u", - "variable_name_2": "Description of variable 2", - "variable_name_3": "Description of variable 3" - }, - "hyperparameters": { - "hyperparameter_name_1": "value of hyperparameter 1", - "hyperparameter_name_2": "value of hyperparameter 2", - "hyperparameter_name_3": "value of hyperparameter 3" - }, - "model_type": "Tabular or TimeSeries or Graph or RandomForest or XGBoost" # If torch & Neural network models are required, the choice should be one of "Tabular", "TimeSeries", or "Graph" - } + "model_name (The name of the model)": { + "description": "A detailed description of the model", + "formulation": "A LaTeX formula representing the model's formulation", + "architecture": "A detailed description of the model's architecture, e.g., neural network layers or tree structures", + "variables": { + "\\hat{y}_u": "The predicted output for node u", + "variable_name_2": "Description of variable 2", + "variable_name_3": "Description of variable 3" + }, + "hyperparameters": { + "hyperparameter_name_1": "value of hyperparameter 1", + "hyperparameter_name_2": "value of hyperparameter 2", + "hyperparameter_name_3": "value of hyperparameter 3" + }, + "training_hyperparameters" { # All values are for reference; you can set them yourself + "n_epochs": "100", + "lr": "1e-3", + "early_stop": 10, + "batch_size": 256, + "weight_decay": 1e-4, + } + "model_type": "Tabular or TimeSeries or Graph or XGBoost" # Should be one of "Tabular", "TimeSeries", "Graph", or "XGBoost" + } } such format content should be begin with ```json and end with ``` and the content should be in json format. diff --git a/rdagent/components/coder/model_coder/task_loader.py b/rdagent/components/coder/model_coder/task_loader.py index a9922e28..4735e703 100644 --- a/rdagent/components/coder/model_coder/task_loader.py +++ b/rdagent/components/coder/model_coder/task_loader.py @@ -3,6 +3,8 @@ from __future__ import annotations import json import re +from pydantic import BaseModel, Field + from rdagent.components.coder.model_coder.model import ModelTask from rdagent.components.document_reader.document_reader import ( load_and_process_pdfs_by_langchain, @@ -12,6 +14,7 @@ from rdagent.log import rdagent_logger as logger from rdagent.oai.llm_utils import APIBackend from rdagent.scenarios.qlib.experiment.model_experiment import QlibModelExperiment from rdagent.utils.agent.tpl import T +from rdagent.utils.workflow import wait_retry def extract_model_from_doc(doc_content: str) -> dict: @@ -105,6 +108,7 @@ class ModelExperimentLoaderFromDict(ModelTaskLoader): architecture=model_data["architecture"], variables=model_data["variables"], hyperparameters=model_data["hyperparameters"], + training_hyperparameters=model_data["training_hyperparameters"], model_type=model_data["model_type"], ) task_l.append(task) @@ -112,6 +116,7 @@ class ModelExperimentLoaderFromDict(ModelTaskLoader): class ModelExperimentLoaderFromPDFfiles(ModelTaskLoader): + @wait_retry(retry_n=5) def load(self, file_or_folder_path: str) -> dict: docs_dict = load_and_process_pdfs_by_langchain(file_or_folder_path) # dict{file_path:content} model_dict = extract_model_from_docs( diff --git a/rdagent/components/proposal/__init__.py b/rdagent/components/proposal/__init__.py index 424233f0..d7cd4e8b 100644 --- a/rdagent/components/proposal/__init__.py +++ b/rdagent/components/proposal/__init__.py @@ -11,6 +11,7 @@ from rdagent.core.proposal import ( ) from rdagent.oai.llm_utils import APIBackend from rdagent.utils.agent.tpl import T +from rdagent.utils.workflow import wait_retry class LLMHypothesisGen(HypothesisGen): @@ -83,6 +84,7 @@ class LLMHypothesis2Experiment(Hypothesis2Experiment[Experiment]): @abstractmethod def convert_response(self, response: str, hypothesis: Hypothesis, trace: Trace) -> Experiment: ... + @wait_retry(retry_n=5) def convert(self, hypothesis: Hypothesis, trace: Trace) -> Experiment: context, json_flag = self.prepare_context(hypothesis, trace) system_prompt = T(".prompts:hypothesis2experiment.system_prompt").r( diff --git a/rdagent/components/workflow/rd_loop.py b/rdagent/components/workflow/rd_loop.py index 5b315d1e..004ceb37 100644 --- a/rdagent/components/workflow/rd_loop.py +++ b/rdagent/components/workflow/rd_loop.py @@ -74,7 +74,7 @@ class RDLoop(LoopBase, metaclass=LoopMeta): e = prev_out.get(self.EXCEPTION_KEY, None) if e is not None: feedback = HypothesisFeedback( - observations=e, + observations=str(e), hypothesis_evaluation="", new_hypothesis="", reason="", diff --git a/rdagent/log/logger.py b/rdagent/log/logger.py index 7cb57146..a3d7df5c 100644 --- a/rdagent/log/logger.py +++ b/rdagent/log/logger.py @@ -20,7 +20,7 @@ from rdagent.core.utils import SingletonBaseClass, import_class from .base import Storage from .storage import FileStorage -from .utils import LogColors, get_caller_info +from .utils import get_caller_info class RDAgentLog(SingletonBaseClass): @@ -106,14 +106,10 @@ class RDAgentLog(SingletonBaseClass): return pid_chain def log_object(self, obj: object, *, tag: str = "") -> None: - caller_info = get_caller_info(level=2) tag = f"{self._tag}.{tag}.{self.get_pids()}".strip(".") for storage in [self.storage] + self.other_storages: - logp = storage.log(obj, tag=tag) - logger.patch(lambda r: r.update(caller_info)).info( - f"{LogColors.GRAY}Log object to [{storage}], uri: {logp}{LogColors.END}" - ) + storage.log(obj, tag=tag) def _log(self, level: str, msg: str, *, tag: str = "", raw: bool = False) -> None: caller_info = get_caller_info(level=3) diff --git a/rdagent/log/ui/app.py b/rdagent/log/ui/app.py index 410703c2..86655e33 100644 --- a/rdagent/log/ui/app.py +++ b/rdagent/log/ui/app.py @@ -24,7 +24,6 @@ from rdagent.core.scenario import Scenario from rdagent.log.base import Message from rdagent.log.storage import FileStorage from rdagent.log.ui.qlib_report_figure import report_figure -from rdagent.scenarios.data_mining.experiment.model_experiment import DMModelScenario from rdagent.scenarios.general_model.scenario import GeneralModelScenario from rdagent.scenarios.kaggle.experiment.scenario import KGScenario from rdagent.scenarios.qlib.experiment.factor_experiment import QlibFactorScenario @@ -63,7 +62,6 @@ QLIB_SELECTED_METRICS = [ SIMILAR_SCENARIOS = ( QlibModelScenario, - DMModelScenario, QlibFactorScenario, QlibFactorFromReportScenario, QlibQuantScenario, @@ -193,9 +191,7 @@ def get_msgs_until(end_func: Callable[[Message], bool] = lambda _: True): and msg.content.based_experiments[-1].result is not None ): sms = msg.content.based_experiments[-1].result - if isinstance(state.scenario, DMModelScenario): - sms.index = ["AUROC"] - elif isinstance( + if isinstance( state.scenario, ( QlibModelScenario, @@ -211,31 +207,23 @@ def get_msgs_until(end_func: Callable[[Message], bool] = lambda _: True): state.all_metric_series.append(sms_all) # common metrics - if msg.content.result is None: - if isinstance(state.scenario, DMModelScenario): - state.metric_series.append( - pd.Series([None], index=["AUROC"], name=f"Round {state.lround}") - ) - else: - sms = msg.content.result - if isinstance(state.scenario, DMModelScenario): - sms.index = ["AUROC"] - elif isinstance( - state.scenario, - ( - QlibModelScenario, - QlibFactorFromReportScenario, - QlibFactorScenario, - QlibQuantScenario, - ), - ): - sms_all = sms - sms = sms.loc[QLIB_SELECTED_METRICS] + sms = msg.content.result + if isinstance( + state.scenario, + ( + QlibModelScenario, + QlibFactorFromReportScenario, + QlibFactorScenario, + QlibQuantScenario, + ), + ): + sms_all = sms + sms = sms.loc[QLIB_SELECTED_METRICS] - sms.name = f"Round {state.lround}" - sms_all.name = f"Round {state.lround}" - state.metric_series.append(sms) - state.all_metric_series.append(sms_all) + sms.name = f"Round {state.lround}" + sms_all.name = f"Round {state.lround}" + state.metric_series.append(sms) + state.all_metric_series.append(sms_all) elif "hypothesis generation" in tags: state.hypotheses[state.lround] = msg.content elif "evolving code" in tags: @@ -564,7 +552,7 @@ def research_window(): st.subheader(title, divider="blue", anchor="_research") if isinstance(state.scenario, SIMILAR_SCENARIOS): # pdf image - if pim := state.msgs[round]["extract_factors_and_implement.load_pdf_screenshot"]: + if pim := state.msgs[round]["load_pdf_screenshot"]: for i in range(min(2, len(pim))): st.image(pim[i].content, use_container_width=True) diff --git a/rdagent/oai/backend/litellm.py b/rdagent/oai/backend/litellm.py index f2667c37..051f2d2b 100644 --- a/rdagent/oai/backend/litellm.py +++ b/rdagent/oai/backend/litellm.py @@ -67,7 +67,11 @@ class LiteLLMAPIBackend(APIBackend): """ model_name = LITELLM_SETTINGS.embedding_model logger.info(f"{LogColors.GREEN}Using emb model{LogColors.END} {model_name}", tag="debug_litellm_emb") - logger.info(f"Creating embedding for: {input_content_list}", tag="debug_litellm_emb") + if LITELLM_SETTINGS.log_llm_chat_content: + logger.info( + f"{LogColors.MAGENTA}Creating embedding{LogColors.END} for: {input_content_list}", + tag="debug_litellm_emb", + ) response = embedding( model=model_name, input=input_content_list, @@ -90,7 +94,8 @@ class LiteLLMAPIBackend(APIBackend): if json_mode and supports_response_schema(model=LITELLM_SETTINGS.chat_model): kwargs["response_format"] = {"type": "json_object"} - logger.info(self._build_log_messages(messages), tag="llm_messages") + if LITELLM_SETTINGS.log_llm_chat_content: + logger.info(self._build_log_messages(messages), tag="llm_messages") # Call LiteLLM completion model = LITELLM_SETTINGS.chat_model temperature = LITELLM_SETTINGS.chat_temperature @@ -124,7 +129,8 @@ class LiteLLMAPIBackend(APIBackend): logger.info(f"{LogColors.GREEN}Using chat model{LogColors.END} {model}", tag="llm_messages") if LITELLM_SETTINGS.chat_stream: - logger.info(f"{LogColors.BLUE}assistant:{LogColors.END}", tag="llm_messages") + if LITELLM_SETTINGS.log_llm_chat_content: + logger.info(f"{LogColors.BLUE}assistant:{LogColors.END}", tag="llm_messages") content = "" finish_reason = None for message in response: @@ -135,9 +141,10 @@ class LiteLLMAPIBackend(APIBackend): message["choices"][0]["delta"]["content"] or "" ) # when finish_reason is "stop", content is None content += chunk - logger.info(LogColors.CYAN + chunk + LogColors.END, raw=True, tag="llm_messages") - - logger.info("\n", raw=True, tag="llm_messages") + if LITELLM_SETTINGS.log_llm_chat_content: + logger.info(LogColors.CYAN + chunk + LogColors.END, raw=True, tag="llm_messages") + if LITELLM_SETTINGS.log_llm_chat_content: + logger.info("\n", raw=True, tag="llm_messages") else: content = str(response.choices[0].message.content) finish_reason = response.choices[0].finish_reason @@ -146,7 +153,10 @@ class LiteLLMAPIBackend(APIBackend): if finish_reason and finish_reason != "stop" else "" ) - logger.info(f"{LogColors.BLUE}assistant:{LogColors.END} {finish_reason_str}\n{content}", tag="llm_messages") + if LITELLM_SETTINGS.log_llm_chat_content: + logger.info( + f"{LogColors.BLUE}assistant:{LogColors.END} {finish_reason_str}\n{content}", tag="llm_messages" + ) global ACC_COST try: diff --git a/rdagent/scenarios/data_mining/developer/feedback.py b/rdagent/scenarios/data_mining/developer/feedback.py deleted file mode 100644 index f8012a37..00000000 --- a/rdagent/scenarios/data_mining/developer/feedback.py +++ /dev/null @@ -1,62 +0,0 @@ -# TODO: -# Implement to feedback. - -import json -from pathlib import Path -from typing import Dict - -from rdagent.core.experiment import Experiment -from rdagent.core.proposal import Experiment2Feedback, HypothesisFeedback, Trace -from rdagent.log import rdagent_logger as logger -from rdagent.oai.llm_utils import APIBackend -from rdagent.utils import convert2bool -from rdagent.utils.agent.tpl import T - -DIRNAME = Path(__file__).absolute().resolve().parent - - -class DMModelExperiment2Feedback(Experiment2Feedback): - """Generated feedbacks on the hypothesis from **Executed** Implementations of different tasks & their comparisons with previous performances""" - - def generate_feedback(self, exp: Experiment, trace: Trace) -> HypothesisFeedback: - """ - The `ti` should be executed and the results should be included, as well as the comparison between previous results (done by LLM). - For example: `mlflow` of Qlib will be included. - """ - hypothesis = exp.hypothesis - - logger.info("Generating feedback...") - # Define the system prompt for hypothesis feedback - system_prompt = T("scenarios.qlib.prompts:model_feedback_generation.system").r() - - # Define the user prompt for hypothesis feedback - context = trace.scen - SOTA_hypothesis, SOTA_experiment = trace.get_sota_hypothesis_and_experiment() - - user_prompt = T("scenarios.qlib.prompts:model_feedback_generation.user").r( - context=context, - last_hypothesis=SOTA_hypothesis, - last_task=SOTA_experiment.sub_tasks[0].get_task_information() if SOTA_hypothesis else None, - last_code=SOTA_experiment.sub_workspace_list[0].file_dict.get("model.py") if SOTA_hypothesis else None, - last_result=SOTA_experiment.result if SOTA_hypothesis else None, - hypothesis=hypothesis, - exp=exp, - ) - - # Call the APIBackend to generate the response for hypothesis feedback - response_hypothesis = APIBackend().build_messages_and_create_chat_completion( - user_prompt=user_prompt, - system_prompt=system_prompt, - json_mode=True, - json_target_type=Dict[str, str | bool | int], - ) - - # Parse the JSON response to extract the feedback - response_json_hypothesis = json.loads(response_hypothesis) - return HypothesisFeedback( - observations=response_json_hypothesis.get("Observations", "No observations provided"), - hypothesis_evaluation=response_json_hypothesis.get("Feedback for Hypothesis", "No feedback provided"), - new_hypothesis=response_json_hypothesis.get("New Hypothesis", "No new hypothesis provided"), - reason=response_json_hypothesis.get("Reasoning", "No reasoning provided"), - decision=convert2bool(response_json_hypothesis.get("Decision", "false")), - ) diff --git a/rdagent/scenarios/data_mining/developer/model_coder.py b/rdagent/scenarios/data_mining/developer/model_coder.py deleted file mode 100644 index 1011a968..00000000 --- a/rdagent/scenarios/data_mining/developer/model_coder.py +++ /dev/null @@ -1,3 +0,0 @@ -from rdagent.components.coder.model_coder import ModelCoSTEER - -DMModelCoSTEER = ModelCoSTEER diff --git a/rdagent/scenarios/data_mining/developer/model_runner.py b/rdagent/scenarios/data_mining/developer/model_runner.py deleted file mode 100644 index f9238323..00000000 --- a/rdagent/scenarios/data_mining/developer/model_runner.py +++ /dev/null @@ -1,26 +0,0 @@ -from rdagent.components.runner import CachedRunner -from rdagent.core.exception import ModelEmptyError -from rdagent.core.utils import cache_with_pickle -from rdagent.log import rdagent_logger as logger -from rdagent.scenarios.data_mining.experiment.model_experiment import DMModelExperiment - - -class DMModelRunner(CachedRunner[DMModelExperiment]): - @cache_with_pickle(CachedRunner.get_cache_key, CachedRunner.assign_cached_result) - def develop(self, exp: DMModelExperiment) -> DMModelExperiment: - if exp.sub_workspace_list[0].file_dict.get("model.py") is None: - raise ModelEmptyError("model.py is empty") - # to replace & inject code - exp.experiment_workspace.inject_files(**{"model.py": exp.sub_workspace_list[0].file_dict["model.py"]}) - - env_to_use = {"PYTHONPATH": "./"} - - result, stdout = exp.experiment_workspace.execute(run_env=env_to_use) - - if result is None: - logger.error(f"Experiment failed to run, stdout: {stdout}") - raise ModelEmptyError(f"Failed to run this experiment, because {stdout}") - - exp.result = result - - return exp diff --git a/rdagent/scenarios/data_mining/docker/Dockerfile b/rdagent/scenarios/data_mining/docker/Dockerfile deleted file mode 100644 index fae6e62e..00000000 --- a/rdagent/scenarios/data_mining/docker/Dockerfile +++ /dev/null @@ -1,25 +0,0 @@ -FROM pytorch/pytorch:2.2.1-cuda12.1-cudnn8-runtime -# For GPU support, please choose the proper tag from https://hub.docker.com/r/pytorch/pytorch/tags - -RUN apt-get clean && apt-get update && apt-get install -y \ - curl \ - vim \ - git \ - build-essential \ - && rm -rf /var/lib/apt/lists/* - -WORKDIR /workspace - -RUN python -m pip install numpy -RUN python -m pip install --upgrade cython -# RUN python -m pip install -e . - -RUN python -m pip install pandas -# RUN pip install pyg_lib torch_scatter torch_sparse torch_cluster -f https://data.pyg.org/whl/torch-2.3.0%2Bcu121.html -RUN pip install torch_geometric -RUN pip install ogb -RUN pip install networkx -RUN pip install scikit-learn -RUN pip install catboost -RUN pip install xgboost -RUN pip install sparse diff --git a/rdagent/scenarios/data_mining/experiment/model_experiment.py b/rdagent/scenarios/data_mining/experiment/model_experiment.py deleted file mode 100644 index 63ab3dfa..00000000 --- a/rdagent/scenarios/data_mining/experiment/model_experiment.py +++ /dev/null @@ -1,75 +0,0 @@ -from pathlib import Path - -from rdagent.components.coder.model_coder.model import ( - ModelExperiment, - ModelFBWorkspace, - ModelTask, -) -from rdagent.core.experiment import Task -from rdagent.core.scenario import Scenario -from rdagent.scenarios.data_mining.experiment.workspace import DMFBWorkspace -from rdagent.utils.agent.tpl import T - - -class DMModelExperiment(ModelExperiment[ModelTask, DMFBWorkspace, ModelFBWorkspace]): - def __init__(self, *args, **kwargs) -> None: - super().__init__(*args, **kwargs) - self.experiment_workspace = DMFBWorkspace(template_folder_path=Path(__file__).parent / "model_template") - - -class DMModelScenario(Scenario): - @property - def background(self) -> str: - return T(".prompts:dm_model_background").r() - - @property - def source_data(self) -> str: - raise NotImplementedError("source_data is not implemented") - - @property - def output_format(self) -> str: - return T(".prompts:dm_model_output_format").r() - - @property - def interface(self) -> str: - return T(".prompts:dm_model_interface").r() - - @property - def simulator(self) -> str: - return T(".prompts:dm_model_simulator").r() - - @property - def rich_style_description(self) -> str: - return """ -### MIMIC-III Model Evolving Automatic R&D Demo - -#### [Overview](#_summary) - -The demo showcases the iterative process of hypothesis generation, knowledge construction, and decision-making in model construction in a clinical prediction task. The model should predict whether a patient would suffer from Acute Respiratory Failure (ARF) based on first 12 hours ICU monitoring data. - -#### [Automated R&D](#_rdloops) - -- **[R (Research)](#_research)** - - Iteration of ideas and hypotheses. - - Continuous learning and knowledge construction. - -- **[D (Development)](#_development)** - - Evolving code generation and model refinement. - - Automated implementation and testing of models. - -#### [Objective](#_summary) - -To demonstrate the dynamic evolution of models through the R&D loop, emphasizing how each iteration enhances the model performance and reliability. The performane is measured by the AUROC score (Area Under the Receiver Operating Characteristic), which is a commonly used metric for binary classification. """ - - def get_scenario_all_desc( - self, task: Task | None = None, filtered_tag: str | None = None, simple_background: bool | None = None - ) -> str: - return f"""Background of the scenario: -{self.background} -The interface you should follow to write the runnable code: -{self.interface} -The output of your code should be in the format: -{self.output_format} -The simulator user can use to test your model: -{self.simulator} -""" diff --git a/rdagent/scenarios/data_mining/experiment/model_template/README.md b/rdagent/scenarios/data_mining/experiment/model_template/README.md deleted file mode 100644 index fb7c22c9..00000000 --- a/rdagent/scenarios/data_mining/experiment/model_template/README.md +++ /dev/null @@ -1,3 +0,0 @@ -## This folder is a template to be copied from for each model implementation & running process. - -Components: Dummy model.py, versatile conf.yaml, and a result reader. diff --git a/rdagent/scenarios/data_mining/experiment/model_template/train.py b/rdagent/scenarios/data_mining/experiment/model_template/train.py deleted file mode 100644 index f00ade8e..00000000 --- a/rdagent/scenarios/data_mining/experiment/model_template/train.py +++ /dev/null @@ -1,116 +0,0 @@ -import os -import random -from pathlib import Path - -import numpy as np -import pandas as pd -import sparse -import torch -import torch.nn as nn -import torch.nn.functional as F -from model import model_cls -from sklearn.metrics import accuracy_score, roc_auc_score -from torch.utils.data import DataLoader, Dataset -from torchvision import datasets, transforms - -# Set device for training -device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") -# device = torch.device("cpu") - - -class MyDataset(Dataset): - def __init__(self, x, label, device): - self.x1 = x - self.label = label - self.device = device - - def __len__(self): - return len(self.label) - - def __getitem__(self, idx): - if torch.is_tensor(idx): - idx = idx.tolist() - return torch.FloatTensor(self.x1[idx]).to(self.device), torch.tensor(self.label[idx], dtype=torch.float).to( - self.device - ) - - -def collate_fn(batch): - x, label = [], [] - for data in batch: - x.append(data[0]) - label.append(data[1]) - return torch.stack(x, 0), torch.stack(label, 0) - - -datapath = "/root/.data" -# datapath = '/home/v-suhancui/RD-Agent/physionet.org/files/mimic-eicu-fiddle-feature/1.0.0/FIDDLE_mimic3' - - -X = sparse.load_npz(datapath + "/features/ARF_12h/X.npz").todense() -df_pop = pd.read_csv(datapath + "/population/ARF_12h.csv")["ARF_LABEL"] - -X = X.transpose(0, 2, 1) - -indices = [i for i in range(len(df_pop))] -random.shuffle(indices) -split_point = int(0.7 * len(df_pop)) - -X_train, y_train = X[indices[:split_point]], np.array(df_pop[indices[:split_point]]) -X_test, y_test = X[indices[split_point:]], np.array(df_pop[indices[split_point:]]) - - -train_dataloader = DataLoader( - MyDataset(X_train, y_train, device), collate_fn=collate_fn, shuffle=True, drop_last=True, batch_size=64 -) -test_dataloader = DataLoader( - MyDataset(X_test, y_test, device), collate_fn=collate_fn, shuffle=False, drop_last=False, batch_size=64 -) - -num_features = 4816 -num_timesteps = 12 -# Define the optimizer and loss function -model = model_cls(num_features=num_features, num_timesteps=num_timesteps).to(device) - -optimizer = torch.optim.Adam(model.parameters(), lr=0.0001) -criterion = nn.CrossEntropyLoss() - - -# Train the model -def eval_auc(model): - y_pred = [] - for data in test_dataloader: - x, y = data - out = model(x) - y_pred.append(out.cpu().detach().numpy()) - return roc_auc_score(y_test, np.concatenate(y_pred)) - - -best = 0.0 -best_model = None - -for i in range(15): - for data in train_dataloader: - x, y = data - out = model(x) - optimizer.zero_grad() - loss = criterion(out.squeeze(), y) - loss.backward() - optimizer.step() - roc = eval_auc(model) - if roc > best: - best = roc - best_model = model - -y_pred = [] -for data in test_dataloader: - x, y = data - out = best_model(x) - y_pred.append(out.cpu().detach().numpy()) - -acc = roc_auc_score(y_test, np.concatenate(y_pred)) - -print(acc) - -res = pd.Series(data=[acc], index=["AUROC"]) -res.to_csv("./submission.csv") diff --git a/rdagent/scenarios/data_mining/experiment/prompts.yaml b/rdagent/scenarios/data_mining/experiment/prompts.yaml deleted file mode 100644 index 43db687f..00000000 --- a/rdagent/scenarios/data_mining/experiment/prompts.yaml +++ /dev/null @@ -1,54 +0,0 @@ -dm_model_background: |- - The model is a machine learning or deep learning structure used in clinical settings to predict whether the patient will suffer from acute respiratory failure (ARF) based on their vital signs monitored in ICU. - The data is extracted from MIMIC-III using FIDDLE pipeline, and we focus on the ARF_12h prediction task, which means we use the first 12 hours data to predict the onset of ARF on discharge. - The model is defined in the following parts: - 1. Name: The name of the model. - 2. Description: The description of the model. - 3. Architecture: The detailed architecture of the model, such as neural network layers or tree structures. - The model should provide clear and detailed documentation of its architecture and hyperparameters. - -dm_model_interface: |- - Your python code should follow the interface to better interact with the user's system. - You code should contain several parts: - 1. The import part: import the necessary libraries. - 2. A class which is a sub-class of pytorch.nn.Module. This class should should have a init function and a forward function which inputs a tensor and outputs a tensor. - 3. Set a variable called "model_cls" to the class you defined. - - The user will save your code into a python file called "model.py". Then the user imports model_cls in file "model.py" after setting the cwd into the directory: - ```python - from model import model_cls - ``` - So your python code should follow the pattern: - ```python - class XXXModel(torch.nn.Module): - ... - model_cls = XXXModel - ``` - - The model has one type, "TimeSeries". The input shape to a time series model is (batch_size, num_features, num_timesteps). The output shape of the model should be (batch_size, 1). - The "batch_size" is a dynamic value which is determined by the input of forward function. - The "num_features" and "num_timesteps" are static which will be provided to the model through init function. - User will initialize the time series model with the following code: - ```python - model = model_cls(num_features=num_features, num_timesteps=num_timesteps) - ``` - No other parameters will be passed to the model so give other parameters a default value or just make them static. - - The input tensor shape is (batch_size, num_features, num_timesteps) which is different from the normal time series input shape of (batch_size, num_timesteps, num_features). Please write code accordingly. - - Note that for nn.Conv1d() layers, please do not permute the input tensor as the in_channel dimension should match the num_feature dimension. - - The output shape should be (batch_size, 1) with sigmoid activation since we have binary labels. - - Don't write any try-except block in your python code. The user will catch the exception message and provide the feedback to you. Also, don't write main function in your python code. The user will call the forward method in the model_cls to get the output tensor. - - Please notice that your model should only use current features as input. The user will provide the input tensor to the model's forward function. - - -dm_model_output_format: |- - Your output should be a tensor with shape (batch_size, 1). - The output tensor should be saved in a file named "output.pth" in the same directory as your python file. - The user will evaluate the shape of the output tensor so the tensor read from "output.pth" should be 8 numbers. - -dm_model_simulator: |- - The models will be put to train on MIMIC-III dataset and evaluate their performance in terms of roc score (Area Under Curve Receiver Operating Characteristics Curve). Hypothesis is improved upon checking the feedback on the results. \ No newline at end of file diff --git a/rdagent/scenarios/data_mining/experiment/workspace.py b/rdagent/scenarios/data_mining/experiment/workspace.py deleted file mode 100644 index 4cfa5593..00000000 --- a/rdagent/scenarios/data_mining/experiment/workspace.py +++ /dev/null @@ -1,31 +0,0 @@ -from pathlib import Path - -import pandas as pd - -from rdagent.app.data_mining.conf import MED_PROP_SETTING -from rdagent.core.experiment import FBWorkspace -from rdagent.log import rdagent_logger as logger -from rdagent.utils.env import DMDockerEnv - - -class DMFBWorkspace(FBWorkspace): - def __init__(self, template_folder_path: Path, *args, **kwargs) -> None: - super().__init__(*args, **kwargs) - self.inject_code_from_folder(template_folder_path) - - def execute(self, run_env: dict = {}, *args, **kwargs) -> str: - qtde = DMDockerEnv() - qtde.prepare(MED_PROP_SETTING.username, MED_PROP_SETTING.password) - - execute_log = qtde.run( - local_path=str(self.workspace_path), - entry=f"python train.py", - env=run_env, - ) - - csv_path = self.workspace_path / "submission.csv" - - if not csv_path.exists(): - logger.error(f"File {csv_path} does not exist.") - return None - return pd.read_csv(csv_path, index_col=0).iloc[:, 0] diff --git a/rdagent/scenarios/data_mining/proposal/model_proposal.py b/rdagent/scenarios/data_mining/proposal/model_proposal.py deleted file mode 100644 index 2b3df085..00000000 --- a/rdagent/scenarios/data_mining/proposal/model_proposal.py +++ /dev/null @@ -1,122 +0,0 @@ -import json -from typing import List, Tuple - -from rdagent.components.coder.model_coder.model import ModelExperiment, ModelTask -from rdagent.components.proposal import ( - Hypothesis, - ModelHypothesis2Experiment, - ModelHypothesisGen, -) -from rdagent.core.proposal import Hypothesis, Scenario, Trace -from rdagent.scenarios.data_mining.experiment.model_experiment import DMModelExperiment -from rdagent.utils.agent.tpl import T - -DMModelHypothesis = Hypothesis - - -class DMModelHypothesisGen(ModelHypothesisGen): - """ - # NOTE: we can share this class across different data mining scenarios - # It may better to move the class into components folder like `rdagent/components/proposal/model_proposal.py` - # Here is the use case: - - .. code-block:: python - - class XXXDMModelHypothesisGen(DMModelHypothesisGen): - prompts: Prompts = a_specifc_prompt_dict - """ - - def __init__(self, scen: Scenario) -> Tuple[dict, bool]: - super().__init__(scen) - - def prepare_context(self, trace: Trace) -> Tuple[dict, bool]: - hypothesis_and_feedback = ( - T("scenarios.qlib.prompts:hypothesis_and_feedback").r( - trace=trace, - ) - if len(trace.hist) > 0 - else "No previous hypothesis and feedback available since it's the first round." - ) - - last_hypothesis_and_feedback = ( - T("scenarios.qlib.prompts:last_hypothesis_and_feedback").r( - experiment=trace.hist[-1][0], feedback=trace.hist[-1][1] - ) - if len(trace.hist) > 0 - else "No previous hypothesis and feedback available since it's the first round." - ) - - context_dict = { - "hypothesis_and_feedback": hypothesis_and_feedback, - "last_hypothesis_and_feedback": last_hypothesis_and_feedback, - "RAG": None, - "hypothesis_output_format": T("scenarios.qlib.prompts:hypothesis_output_format").r(), - "hypothesis_specification": T("scenarios.qlib.prompts:model_hypothesis_specification").r(), - } - return context_dict, True - - def convert_response(self, response: str) -> Hypothesis: - response_dict = json.loads(response) - hypothesis = DMModelHypothesis( - hypothesis=response_dict["hypothesis"], - reason=response_dict["reason"], - concise_reason=response_dict["concise_reason"], - concise_observation=response_dict["concise_observation"], - concise_justification=response_dict["concise_justification"], - concise_knowledge=response_dict["concise_knowledge"], - ) - return hypothesis - - -class DMModelHypothesis2Experiment(ModelHypothesis2Experiment): - def prepare_context(self, hypothesis: Hypothesis, trace: Trace) -> Tuple[dict, bool]: - scenario = trace.scen.get_scenario_all_desc() - experiment_output_format = T("scenarios.qlib.prompts:model_experiment_output_format").r() - - hypothesis_and_feedback = ( - T("scenarios.qlib.prompts:hypothesis_and_feedback").r( - trace=trace, - ) - if len(trace.hist) > 0 - else "No previous hypothesis and feedback available since it's the first round." - ) - - experiment_list: List[ModelExperiment] = [t[0] for t in trace.hist] - - model_list = [] - for experiment in experiment_list: - model_list.extend(experiment.sub_tasks) - - return { - "target_hypothesis": str(hypothesis), - "scenario": scenario, - "hypothesis_and_feedback": hypothesis_and_feedback, - "experiment_output_format": experiment_output_format, - "target_list": model_list, - "RAG": None, - }, True - - def convert_response(self, response: str, hypothesis: Hypothesis, trace: Trace) -> ModelExperiment: - response_dict = json.loads(response) - tasks = [] - for model_name in response_dict: - description = response_dict[model_name]["description"] - formulation = response_dict[model_name]["formulation"] - architecture = response_dict[model_name]["architecture"] - variables = response_dict[model_name]["variables"] - hyperparameters = response_dict[model_name]["hyperparameters"] - model_type = response_dict[model_name]["model_type"] - tasks.append( - ModelTask( - name=model_name, - description=description, - formulation=formulation, - architecture=architecture, - variables=variables, - hyperparameters=hyperparameters, - model_type=model_type, - ) - ) - exp = DMModelExperiment(tasks, hypothesis=hypothesis) - exp.based_experiments = [t[0] for t in trace.hist if t[1]] - return exp diff --git a/rdagent/scenarios/data_science/debug/data.py b/rdagent/scenarios/data_science/debug/data.py index cefb4979..77791a96 100644 --- a/rdagent/scenarios/data_science/debug/data.py +++ b/rdagent/scenarios/data_science/debug/data.py @@ -1,5 +1,3 @@ -import os -import platform import shutil from collections import Counter, defaultdict from pathlib import Path @@ -13,8 +11,6 @@ try: except: pass -from rdagent.app.kaggle.conf import KAGGLE_IMPLEMENT_SETTING - class DataHandler: """Base DataHandler interface.""" @@ -178,10 +174,10 @@ def copy_file(src_fp, target_folder, data_folder): def create_debug_data( competition: str, + dataset_path: str | Path, dr_cls: type[DataReducer] = UniqueIDDataReducer, min_frac=0.01, min_num=5, - dataset_path=None, sample_path=None, ): """ @@ -189,14 +185,12 @@ def create_debug_data( and renames/moves files for easier debugging. Automatically detects file type (csv, pkl, parquet, hdf, etc.). """ - if dataset_path is None: - dataset_path = KAGGLE_IMPLEMENT_SETTING.local_data_path # FIXME: don't hardcode this KAGGLE_IMPLEMENT_SETTING - + dataset_path = Path(dataset_path) if sample_path is None: - sample_path = Path(dataset_path) / "sample" + sample_path = dataset_path / "sample" - data_folder = Path(dataset_path) / competition - sample_folder = Path(sample_path) / competition + data_folder = dataset_path / competition + sample_folder = sample_path / competition # Traverse the folder and exclude specific file types included_extensions = {".csv", ".pkl", ".parquet", ".h5", ".hdf", ".hdf5", ".jsonl", ".bson"} diff --git a/rdagent/scenarios/kaggle/kaggle_crawler.py b/rdagent/scenarios/kaggle/kaggle_crawler.py index 3e6e38be..2fc6770d 100644 --- a/rdagent/scenarios/kaggle/kaggle_crawler.py +++ b/rdagent/scenarios/kaggle/kaggle_crawler.py @@ -14,7 +14,6 @@ from selenium import webdriver from selenium.webdriver.chrome.service import Service from selenium.webdriver.common.by import By -from rdagent.app.kaggle.conf import KAGGLE_IMPLEMENT_SETTING from rdagent.core.conf import ExtendedBaseSettings from rdagent.core.exception import KaggleError from rdagent.core.utils import cache_with_pickle @@ -107,9 +106,7 @@ def crawl_descriptions( return descriptions -def download_data( - competition: str, settings: ExtendedBaseSettings = KAGGLE_IMPLEMENT_SETTING, enable_create_debug_data: bool = True -) -> None: +def download_data(competition: str, settings: ExtendedBaseSettings, enable_create_debug_data: bool = True) -> None: local_path = settings.local_data_path if settings.if_using_mle_data: zipfile_path = f"{local_path}/zip_files" @@ -227,9 +224,7 @@ def score_rank(competition: str, score: float) -> tuple[int, float]: return rank, rank_percent -def download_notebooks( - competition: str, local_path: str = f"{KAGGLE_IMPLEMENT_SETTING.local_data_path}/notebooks", num: int = 15 -) -> None: +def download_notebooks(competition: str, local_path: str, num: int = 15) -> None: data_path = Path(f"{local_path}/{competition}") from kaggle.api.kaggle_api_extended import KaggleApi @@ -264,9 +259,7 @@ def notebook_to_knowledge(notebook_text: str) -> str: return response -def convert_notebooks_to_text( - competition: str, local_path: str = f"{KAGGLE_IMPLEMENT_SETTING.local_data_path}/notebooks" -) -> None: +def convert_notebooks_to_text(competition: str, local_path: str) -> None: data_path = Path(f"{local_path}/{competition}") converted_num = 0 @@ -302,7 +295,7 @@ def convert_notebooks_to_text( print(f"Converted {converted_num} notebooks to text files.") -def collect_knowledge_texts(local_path: str = KAGGLE_IMPLEMENT_SETTING.local_data_path) -> dict[str, list[str]]: +def collect_knowledge_texts(notebooks_path: str | Path) -> dict[str, list[str]]: """ { "competition1": [ @@ -318,7 +311,7 @@ def collect_knowledge_texts(local_path: str = KAGGLE_IMPLEMENT_SETTING.local_dat ... } """ - notebooks_dir = Path(local_path) / "notebooks" + notebooks_dir = Path(notebooks_path) competition_knowledge_texts_dict = {} for competition_dir in notebooks_dir.iterdir(): diff --git a/rdagent/scenarios/qlib/factor_experiment_loader/json_loader.py b/rdagent/scenarios/qlib/factor_experiment_loader/json_loader.py index b6c97d7f..a6558e2c 100644 --- a/rdagent/scenarios/qlib/factor_experiment_loader/json_loader.py +++ b/rdagent/scenarios/qlib/factor_experiment_loader/json_loader.py @@ -13,7 +13,7 @@ from rdagent.scenarios.qlib.experiment.factor_experiment import QlibFactorExperi class FactorExperimentLoaderFromDict(FactorExperimentLoader): - def load(self, factor_dict: dict) -> list: + def load(self, factor_dict: dict) -> QlibFactorExperiment: """Load data from a dict.""" task_l = [] for factor_name, factor_data in factor_dict.items(): diff --git a/rdagent/scenarios/qlib/factor_experiment_loader/pdf_loader.py b/rdagent/scenarios/qlib/factor_experiment_loader/pdf_loader.py index 822bf749..fa2f05f6 100644 --- a/rdagent/scenarios/qlib/factor_experiment_loader/pdf_loader.py +++ b/rdagent/scenarios/qlib/factor_experiment_loader/pdf_loader.py @@ -19,6 +19,7 @@ from rdagent.core.utils import multiprocessing_wrapper from rdagent.log import rdagent_logger as logger from rdagent.oai.llm_conf import LLM_SETTINGS from rdagent.oai.llm_utils import APIBackend +from rdagent.scenarios.qlib.experiment.factor_experiment import QlibFactorExperiment from rdagent.scenarios.qlib.factor_experiment_loader.json_loader import ( FactorExperimentLoaderFromDict, ) @@ -566,7 +567,7 @@ def deduplicate_factors_by_llm( # noqa: C901, PLR0912 class FactorExperimentLoaderFromPDFfiles(FactorExperimentLoader): - def load(self, file_or_folder_path: str) -> dict: + def load(self, file_or_folder_path: str) -> QlibFactorExperiment: with logger.tag("docs"): docs_dict = load_and_process_pdfs_by_langchain(file_or_folder_path) logger.log_object(docs_dict) diff --git a/rdagent/utils/env.py b/rdagent/utils/env.py index bb4ff7df..7c668442 100644 --- a/rdagent/utils/env.py +++ b/rdagent/utils/env.py @@ -470,7 +470,7 @@ class LocalEnv(Env[ASpecificLocalConf]): print(Rule("[bold green]LocalEnv Logs Begin[/bold green]", style="dark_orange")) table = Table(title="Run Info", show_header=False) table.add_column("Key", style="bold cyan") - table.add_column("Value", style="bold magenta", no_wrap=True) + table.add_column("Value", style="bold magenta") table.add_row("Entry", entry) table.add_row("Local Path", local_path or "") table.add_row("Env", "\n".join(f"{k}:{v}" for k, v in env.items())) @@ -644,25 +644,6 @@ class QlibDockerConf(DockerConf): enable_cache: bool = False -class DMDockerConf(DockerConf): - model_config = SettingsConfigDict(env_prefix="DM_DOCKER_") - - build_from_dockerfile: bool = True - dockerfile_folder_path: Path = Path(__file__).parent.parent / "scenarios" / "data_mining" / "docker" - image: str = "local_dm:latest" - mount_path: str = "/workspace/dm_workspace/" - default_entry: str = "python train.py" - extra_volumes: dict = { - str( - Path("~/.rdagent/.data/physionet.org/files/mimic-eicu-fiddle-feature/1.0.0/FIDDLE_mimic3/") - .expanduser() - .resolve() - .absolute() - ): "/root/.data/" - } - shm_size: str | None = "16g" - - class KGDockerConf(DockerConf): model_config = SettingsConfigDict(env_prefix="KG_DOCKER_") @@ -880,7 +861,7 @@ class DockerEnv(Env[DockerConf]): table.add_row("Container Name", container.name) table.add_row("Entry", entry) table.add_row("Env", "\n".join(f"{k}:{v}" for k, v in env.items())) - table.add_row("Volumes", "\n".join(f"{k}:{v}" for k, v in volumes.items())) + table.add_row("Volumes", "\n".join(f"{k}:\n {v}" for k, v in volumes.items())) print(table) for log in logs: decoded_log = log.strip().decode() @@ -920,28 +901,6 @@ class QTDockerEnv(DockerEnv): logger.info("Data already exists. Download skipped.") -class DMDockerEnv(DockerEnv): - """Qlib Torch Docker""" - - def __init__(self, conf: DockerConf = DMDockerConf()): - super().__init__(conf) - - def prepare(self, username: str, password: str) -> None: - """ - Download image & data if it doesn't exist - """ - super().prepare() - data_path = next(iter(self.conf.extra_volumes.keys())) - if not (Path(data_path)).exists(): - logger.info("We are downloading!") - cmd = "wget -r -N -c -np --user={} --password={} -P ~/.rdagent/.data/ https://physionet.org/files/mimic-eicu-fiddle-feature/1.0.0/".format( - username, password - ) - os.system(cmd) - else: - logger.info("Data already exists. Download skipped.") - - class KGDockerEnv(DockerEnv): """Kaggle Competition Docker""" diff --git a/rdagent/utils/workflow/loop.py b/rdagent/utils/workflow/loop.py index 92271be0..36b32345 100644 --- a/rdagent/utils/workflow/loop.py +++ b/rdagent/utils/workflow/loop.py @@ -309,7 +309,10 @@ class LoopBase: if all_duration is not None and not self.timer.started: self.timer.reset(all_duration=all_duration) - self.step_n, self.loop_n = step_n, loop_n + if step_n is not None: + self.step_n = step_n + if loop_n is not None: + self.loop_n = loop_n # empty the queue when restarting while not self.queue.empty(): diff --git a/requirements.txt b/requirements.txt index 71f3d68c..a4cdec44 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,7 +8,7 @@ loguru fire fuzzywuzzy openai -litellm +litellm==1.72.4 azure.identity pyarrow rich diff --git a/test/utils/test_kaggle.py b/test/utils/test_kaggle.py index 70cd6749..fae43b2c 100644 --- a/test/utils/test_kaggle.py +++ b/test/utils/test_kaggle.py @@ -1,15 +1,11 @@ import unittest from pathlib import Path -import nbformat from rich import print from rdagent.app.kaggle.conf import KAGGLE_IMPLEMENT_SETTING -from rdagent.oai.llm_utils import APIBackend from rdagent.scenarios.kaggle.experiment.workspace import KGFBWorkspace from rdagent.scenarios.kaggle.kaggle_crawler import download_data -from rdagent.utils.agent.ret import PythonAgentOut -from rdagent.utils.agent.tpl import T class TestTpl(unittest.TestCase): @@ -19,7 +15,7 @@ class TestTpl(unittest.TestCase): """ competition = KAGGLE_IMPLEMENT_SETTING.competition print(f"[bold orange]{competition}[/bold orange]") - download_data(competition) + download_data(competition, settings=KAGGLE_IMPLEMENT_SETTING) ws = KGFBWorkspace( template_folder_path=Path(__file__).parent.parent.parent / KAGGLE_IMPLEMENT_SETTING.template_path