chore: release 0.6.1

Release-As: 0.6.1

.bumpversion.cfg

@@ -0,0 +1,6 @@
+[bumpversion]
+current_version = 0.0.0
+commit = True
+tag = True
+
+[bumpversion:file:pyproject.toml]

.commitlintrc.js

@@ -0,0 +1,21 @@
+module.exports = {
+    extends: ["@commitlint/config-conventional"],
+    rules: {
+        // Configuration Format: [level, applicability, value]
+        // level: Error level, usually expressed as a number:
+        //     0 - disable rule
+        //     1 - Warning (does not prevent commits)
+        //     2 - Error (will block the commit)
+        // applicability: the conditions under which the rule applies, commonly used values:
+        //     “always” - always apply the rule
+        //     “never” - never apply the rule
+        // value: the specific value of the rule, e.g. a maximum length of 100.
+        // Refs: https://commitlint.js.org/reference/rules-configuration.html
+      "header-max-length": [2, "always", 100],
+      "type-enum": [
+        2,
+        "always",
+        ["build", "chore", "ci", "docs", "feat", "fix", "perf", "refactor", "revert", "style", "test", "Release-As"]
+      ]
+    }
+  };

.env.example

@@ -1,27 +1,59 @@
+"""
+This file is a template for the .env file.
+
+Please copy this file to .env and fill in the values.
+
+For more information about configuration options, please refer to the documentation
+
+"""
+
+# ==========================================
 # Global configs:
-USE_AZURE=True
 MAX_RETRY=10
 RETRY_WAIT_SECONDS=20
-DUMP_CHAT_CACHE=True
-USE_CHAT_CACHE=True
-DUMP_EMBEDDING_CACHE=True
-USE_EMBEDDING_CACHE=True
-LOG_LLM_CHAT_CONTENT=False
-CHAT_FREQUENCY_PENALTY=0.0
-CHAT_PRESENCE_PENALTY=0.0
-
-# embedding model configs:
-EMBEDDING_OPENAI_API_KEY=your_api_key
-EMBEDDING_AZURE_API_BASE=your_api_base
-EMBEDDING_AZURE_API_VERSION=your_api_version
-EMBEDDING_MODEL=text-embedding-3-small
+# ==========================================
 
 
-# chat model configs:
-CHAT_OPENAI_API_KEY=your_api_key # 5c
-CHAT_AZURE_API_BASE=your_api_base
-CHAT_AZURE_API_VERSION=your_api_version
-CHAT_MODEL=your_model_version
-CHAT_MAX_TOKENS=3000
-CHAT_TEMPERATURE=0.7
-CHAT_STREAM=True
+# ==========================================
+# Backend Configuration
+# ==========================================
+# BACKEND=rdagent.oai.backend.LiteLLMAPIBackend
+# ==========================================
+
+# ==========================================
+# Backend Configuration (choose one)
+# ==========================================
+
+# 1. Set universal API key
+# CHAT_MODEL="gpt-4o"
+# EMBEDDING_MODEL="text-embedding-3-small"
+# OPENAI_API_BASE="https://your-endpoint.com/v1"
+# OPENAI_API_KEY="sk-your-api-key-here"
+
+# 2. Set separate API KEY
+# Chat configuration
+OPENAI_API_KEY="sk-chat-key"
+OPENAI_API_BASE="https://xxx-litellm.com/v1"
+CHAT_MODEL='gpt-4o'
+
+# Embedding configuration (using other service)
+# Use siliconflow as example, pay attention to the litellm_proxy prefix
+LITELLM_PROXY_API_KEY="sk-embedding-service-key"
+LITELLM_PROXY_API_BASE="https://api.siliconflow.cn/v1"
+EMBEDDING_MODEL="litellm_proxy/BAAI/bge-large-en-v1.5"
+# ==========================================
+
+# ==========================================
+# Other Configuration 
+# ==========================================
+# CHAT_AZURE_API_BASE=<for_Azure_user>
+# CHAT_AZURE_API_VERSION=<for_Azure_user>
+
+# EMBEDDING_AZURE_API_BASE=<for_Azure_user>
+# EMBEDDING_AZURE_API_VERSION=<for_Azure_user>
+
+# Cache Setting (Optional):
+# USE_CHAT_CACHE=True
+# USE_EMBEDDING_CACHE=True
+# Senario Configs:
+# ==========================================

.github/ISSUE_TEMPLATE/bug-report.md

@@ -0,0 +1,51 @@
+---
+name: "\U0001F41B Bug Report"
+about: Submit a bug report to help us improve RD-Agent
+labels: bug
+
+---
+
+## 🐛 Bug Description
+
+<!-- A clear and concise description of what the bug is. -->
+
+## To Reproduce
+
+Steps to reproduce the behavior:
+
+1.
+2.
+3.
+
+
+## Expected Behavior
+
+<!-- A clear and concise description of what you expected to happen. -->
+
+## Screenshot
+
+<!-- A screenshot of the error message or anything shouldn't appear-->
+
+## Environment
+
+**Note**: Users can run `rdagent collect_info` to get system information and paste it directly here.
+
+ - Name of current operating system:
+ - Processor architecture:
+ - System, version, and hardware information:
+ - Version number of the system:
+ - Python version:
+ - Container ID:
+ - Container Name:
+ - Container Status:
+ - Image ID used by the container:
+ - Image tag used by the container:
+ - Container port mapping:
+ - Container Label:
+ - Startup Commands:
+ - RD-Agent version:
+ - Package version:
+
+## Additional Notes
+
+<!-- Add any other information about the problem here. -->

.github/ISSUE_TEMPLATE/documentation.md

@@ -0,0 +1,9 @@
+---
+name: "\U0001F4D6 Documentation"
+about: Report an issue related to documentation
+
+---
+
+## 📖 Documentation
+
+<!-- Please specify whether it's tutorial part or API reference part, and describe it.-->

.github/ISSUE_TEMPLATE/feature-request.md

@@ -0,0 +1,25 @@
+---
+name: "\U0001F31FFeature Request"
+about: Request for a new RD-Agent feature
+labels: enhancement
+
+---
+
+## 🌟 Feature Description
+<!-- A clear and concise description of the feature proposal -->
+
+## Motivation
+
+1. Application scenario
+2. Related works (Papers, Github repos etc.):
+3. Any other relevant and important information:
+
+<!-- Please describe why the feature is important. -->
+
+## Alternatives
+
+<!-- A short description of any alternative solutions or features you've considered. -->
+
+## Additional Notes
+
+<!-- Add any other context or screenshots about the feature request here. -->

.github/ISSUE_TEMPLATE/question.md

@@ -0,0 +1,10 @@
+---
+name: "❓Questions & Help"
+about: Have some questions? We can offer help.
+labels: question
+
+---
+
+## ❓ Questions and Help
+
+We sincerely suggest you to carefully read the [documentation](http://rdagent.readthedocs.io/). After that, if you still feel puzzled, please describe the question clearly under this issue.

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,34 @@
+<!--- Thank you for submitting a Pull Request! In order to make our work smoother. -->
+<!--- please make sure your Pull Request meets the following requirements: -->
+<!---   1. Provide a general summary of your changes in the Title above; -->
+<!---   2. Add appropriate prefixes to titles, such as `build:`, `chore:`, `ci:`, `docs:`, `feat:`, `fix:`, `perf:`, `refactor:`, `revert:`, `style:`, `test:`(Ref: https://www.conventionalcommits.org/). -->
+<!--- Category: -->
+<!--- Patch Updates: `fix:` -->
+<!---   Example: fix(auth): correct login validation issue -->
+<!--- minor update (introduces new functionality): `feat` -->
+<!---   Example: feature(parser): add ability to parse arrays -->
+<!--- major update(destructive update): Include BREAKING CHANGE in the commit message footer, or add `! ` in the commit footer to indicate that there is a destructive update. -->
+<!---   Example: feat(auth)! : remove support for old authentication method -->
+<!--- Other updates: `build:`, `chore:`, `ci:`, `docs:`, `perf:`, `refactor:`, `revert:`, `style:`, `test:`. -->
+
+## Description
+<!--- Describe your changes in detail -->
+
+## Motivation and Context
+<!--- Are there any related issues? If so, please put the link here. -->
+<!--- Why is this change required? What problem does it solve? -->
+
+## How Has This Been Tested?
+<!---  Put an `x` in all the boxes that apply: --->
+- [ ] If you are adding a new feature, test on your own test scripts.
+
+<!--- **ATTENTION**: If you are adding a new feature, please make sure your codes are **correctly tested**. If our test scripts do not cover your cases, please provide your own test scripts under the `tests` folder and test them. More information about test scripts can be found [here](https://docs.python.org/3/library/unittest.html#basic-example), or you could refer to those we provide under the `tests` folder. -->
+
+## Screenshots of Test Results (if appropriate):
+1. Your own tests:
+
+## Types of changes
+<!--- What types of changes does your code introduce? Put an `x` in all the boxes that apply: -->
+- [ ] Fix bugs
+- [ ] Add new feature
+- [ ] Update documentation

.github/workflows/ci.yml

@@ -19,22 +19,11 @@ jobs:
           python-version: ${{ matrix.python-version }}
       - run: env | sort
       - run: make dev
-      - env:
-          CHAT_AZURE_API_BASE: ${{ secrets.CHAT_AZURE_API_BASE }}
-          CHAT_AZURE_API_VERSION: ${{ secrets.CHAT_AZURE_API_VERSION }}
-          CHAT_MAX_TOKENS: ${{ secrets.CHAT_MAX_TOKENS }}
-          CHAT_MODEL: ${{ secrets.CHAT_MODEL }}
-          CHAT_TEMPERATURE: ${{ secrets.CHAT_TEMPERATURE }}
-          EMBEDDING_AZURE_API_BASE: ${{ secrets.CHAT_AZURE_API_BASE }}
-          EMBEDDING_AZURE_API_VERSION: ${{ secrets.CHAT_AZURE_API_VERSION }}
-          EMBEDDING_MODEL: ${{ secrets.EMBEDDING_MODEL }}
-        name: lint test docs and build
-        run: make lint test docs build
+      - name: lint test docs and build
+        run: make lint docs-gen test-offline # test docs build
     strategy:
       matrix:
         python-version:
-          - '3.8'
-          - '3.9'
           - '3.10'
           - '3.11'
   dependabot:
@@ -56,16 +45,12 @@ jobs:
         with:
           cache: pip
           python-version: |
-            3.8
-            3.9
             3.10
             3.11
       - name: Install pipenv using pipx
         run: pipx install pipenv
       - name: Generate constraints for all supported Python versions
         run: |
-          CI= PYTHON_VERSION=3.8 make constraints
-          CI= PYTHON_VERSION=3.9 make constraints
           CI= PYTHON_VERSION=3.10 make constraints
           CI= PYTHON_VERSION=3.11 make constraints
       - name: Push changes if applicable

.github/workflows/pr.yml

@@ -0,0 +1,35 @@
+name: Lint pull request title
+
+on:
+  pull_request:
+    types:
+      - opened
+      - synchronize
+      - reopened
+      - edited
+
+concurrency:
+  cancel-in-progress: true
+  group: ${{ github.workflow }}-${{ github.ref }}
+
+jobs:
+  lint-title:
+    runs-on: ubuntu-latest
+    steps:
+      # This step is necessary because the lint title uses the .commitlintrc.js file in the project root directory.
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: '16'
+
+      - name: Install commitlint
+        run: npm install --save-dev @commitlint/{config-conventional,cli}
+
+      - name: Validate PR Title with commitlint
+        env:
+          BODY: ${{ github.event.pull_request.title }}
+        run: |
+          echo "$BODY" | npx commitlint --config .commitlintrc.js

.github/workflows/release.yml

@@ -1,91 +1,50 @@
-jobs:
-  package:
-    needs: release
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          cache: pip
-          python-version: '3.8'
-      - run: env | sort
-      - run: make dev-package
-      - run: make build
-      - env:
-          TWINE_NON_INTERACTIVE: true
-          TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }}
-          TWINE_USERNAME: ${{ vars.TWINE_USERNAME != '' && vars.TWINE_USERNAME || '__token__' }}
-        run: make upload
-  pages-build:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          cache: pip
-          python-version: '3.8'
-      - run: env | sort
-      - run: make dev-docs
-      - run: make docs
-      - name: Upload changelog
-        uses: actions/upload-artifact@v4
-        with:
-          name: changelog
-          path: docs/changelog.md
-      - name: Upload pages artifact
-        uses: actions/upload-pages-artifact@v3
-        with:
-          path: public
-  pages-deploy:
-    needs: release
-    permissions:
-      id-token: write
-      pages: write
-    runs-on: ubuntu-latest
-    steps:
-      - id: deployment
-        name: Deploy to GitHub Pages
-        uses: actions/deploy-pages@v4
-  release:
-    needs: pages-build
-    permissions:
-      contents: write
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - name: Install git-changelog using pipx
-        run: pipx install git-changelog
-      - name: Remove changelog to avoid file already exists error
-        run: rm -v docs/changelog.md
-      - name: Download changelog
-        uses: actions/download-artifact@v4
-        with:
-          name: changelog
-          path: docs/
-      - name: Prepare release notes
-        run: make release-notes > release-notes.md
-      - id: prerelease
-        name: Determine prerelease
-        run: |
-          if [[ "${{ github.ref }}" =~ (a|b|rc)(0|[1-9][0-9]*)?$ ]]; then
-            echo "is_prerelease=true" > $GITHUB_OUTPUT
-          else
-            echo "is_prerelease=false" > $GITHUB_OUTPUT
-          fi
-      - name: Create GitHub release
-        uses: softprops/action-gh-release@v2
-        with:
-          body_path: release-notes.md
-          prerelease: ${{ steps.prerelease.outputs.is_prerelease }}
 name: Release
 on:
   push:
-    tags:
-      - v?[0-9]+.[0-9]+.[0-9]+
-      - v?[0-9]+.[0-9]+.[0-9]+-?a[0-9]*
-      - v?[0-9]+.[0-9]+.[0-9]+-?b[0-9]*
-      - v?[0-9]+.[0-9]+.[0-9]+-?rc[0-9]*
+    branches:
+      - main
+permissions:
+  contents: read
+jobs:
+  release_and_publish:
+    permissions:
+      contents: write
+      pull-requests: read
+    runs-on: ubuntu-latest
+    steps:
+      - name: Release please
+        id: release_please
+        uses: googleapis/release-please-action@v4
+        with:
+          # The current PAT (personal access token) was created on 2024-08-05,
+          # since the maximum validity of PAT is 1 year, you need to change the PAT before 2025-08-05.
+          token: ${{ secrets.PAT }}
+          release-type: simple
+      - uses: actions/checkout@v4
+        if: ${{ steps.release_please.outputs.release_created }}
+        with:
+          fetch-depth: 0
+      - name: Set up Python
+        if: ${{ steps.release_please.outputs.release_created }}
+        uses: actions/setup-python@v5
+        with:
+          cache: pip
+          python-version: '3.10'
+      - name: Install dependencies
+        if: ${{ steps.release_please.outputs.release_created }}
+        run: |
+          python -m pip install --upgrade pip
+          pip install setuptools wheel twine  #  better-exceptions(optional for debug)
+      - run: env | sort
+        if: ${{ steps.release_please.outputs.release_created }}
+      - run: make dev
+        if: ${{ steps.release_please.outputs.release_created }}
+      - run: make build
+        if: ${{ steps.release_please.outputs.release_created }}
+      - name: upload
+        if: ${{ steps.release_please.outputs.release_created }}
+        env:
+          TWINE_USERNAME: __token__
+          TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
+        run: |
+          make upload

.gitignore

@@ -4,6 +4,7 @@
 Pipfile
 public
 release-notes.md
+typescript*
 
 # Byte-compiled / optimized / DLL files
 __pycache__/
@@ -64,6 +65,7 @@ coverage.xml
 
 # Django stuff:
 *.log
+/log*/
 local_settings.py
 db.sqlite3
 db.sqlite3-journal
@@ -109,9 +111,10 @@ celerybeat.pid
 *.sage.py
 
 # Environments
-.env
+.env*
+*.env
 .venv
-env/
+^env/
 venv/
 ENV/
 env.bak/
@@ -138,6 +141,9 @@ dmypy.json
 # all pkl files
 *.pkl
 
+# all h5 files
+*.h5
+
 # all vs-code files
 .vscode/
 
@@ -153,3 +159,22 @@ git_ignore_folder/
 
 # DB files
 *.db
+
+# Docker
+factor_template/mlruns/
+env_tpl
+mlruns/
+
+# possible output from coder or runner
+*.pth
+*qlib_res.csv
+
+# shell script
+*.out
+/*.sh
+.aider*
+rdagent/app/benchmark/factor/example.json
+
+# UI Server resources
+videos/
+static/

.readthedocs.yaml

@@ -0,0 +1,38 @@
+# .readthedocs.yml
+# Read the Docs configuration file
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+# Required
+version: 2
+
+# Set the version of Python and other tools you might need
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.10"
+  # During the build process, you need to fetch tags, and since the default command to read the docs only pulls shallow code, it will cause an error.
+  # So we added the `git fetch --tags --unshallow || true` command to fetch the full tag record.
+  # Adding this command overrides the default command, so we copied it over to make sure the build was successful.
+  commands:
+    - python -mvirtualenv $READTHEDOCS_VIRTUALENV_PATH 
+    - python -m pip install --upgrade --no-cache-dir pip setuptools 
+    - python -m pip install --upgrade --no-cache-dir sphinx 
+    - python -m pip install --exists-action=w --no-cache-dir -r requirements/docs.txt 
+    - python -m pip install --upgrade --upgrade-strategy only-if-needed --no-cache-dir . 
+    - git fetch --tags --unshallow || true
+    - mkdir -p $READTHEDOCS_OUTPUT/html/
+    - python -m sphinx -T -b html -d _build/doctrees -D language=en ./docs $READTHEDOCS_OUTPUT/html
+
+# Build documentation in the docs/ directory with Sphinx
+sphinx:
+  configuration: docs/conf.py
+
+# Build all formats
+formats: all
+
+# Optionally set the version of Python and requirements required to build your docs
+python:
+  install:
+    - requirements: requirements/docs.txt
+    - method: pip
+      path: .

.streamlit/config.toml

@@ -0,0 +1,2 @@
+[client]
+showSidebarNavigation = false

CHANGELOG.md

@@ -0,0 +1,446 @@
+# Changelog
+
+## [0.6.1](https://github.com/microsoft/RD-Agent/compare/v0.6.0...v0.6.1) (2025-06-28)
+
+
+### Bug Fixes
+
+* fix mount ([#1001](https://github.com/microsoft/RD-Agent/issues/1001)) ([4ae2f13](https://github.com/microsoft/RD-Agent/commit/4ae2f1303dfcbaea53d459be7c8e85bf85ce5f4f))
+* handle the bug of wrong dag_parant index ([#996](https://github.com/microsoft/RD-Agent/issues/996)) ([bda12ff](https://github.com/microsoft/RD-Agent/commit/bda12ffecf9ae116e0d04eece0c6a1b61413d916))
+* improve log folder sorting and selection UX ([#993](https://github.com/microsoft/RD-Agent/issues/993)) ([b116807](https://github.com/microsoft/RD-Agent/commit/b11680777f116b6c40f9e535e0da10c186c95050))
+
+## [0.6.0](https://github.com/microsoft/RD-Agent/compare/v0.5.0...v0.6.0) (2025-06-26)
+
+
+### Features
+
+* async mechanism for multi-trace ([#981](https://github.com/microsoft/RD-Agent/issues/981)) ([9e60c32](https://github.com/microsoft/RD-Agent/commit/9e60c32cf348481eb55617809c059c359d7603b8))
+
+
+### Bug Fixes
+
+* add async to direct_exp_gen avoid infinite loop ([#992](https://github.com/microsoft/RD-Agent/issues/992)) ([78c203d](https://github.com/microsoft/RD-Agent/commit/78c203d8eefbba67fc120b35cb25e85b2200ac49))
+* docker container cleanup to prevent accumulation and system slowdown ([#975](https://github.com/microsoft/RD-Agent/issues/975)) ([05cf094](https://github.com/microsoft/RD-Agent/commit/05cf094913e48c903c8a4476d6c609d8bfa10681))
+* fix a bug and update the docs ([#978](https://github.com/microsoft/RD-Agent/issues/978)) ([d1ae9e1](https://github.com/microsoft/RD-Agent/commit/d1ae9e1dcc2ccd1ffe05cb1c6db3e905fa70425c))
+* merge datascience v3 and v2 ([#974](https://github.com/microsoft/RD-Agent/issues/974)) ([1ba7548](https://github.com/microsoft/RD-Agent/commit/1ba754853ce2010ce1cb0bbd217b67689fa1ebdf))
+* refine details ([#979](https://github.com/microsoft/RD-Agent/issues/979)) ([25caa3d](https://github.com/microsoft/RD-Agent/commit/25caa3d00c255286dce27915b9355987b87ed2e8))
+* refine prompt ([#987](https://github.com/microsoft/RD-Agent/issues/987)) ([76df96e](https://github.com/microsoft/RD-Agent/commit/76df96ee88212a8aee7f518b9cacf80591dc2939))
+
+## [0.5.0](https://github.com/microsoft/RD-Agent/compare/v0.4.0...v0.5.0) (2025-06-18)
+
+
+### Features
+
+* add a check for whether values in score_df are NaN ([#756](https://github.com/microsoft/RD-Agent/issues/756)) ([d9cc780](https://github.com/microsoft/RD-Agent/commit/d9cc78098beb27f3a1bf2f2d461302db177b7d41))
+* add competition level filter and extract constants to utils ([#869](https://github.com/microsoft/RD-Agent/issues/869)) ([b40b605](https://github.com/microsoft/RD-Agent/commit/b40b6055368e6c72d8435352104b1c281b06da7f))
+* add DocDev for auto-generating workspace documentation ([#781](https://github.com/microsoft/RD-Agent/issues/781)) ([bcba6ea](https://github.com/microsoft/RD-Agent/commit/bcba6eac32684ebb267c93b4e85dbfa9561d15d1))
+* add drafting pipeline ([#832](https://github.com/microsoft/RD-Agent/issues/832)) ([efedddf](https://github.com/microsoft/RD-Agent/commit/efedddf39bc19221fdffc2e39ee0a09097fc82b0))
+* add last_exp_fb to DSTrace and update feedback retrieval usage ([#910](https://github.com/microsoft/RD-Agent/issues/910)) ([10531fd](https://github.com/microsoft/RD-Agent/commit/10531fda9438c6915b26d5013bd2413e1333ceb9))
+* add mlflow logger in RD loop to log ([#815](https://github.com/microsoft/RD-Agent/issues/815)) ([b91b54f](https://github.com/microsoft/RD-Agent/commit/b91b54f355c26b751087d0c14774f466e82866de))
+* add naive experiment generator and update proposal configurations ([#759](https://github.com/microsoft/RD-Agent/issues/759)) ([75494f4](https://github.com/microsoft/RD-Agent/commit/75494f4fed5bc845acfd7f7bacef385f0f96c514))
+* add RD-Agent-Quant scenario ([#838](https://github.com/microsoft/RD-Agent/issues/838)) ([6e42d52](https://github.com/microsoft/RD-Agent/commit/6e42d523a85df67aa13927abbf0894564c71880e))
+* add reasoning_effort parameter to LiteLLMAPIBackend and LLMSett… ([#754](https://github.com/microsoft/RD-Agent/issues/754)) ([113889f](https://github.com/microsoft/RD-Agent/commit/113889fefe9b09aaea1b564704c81664b8f77ec5))
+* add reviewer in feedback ([#765](https://github.com/microsoft/RD-Agent/issues/765)) ([1a95bee](https://github.com/microsoft/RD-Agent/commit/1a95bee6aa6bc6f45fdeb484f3a6f81caa273038))
+* advanced checkpoint selectors ([#790](https://github.com/microsoft/RD-Agent/issues/790)) ([50ea033](https://github.com/microsoft/RD-Agent/commit/50ea0336e93d8cb39fb871e81a3f61abdf293bc7))
+* archive python and csv files in workspace to maintain results ([#814](https://github.com/microsoft/RD-Agent/issues/814)) ([67d0e01](https://github.com/microsoft/RD-Agent/commit/67d0e01e7c9237da1371d93cbf9d86f5f46faac4))
+* checkpoint selection ([#744](https://github.com/microsoft/RD-Agent/issues/744)) ([a15a06a](https://github.com/microsoft/RD-Agent/commit/a15a06ad643977db59d7cac9da52e637cf80395a))
+* custom data ([#810](https://github.com/microsoft/RD-Agent/issues/810)) ([6322916](https://github.com/microsoft/RD-Agent/commit/632291608cf605bd8bcfcab0017824823bdecdb8))
+* dump model ([#776](https://github.com/microsoft/RD-Agent/issues/776)) ([b49481e](https://github.com/microsoft/RD-Agent/commit/b49481e073e6f536d2b1b3bd2d01229ed05abdea))
+* enable to set different version of idea-proposal for multi traces ([#895](https://github.com/microsoft/RD-Agent/issues/895)) ([236c28f](https://github.com/microsoft/RD-Agent/commit/236c28f29c6bc5da62129632e464bbc32056ebdb))
+* enhance compatibility with more LLM models ([#905](https://github.com/microsoft/RD-Agent/issues/905)) ([8800624](https://github.com/microsoft/RD-Agent/commit/8800624ad4749d6e798785a082c9f94c306792ef))
+* idea pool integrated to exp_gen & add timer to RD-Agent & pause-resume to RD-loops ([#795](https://github.com/microsoft/RD-Agent/issues/795)) ([e62aefa](https://github.com/microsoft/RD-Agent/commit/e62aefa56e34ff45a8ed033f7bf28b95c8e63656))
+* joblib cache ([#749](https://github.com/microsoft/RD-Agent/issues/749)) ([83a0411](https://github.com/microsoft/RD-Agent/commit/83a041148ff908871b1906f9e6889d80ab513412))
+* log api status to mlflow ([#860](https://github.com/microsoft/RD-Agent/issues/860)) ([049921b](https://github.com/microsoft/RD-Agent/commit/049921beb0b4ed0ba1ab7508d9857d2c1e729349))
+* log reaching max time limit before breaking CoSTEER evolution ([#921](https://github.com/microsoft/RD-Agent/issues/921)) ([837fff2](https://github.com/microsoft/RD-Agent/commit/837fff29096fefe1369d386ef8a860395b737173))
+* merge failed and successful traces together ([#766](https://github.com/microsoft/RD-Agent/issues/766)) ([3a2aa8c](https://github.com/microsoft/RD-Agent/commit/3a2aa8cf0102647950b2dfc0007c118b0c799cd4))
+* merge selectively ([#888](https://github.com/microsoft/RD-Agent/issues/888)) ([06ba314](https://github.com/microsoft/RD-Agent/commit/06ba314ff0f91e7e78e8d456c719ac3194a8c774))
+* multi-trace online merge ([#886](https://github.com/microsoft/RD-Agent/issues/886)) ([2112d67](https://github.com/microsoft/RD-Agent/commit/2112d676d0938de6fea163b2e5eb9c36771e7041))
+* new proposal (structured outputs) prompts ([#887](https://github.com/microsoft/RD-Agent/issues/887)) ([150796a](https://github.com/microsoft/RD-Agent/commit/150796aaa72eaa5037fd7db8e785058fbc4d4967))
+* parallel loop running based on asyncio ([#932](https://github.com/microsoft/RD-Agent/issues/932)) ([c63e207](https://github.com/microsoft/RD-Agent/commit/c63e2071f3179feef69f88061c0172cb5c3157f2))
+* propose hypothesis across multiple parts in pipeline ([#827](https://github.com/microsoft/RD-Agent/issues/827)) ([acb0e21](https://github.com/microsoft/RD-Agent/commit/acb0e21a331410d044849e12e2887f41e5ff1c3a))
+* pull image with progress ([#777](https://github.com/microsoft/RD-Agent/issues/777)) ([5cad086](https://github.com/microsoft/RD-Agent/commit/5cad0860204ede974533dc7bdc9808cfd135fa24))
+* raise error when timeout in api call ([#793](https://github.com/microsoft/RD-Agent/issues/793)) ([eafd4df](https://github.com/microsoft/RD-Agent/commit/eafd4dfc6263f19a8cdaf27498a1d07b43815306))
+* raise policy violation ([#894](https://github.com/microsoft/RD-Agent/issues/894)) ([5b9d007](https://github.com/microsoft/RD-Agent/commit/5b9d0072aebe15369e9a0010af83e71684baeae7))
+* reanalyze competition info & pipeline coding evaluator prompt ([#837](https://github.com/microsoft/RD-Agent/issues/837)) ([f7b5258](https://github.com/microsoft/RD-Agent/commit/f7b52580080c75d311355bcc6193b49495801809))
+* refine merge ([#842](https://github.com/microsoft/RD-Agent/issues/842)) ([99463b4](https://github.com/microsoft/RD-Agent/commit/99463b46819b3a0dcb2bb12a823a9cdf7ec560b4))
+* refine prompt ([#760](https://github.com/microsoft/RD-Agent/issues/760)) ([a91b182](https://github.com/microsoft/RD-Agent/commit/a91b182c4c9510eb34e4aab956588e909fa5d70b))
+* replace hard-coded cache paths with dynamic cache_path config ([#952](https://github.com/microsoft/RD-Agent/issues/952)) ([db56894](https://github.com/microsoft/RD-Agent/commit/db568947f1084a80d603718f5a13fdbd72b90a47))
+* revert draft stage into a soft decay in hypothesis selection ([#849](https://github.com/microsoft/RD-Agent/issues/849)) ([d41db0c](https://github.com/microsoft/RD-Agent/commit/d41db0ca357b07091825ebd9d18c303b6db3cc6a))
+* trace merging ([#836](https://github.com/microsoft/RD-Agent/issues/836)) ([a3d5473](https://github.com/microsoft/RD-Agent/commit/a3d547369e408a05cff570c1239b6320be40418d))
+* truncate by time ([#863](https://github.com/microsoft/RD-Agent/issues/863)) ([2b9427a](https://github.com/microsoft/RD-Agent/commit/2b9427ae036ffe1e28a717502f45500fe91fe5ac))
+* update prompt to improve json respond format of some LLM models ([#928](https://github.com/microsoft/RD-Agent/issues/928)) ([0b84709](https://github.com/microsoft/RD-Agent/commit/0b84709e59c7abb9754961cd17cc9673fcf508aa))
+* using different chat model in different part ([#822](https://github.com/microsoft/RD-Agent/issues/822)) ([c052ea6](https://github.com/microsoft/RD-Agent/commit/c052ea6d1f8948183a4a6ebc873ec01b57373cce))
+
+
+### Bug Fixes
+
+* 'DSProposalV2ExpGen' object has no attribute 'COMPONENT_TASK_MAP… ([#950](https://github.com/microsoft/RD-Agent/issues/950)) ([e353895](https://github.com/microsoft/RD-Agent/commit/e353895251f231fee85abdcb1b22b022a577af77))
+* adapting UI to mock trace ([#841](https://github.com/microsoft/RD-Agent/issues/841)) ([8a5754c](https://github.com/microsoft/RD-Agent/commit/8a5754c9b9c9410d0943aeed777a93c13422e54a))
+* add missing semicolon after chmod in env shell command ([#955](https://github.com/microsoft/RD-Agent/issues/955)) ([1128eaa](https://github.com/microsoft/RD-Agent/commit/1128eaa89ec1dcab4a05ef50d64c7f7e6aae88a8))
+* add time to timer when api timeout bug ([#826](https://github.com/microsoft/RD-Agent/issues/826)) ([f45d6ae](https://github.com/microsoft/RD-Agent/commit/f45d6ae6595c1c39b389485b637a0ae53ffc8782))
+* add wait_retry to exp_gen v2 ([#783](https://github.com/microsoft/RD-Agent/issues/783)) ([b9fb7cf](https://github.com/microsoft/RD-Agent/commit/b9fb7cf4e3070062d91b5b67d0f10d6266b45142))
+* adjust ds_trace lookup and add stderr redirect to mlebench command ([#853](https://github.com/microsoft/RD-Agent/issues/853)) ([4e53108](https://github.com/microsoft/RD-Agent/commit/4e53108e020db719b39cba3a67e0c6dae3de19cf))
+* align competion_full_desc and scenario_all_desc, remove redundant info in problems proposal ([#808](https://github.com/microsoft/RD-Agent/issues/808)) ([76d8536](https://github.com/microsoft/RD-Agent/commit/76d8536d9ec53952383019306781d49cb3e9f75c))
+* bug fix in timer start ([#807](https://github.com/microsoft/RD-Agent/issues/807)) ([9af7161](https://github.com/microsoft/RD-Agent/commit/9af7161eb57bdd2e24b072335e9d185951c32472))
+* bug in problem identification ([#806](https://github.com/microsoft/RD-Agent/issues/806)) ([e1d5a29](https://github.com/microsoft/RD-Agent/commit/e1d5a2914046476f2f10d5884ed3c3ff956d65ff))
+* conda error information ([#941](https://github.com/microsoft/RD-Agent/issues/941)) ([fd39a94](https://github.com/microsoft/RD-Agent/commit/fd39a947763fb4a9be87b907c399bebe384df505))
+* default cost to NaN when calculation fails in LiteLLM backend ([#912](https://github.com/microsoft/RD-Agent/issues/912)) ([51a4048](https://github.com/microsoft/RD-Agent/commit/51a4048129cbfbc3b84bcf50fd8866fafb3e2da3))
+* ds trace ([#929](https://github.com/microsoft/RD-Agent/issues/929)) ([127e441](https://github.com/microsoft/RD-Agent/commit/127e441602e21a46d6313ff39133ab8ca841937e))
+* duplicate model names test in pipeline coder & runner ([#763](https://github.com/microsoft/RD-Agent/issues/763)) ([be3ee9d](https://github.com/microsoft/RD-Agent/commit/be3ee9da9882edda3c06ff7d1099d1bbda2203c3))
+* filter system metadata dirs and init missing DSTrace attribute ([#946](https://github.com/microsoft/RD-Agent/issues/946)) ([10050ef](https://github.com/microsoft/RD-Agent/commit/10050ef368ae7ec07cbf20ac4e52e21c2875eaab))
+* fix a bug in docker result extraction ([#824](https://github.com/microsoft/RD-Agent/issues/824)) ([e1c0f98](https://github.com/microsoft/RD-Agent/commit/e1c0f9826abcbc11dda215a600a2637c9ac6e984))
+* fix competition metric direction ([#784](https://github.com/microsoft/RD-Agent/issues/784)) ([3be0057](https://github.com/microsoft/RD-Agent/commit/3be0057556f46c899065ee1c7f9bafe33e79249c))
+* fix model input shape bug and costeer_model bug ([#821](https://github.com/microsoft/RD-Agent/issues/821)) ([b34bd89](https://github.com/microsoft/RD-Agent/commit/b34bd895d6d9c326aab85856a15be0cb72b2c4c8))
+* fix some minor bugs ([#758](https://github.com/microsoft/RD-Agent/issues/758)) ([963f96e](https://github.com/microsoft/RD-Agent/commit/963f96e5596bee04074135c2a0e31a8adc39ad8c))
+* fix some minor bugs in qlib scenario ([#817](https://github.com/microsoft/RD-Agent/issues/817)) ([79962a7](https://github.com/microsoft/RD-Agent/commit/79962a7ca40c77a3997a68da9ad1b5ab16728483))
+* fix the bug in the regular expression matching for stdout ([#890](https://github.com/microsoft/RD-Agent/issues/890)) ([ee57e37](https://github.com/microsoft/RD-Agent/commit/ee57e37a22af874b262c033d1606dbe7799706db))
+* fix the bug of Exceed-LLM-Context in online merge of multi-tarce ([#892](https://github.com/microsoft/RD-Agent/issues/892)) ([f760a3e](https://github.com/microsoft/RD-Agent/commit/f760a3eff7bd927a31e4958ed2f706312e83e3e3))
+* fix the problems weights bug ([#898](https://github.com/microsoft/RD-Agent/issues/898)) ([013d79f](https://github.com/microsoft/RD-Agent/commit/013d79f12060e908aeb57c3eb1bb56eea86df086))
+* fixed CI execution failures caused by document builds ([#857](https://github.com/microsoft/RD-Agent/issues/857)) ([5c116b2](https://github.com/microsoft/RD-Agent/commit/5c116b24ce727f6ed9ef39d5aa5b60442038c344))
+* get_metric_direction for aerial-cactus-identification ([#970](https://github.com/microsoft/RD-Agent/issues/970)) ([70dc62d](https://github.com/microsoft/RD-Agent/commit/70dc62de5fbd4272ecda1b6fcbcf898b3624a991))
+* import path of T ([#787](https://github.com/microsoft/RD-Agent/issues/787)) ([ac008a6](https://github.com/microsoft/RD-Agent/commit/ac008a61d03b4737ab3d994024e922839d8f3fe1))
+* improve eval alignment check (e.g. small-scale finetuning) ([#802](https://github.com/microsoft/RD-Agent/issues/802)) ([d391578](https://github.com/microsoft/RD-Agent/commit/d3915788082de640a4ce1eea6d2e607319b89c3e))
+* improve file tree and _walk symlink handling ([#877](https://github.com/microsoft/RD-Agent/issues/877)) ([516cb69](https://github.com/microsoft/RD-Agent/commit/516cb69357483ddd99f84b221a056d8491c34f9b))
+* log info ([#965](https://github.com/microsoft/RD-Agent/issues/965)) ([f1dbc21](https://github.com/microsoft/RD-Agent/commit/f1dbc2100498e22c8e5edbb2e4563c99c3d54775))
+* main bug ([#938](https://github.com/microsoft/RD-Agent/issues/938)) ([c6d34d6](https://github.com/microsoft/RD-Agent/commit/c6d34d67b8aedf5496bf6a875915ce657fc58448))
+* non-exist variable test_eval.py ([#847](https://github.com/microsoft/RD-Agent/issues/847)) ([4948c38](https://github.com/microsoft/RD-Agent/commit/4948c38560f4cf021d9354b201b22dfa5ccb9441))
+* refine feedback prompt ([#901](https://github.com/microsoft/RD-Agent/issues/901)) ([12bb2c4](https://github.com/microsoft/RD-Agent/commit/12bb2c4a1494b9aa29962905abb5e433a60eb716))
+* refine the time/memory constraints prompt in hypothesis proposal ([#856](https://github.com/microsoft/RD-Agent/issues/856)) ([51ce8ef](https://github.com/microsoft/RD-Agent/commit/51ce8ef84b4fe6590ce20599a56eee596f2f04e6))
+* Set PYTHONPATH in env.run_ret_code call in FBWorkspace class ([#755](https://github.com/microsoft/RD-Agent/issues/755)) ([68b5018](https://github.com/microsoft/RD-Agent/commit/68b501889caca754f27b57d9ab6f72184e93b15c))
+* task_gen for better understanding ([#752](https://github.com/microsoft/RD-Agent/issues/752)) ([6bfc1e5](https://github.com/microsoft/RD-Agent/commit/6bfc1e570449ee69ac110a4ced9a7cecbc0e6a73))
+* trace list but ([#852](https://github.com/microsoft/RD-Agent/issues/852)) ([32cdc57](https://github.com/microsoft/RD-Agent/commit/32cdc575bde103d71a358d4d99bd413076328ebd))
+* typo in workflow ([#861](https://github.com/microsoft/RD-Agent/issues/861)) ([0e54c9f](https://github.com/microsoft/RD-Agent/commit/0e54c9fe41d25a4cc45ab9e61bb2c2c01b854751))
+* update DS env setup with competition volume and timeout ([#878](https://github.com/microsoft/RD-Agent/issues/878)) ([816ada0](https://github.com/microsoft/RD-Agent/commit/816ada096afabe90578672b0e61b656802a30b62))
+* update feedback.py ([#772](https://github.com/microsoft/RD-Agent/issues/772)) ([133778c](https://github.com/microsoft/RD-Agent/commit/133778c67ee3349f1c2fe029bcf6a9ee14568efe))
+* update metric direction to return bool ([#791](https://github.com/microsoft/RD-Agent/issues/791)) ([0bf365e](https://github.com/microsoft/RD-Agent/commit/0bf365e7830aa86d2350b9d1c47410af46b3a7e8))
+* update runner max loop to 1 in DS scenario ([#820](https://github.com/microsoft/RD-Agent/issues/820)) ([3da378e](https://github.com/microsoft/RD-Agent/commit/3da378e986e8b776a17dbc694d29ef211192ed3e))
+* use fallback messages for missing submission and scores files ([#882](https://github.com/microsoft/RD-Agent/issues/882)) ([898fdea](https://github.com/microsoft/RD-Agent/commit/898fdeae80801d537ebc5c4a3b7df9de74c3403a))
+* use simple stdout and stderr ([#966](https://github.com/microsoft/RD-Agent/issues/966)) ([0b1c445](https://github.com/microsoft/RD-Agent/commit/0b1c445f1f0c212887ffff9f8fac44236df3607c))
+* use trace count as index ([#909](https://github.com/microsoft/RD-Agent/issues/909)) ([b87de56](https://github.com/microsoft/RD-Agent/commit/b87de56e54b206b3aada53850804474eff80b96d))
+* wrong variable test_eval.py ([#846](https://github.com/microsoft/RD-Agent/issues/846)) ([808ea6c](https://github.com/microsoft/RD-Agent/commit/808ea6cba541e60c35dd283cee9098ce46f2a59e))
+
+## [0.4.0](https://github.com/microsoft/RD-Agent/compare/v0.3.0...v0.4.0) (2025-04-04)
+
+
+### Features
+
+* (Kaggle) add base template for competition: tabular-playground-series-may-2022 ([#481](https://github.com/microsoft/RD-Agent/issues/481)) ([f3405ca](https://github.com/microsoft/RD-Agent/commit/f3405ca732eb0ddca8e18ea72f69cbd86055c4ab))
+* a unified CoSTEER to fit more scenarios ([#491](https://github.com/microsoft/RD-Agent/issues/491)) ([cddbd02](https://github.com/microsoft/RD-Agent/commit/cddbd02e3ad3ccf6ad01443777319dc5c7eb08a7))
+* add a new competition ([#474](https://github.com/microsoft/RD-Agent/issues/474)) ([2fc0d77](https://github.com/microsoft/RD-Agent/commit/2fc0d77c485a31f647e21f4578e2e326f7032964))
+* add a tool to enable saving workspace files into a specific folder ([#728](https://github.com/microsoft/RD-Agent/issues/728)) ([bca864b](https://github.com/microsoft/RD-Agent/commit/bca864b7edeafe3f88405efb695ca8acad6252f8))
+* add baseline score stat ([#590](https://github.com/microsoft/RD-Agent/issues/590)) ([2948026](https://github.com/microsoft/RD-Agent/commit/2948026c390d067b643f8c8247c1447f1dc023e4))
+* add configurable volume mode for Docker volumes in env.py ([#537](https://github.com/microsoft/RD-Agent/issues/537)) ([642a022](https://github.com/microsoft/RD-Agent/commit/642a02239431411b91959f23e69b454997ca75d5))
+* add constraint labels for semantic search ([#680](https://github.com/microsoft/RD-Agent/issues/680)) ([0584cfc](https://github.com/microsoft/RD-Agent/commit/0584cfcd13ca1a62c85390ea2ee7574370748d31))
+* add cross validation to workflow ([#700](https://github.com/microsoft/RD-Agent/issues/700)) ([82e9b00](https://github.com/microsoft/RD-Agent/commit/82e9b00be62b01673353a7aaa3ab0e2e3ecaf3ca))
+* add describe_data_folder_v2 ([#738](https://github.com/microsoft/RD-Agent/issues/738)) ([bc8e846](https://github.com/microsoft/RD-Agent/commit/bc8e8460e0246321792ff3347b1b8905416ad075))
+* add do_truncate control for the load function ([#656](https://github.com/microsoft/RD-Agent/issues/656)) ([2b960a5](https://github.com/microsoft/RD-Agent/commit/2b960a58dfdeba69522a0f72ecf0975bb6ae87ee))
+* add do_truncate control for the load function ([#656](https://github.com/microsoft/RD-Agent/issues/656)) ([2b960a5](https://github.com/microsoft/RD-Agent/commit/2b960a58dfdeba69522a0f72ecf0975bb6ae87ee))
+* add eda to data science scenario ([#639](https://github.com/microsoft/RD-Agent/issues/639)) ([35aa479](https://github.com/microsoft/RD-Agent/commit/35aa479f00edf118d43ec228e0a84c155332957a))
+* add hypothesis guidelines and rule-based ranking ([#746](https://github.com/microsoft/RD-Agent/issues/746)) ([c077b82](https://github.com/microsoft/RD-Agent/commit/c077b8239cc72904c4bc450845ed2a11aa5445f0))
+* Add line length limit to shrink_text function and settings ([#715](https://github.com/microsoft/RD-Agent/issues/715)) ([75ed5e1](https://github.com/microsoft/RD-Agent/commit/75ed5e1c2ce1bf20bb55190c10a4134e04694d2b))
+* add loop_n parameter to the main loop ([#611](https://github.com/microsoft/RD-Agent/issues/611)) ([778c166](https://github.com/microsoft/RD-Agent/commit/778c166962250e3b9e7ad85de37f62297d370b45))
+* add max time config to costeer in data science ([#645](https://github.com/microsoft/RD-Agent/issues/645)) ([534686c](https://github.com/microsoft/RD-Agent/commit/534686c2ba7d9fa979c0762ad3177c36f6d7f4cb))
+* add mlebench submission validitor ([#545](https://github.com/microsoft/RD-Agent/issues/545)) ([712d94a](https://github.com/microsoft/RD-Agent/commit/712d94a7d6f22187fc3d18bd434e71ec6997aa9f))
+* add model removal and adjust some framework logic ([#681](https://github.com/microsoft/RD-Agent/issues/681)) ([1edf881](https://github.com/microsoft/RD-Agent/commit/1edf881c63512d351c0dd074d7a1c0965ff3119b))
+* add output_path to load function of LoopBase ([#628](https://github.com/microsoft/RD-Agent/issues/628)) ([dd33726](https://github.com/microsoft/RD-Agent/commit/dd33726ac5de75dc2030d193d457d59490b3361e))
+* add pipeline coder ([#742](https://github.com/microsoft/RD-Agent/issues/742)) ([759f295](https://github.com/microsoft/RD-Agent/commit/759f295dbf1224e177006e72d694e42dd6f372b6))
+* add rank into report (mle_summary) ([#665](https://github.com/microsoft/RD-Agent/issues/665)) ([13f7922](https://github.com/microsoft/RD-Agent/commit/13f7922aaae9e4143aac4ad08ec1c556c2faf04e))
+* add restart and fix unzip ([#538](https://github.com/microsoft/RD-Agent/issues/538)) ([ed2c7d1](https://github.com/microsoft/RD-Agent/commit/ed2c7d175f1f44ca06ad7a63b08da12f6c4df9ab))
+* add retry mechanism with wait_retry decorator and refactor diff generation ([#572](https://github.com/microsoft/RD-Agent/issues/572)) ([de1cd72](https://github.com/microsoft/RD-Agent/commit/de1cd72f068ebd1e1bd5bc2ad2b12ae484d54831))
+* add the shape of the CSV to the dataset description ([#561](https://github.com/microsoft/RD-Agent/issues/561)) ([a10c881](https://github.com/microsoft/RD-Agent/commit/a10c881bd86796e6167257ad26dd165f7e46d813))
+* add timeout settings and cleanup step in data science runner ([#539](https://github.com/microsoft/RD-Agent/issues/539)) ([295abd5](https://github.com/microsoft/RD-Agent/commit/295abd56f7b58055bd27b247dfed47eb85e9b0cd))
+* add type checker to api backend & align litellm and old backend ([#647](https://github.com/microsoft/RD-Agent/issues/647)) ([d38eae9](https://github.com/microsoft/RD-Agent/commit/d38eae986a0ba69d71288fa09fcc21e227551a02))
+* align mlebench data and evaluation & several fix on kaggle workflow ([#477](https://github.com/microsoft/RD-Agent/issues/477)) ([f6c522b](https://github.com/microsoft/RD-Agent/commit/f6c522b651db3c1f6af6815347589917f46e433a))
+* **backend:** integrate LiteLLM API Backend ([#564](https://github.com/microsoft/RD-Agent/issues/564)) ([f477687](https://github.com/microsoft/RD-Agent/commit/f4776879c76a213d53875b307c94be1ea5cfd9ba))
+* base data science scenario UI ([#525](https://github.com/microsoft/RD-Agent/issues/525)) ([39917b3](https://github.com/microsoft/RD-Agent/commit/39917b354b22a8488a17396fe2245cb41e3def03))
+* condaenv & full docker env ([#668](https://github.com/microsoft/RD-Agent/issues/668)) ([084dd6d](https://github.com/microsoft/RD-Agent/commit/084dd6d748a89492ea0888acb316b9bb9efeb62f))
+* diff mode fix ([#569](https://github.com/microsoft/RD-Agent/issues/569)) ([0c509f5](https://github.com/microsoft/RD-Agent/commit/0c509f599ce19303b44d8192ec3eb634c24992d6))
+* display LLM prompt ([#676](https://github.com/microsoft/RD-Agent/issues/676)) ([8c93bba](https://github.com/microsoft/RD-Agent/commit/8c93bba82e185edcf4204cc574df5f41bcdfa9d2))
+* Dynamically find and use sample submission file in eval tests ([#542](https://github.com/microsoft/RD-Agent/issues/542)) ([5f12b44](https://github.com/microsoft/RD-Agent/commit/5f12b44c89dd26b250e914192f9beb2da38fb3ab))
+* end-to-end optimization ([#473](https://github.com/microsoft/RD-Agent/issues/473)) ([d41343a](https://github.com/microsoft/RD-Agent/commit/d41343a63d87bf3479f5ec30745ea788580495bf))
+* Enhance eval script with file cleanup and detailed submission checks ([#529](https://github.com/microsoft/RD-Agent/issues/529)) ([cf2ff92](https://github.com/microsoft/RD-Agent/commit/cf2ff9213d3a8b0fad64df7cae0c35f996d72e27))
+* exclude invalid session log folder ([#554](https://github.com/microsoft/RD-Agent/issues/554)) ([fa86e4d](https://github.com/microsoft/RD-Agent/commit/fa86e4d1805000e0e5779c662ccbb5273fda623c))
+* improve the framework's ability to adaptively adjust the model ([#629](https://github.com/microsoft/RD-Agent/issues/629)) ([93806f3](https://github.com/microsoft/RD-Agent/commit/93806f33a1e0f29a125e29303d4b984a9817c3c0))
+* independent use_azure_token_provider on chat and embedding ([#452](https://github.com/microsoft/RD-Agent/issues/452)) ([d223004](https://github.com/microsoft/RD-Agent/commit/d223004917692e231b251330cbc8676081d5a10d))
+* integrate azure deepseek r1 ([#591](https://github.com/microsoft/RD-Agent/issues/591)) ([e79ce5c](https://github.com/microsoft/RD-Agent/commit/e79ce5c38539138abe04eb9809fbde437e97bbb7))
+* kaggle refactor ([#489](https://github.com/microsoft/RD-Agent/issues/489)) ([1b057d0](https://github.com/microsoft/RD-Agent/commit/1b057d0d63a861fba4b3cb59c6c5fc1a0e3da383))
+* **kaggle:** several update in kaggle scenarios ([#476](https://github.com/microsoft/RD-Agent/issues/476)) ([245d211](https://github.com/microsoft/RD-Agent/commit/245d211dcbfb18ebcc554247a0e3a8dbecf6f3bd))
+* loader prompt &  simplify YAML loading and update data loader specifications ([#736](https://github.com/microsoft/RD-Agent/issues/736)) ([86f8bbf](https://github.com/microsoft/RD-Agent/commit/86f8bbf15895e7c198f9bc395d055ca5f02a5bb6))
+* make spec optional ([#719](https://github.com/microsoft/RD-Agent/issues/719)) ([a16b70f](https://github.com/microsoft/RD-Agent/commit/a16b70ff34c66d7e1c4c7ff5236eca8e7d8abea9))
+* Make system prompt role customizable in LLM settings ([#632](https://github.com/microsoft/RD-Agent/issues/632)) ([e4acd92](https://github.com/microsoft/RD-Agent/commit/e4acd92cc5eec6db5c29cb2d4788020fb89099b7))
+* multi log folder, replace "epxx" in workspace path ([#555](https://github.com/microsoft/RD-Agent/issues/555)) ([8a69c9c](https://github.com/microsoft/RD-Agent/commit/8a69c9c9630860c9b644356e1f71654aea222328))
+* new exp gen v2 implementation ([#725](https://github.com/microsoft/RD-Agent/issues/725)) ([5dcc2d5](https://github.com/microsoft/RD-Agent/commit/5dcc2d5fa63bbe9ae8c4817d9b40b77600440edb))
+* new-york-city-taxi-fare-prediction_template ([#488](https://github.com/microsoft/RD-Agent/issues/488)) ([a9caab7](https://github.com/microsoft/RD-Agent/commit/a9caab7bc5dc86f395a008e523355922137aef17))
+* out spec change for o1-preview ([#666](https://github.com/microsoft/RD-Agent/issues/666)) ([22894bd](https://github.com/microsoft/RD-Agent/commit/22894bdbee26b9cad73646d2975857787e515f75))
+* refactor for general data science ([#498](https://github.com/microsoft/RD-Agent/issues/498)) ([7002dc4](https://github.com/microsoft/RD-Agent/commit/7002dc4981a4f72096b438d2fe4fd9ff268c54f3))
+* refine logic for qlib_factor_from_report ([#463](https://github.com/microsoft/RD-Agent/issues/463)) ([21348d8](https://github.com/microsoft/RD-Agent/commit/21348d89e0e0eec1b4fab4e7a497f1eb34b8fe72))
+* run benchmark on gpt-4o & llama 3.1 ([#497](https://github.com/microsoft/RD-Agent/issues/497)) ([64af0b5](https://github.com/microsoft/RD-Agent/commit/64af0b5529b687cce8b5b7a1893946e15edca626))
+* summary and UI update ([#581](https://github.com/microsoft/RD-Agent/issues/581)) ([efa51f9](https://github.com/microsoft/RD-Agent/commit/efa51f9c259a06fe219f3137f0a1005e50d2bfdd))
+* template changes for some kaggle competitions ([#484](https://github.com/microsoft/RD-Agent/issues/484)) ([2e38000](https://github.com/microsoft/RD-Agent/commit/2e38000091030811fc081d72016c7bbadf7efd50))
+* track and log accumulated completion cost in LiteLLMAPIBackend ([#727](https://github.com/microsoft/RD-Agent/issues/727)) ([b294a95](https://github.com/microsoft/RD-Agent/commit/b294a95e0b7b2ef96af355cebac92d9c87f3acab))
+* update prompts and descriptions for data science components ([#731](https://github.com/microsoft/RD-Agent/issues/731)) ([c20e226](https://github.com/microsoft/RD-Agent/commit/c20e226c3e7771c9fcd1c879a8937e4694dc03eb))
+* variable printing tool of data_science coder testing ([#658](https://github.com/microsoft/RD-Agent/issues/658)) ([116c061](https://github.com/microsoft/RD-Agent/commit/116c06190b01f0b621c021726a1be23458ab1154))
+
+
+### Bug Fixes
+
+* a default conf in scen qlib ([#503](https://github.com/microsoft/RD-Agent/issues/503)) ([d64a228](https://github.com/microsoft/RD-Agent/commit/d64a228525cbedd7687c1e06132eacd0d0647697))
+* a small bug in exp_gen ([#606](https://github.com/microsoft/RD-Agent/issues/606)) ([f734dde](https://github.com/microsoft/RD-Agent/commit/f734dde0b0101e13f38151468c8ddf9e23af26ac))
+* add check when retrying gen model codes ([#699](https://github.com/microsoft/RD-Agent/issues/699)) ([3b82f15](https://github.com/microsoft/RD-Agent/commit/3b82f159474087902d3c6007d370e3282b549015))
+* add DSExperiment type check and directory validation in log proc… ([#535](https://github.com/microsoft/RD-Agent/issues/535)) ([f59b12c](https://github.com/microsoft/RD-Agent/commit/f59b12c9cc9afde82b74bc133797ff1396678627))
+* add ensemble test, change to "use cross-validation if possible" in workflow spec ([#634](https://github.com/microsoft/RD-Agent/issues/634)) ([acc97a8](https://github.com/microsoft/RD-Agent/commit/acc97a8217253497afedcfa829902b4432e1031e))
+* add force parameter for cache_with_pickle & using cache when get kaggle leaderboard ([#687](https://github.com/microsoft/RD-Agent/issues/687)) ([c8841e5](https://github.com/microsoft/RD-Agent/commit/c8841e590a925200859acba9fda4a17d4c3aa1c7))
+* add metric name check for valid scores ([#724](https://github.com/microsoft/RD-Agent/issues/724)) ([acc2ffb](https://github.com/microsoft/RD-Agent/commit/acc2ffbde4df3b53654559d14cd035ee6be6b35e))
+* add retry mechanism for GPU device check in DockerEnv ([#573](https://github.com/microsoft/RD-Agent/issues/573)) ([a780cfb](https://github.com/microsoft/RD-Agent/commit/a780cfb621dc487cc17072bfd4aedd7d581249ab))
+* add scores.csv checking in ensemble_test ([#567](https://github.com/microsoft/RD-Agent/issues/567)) ([01808b4](https://github.com/microsoft/RD-Agent/commit/01808b47c314d1daffacc0a65e0ab934a1c41d65))
+* add stdout context length setting and improve text shrinking logic ([#559](https://github.com/microsoft/RD-Agent/issues/559)) ([4ac26a6](https://github.com/microsoft/RD-Agent/commit/4ac26a65c1f18f7513480dd562566c8a96298aa7))
+* align components' name ([#701](https://github.com/microsoft/RD-Agent/issues/701)) ([295a114](https://github.com/microsoft/RD-Agent/commit/295a1148c53d00b716b2d540573a7f43e7e2d762))
+* auto continue small bug ([#598](https://github.com/microsoft/RD-Agent/issues/598)) ([75eaecf](https://github.com/microsoft/RD-Agent/commit/75eaecf36b9f70dfc2d7fedd35836acdb05f89d6))
+* avoid try-except in ensemble eval prompts ([#637](https://github.com/microsoft/RD-Agent/issues/637)) ([5c58d6e](https://github.com/microsoft/RD-Agent/commit/5c58d6e524ef848024578033ab6d47bc9b220822))
+* avoid warning for missing llama installation when not in use ([#509](https://github.com/microsoft/RD-Agent/issues/509)) ([5ec3422](https://github.com/microsoft/RD-Agent/commit/5ec342224c2c8c4cf591f1eae673e25b14218726))
+* change devault to default ([#688](https://github.com/microsoft/RD-Agent/issues/688)) ([7f401cd](https://github.com/microsoft/RD-Agent/commit/7f401cd1c3b333285acf6d6e57654f4b9f0cb6c5))
+* change ensemble test ([#622](https://github.com/microsoft/RD-Agent/issues/622)) ([5de3595](https://github.com/microsoft/RD-Agent/commit/5de35953ed0d3e2e1f4dff0e0522f2d6475079ec))
+* change summary info of log folder ([#552](https://github.com/microsoft/RD-Agent/issues/552)) ([0eb258d](https://github.com/microsoft/RD-Agent/commit/0eb258d734e9a1280a238b9a6f63eb33047ee0a7))
+* clarify an ambiguous explanation ([#705](https://github.com/microsoft/RD-Agent/issues/705)) ([5dbfc68](https://github.com/microsoft/RD-Agent/commit/5dbfc6859cbf6cc31932dae30cf05506108fc871))
+* clarify cross_validation ([#644](https://github.com/microsoft/RD-Agent/issues/644)) ([906993e](https://github.com/microsoft/RD-Agent/commit/906993ef6482f88131d1af46f5bc66a77034b549))
+* coder prompt & model test text ([#583](https://github.com/microsoft/RD-Agent/issues/583)) ([0a41227](https://github.com/microsoft/RD-Agent/commit/0a41227f267050feaeeb47ddd4d749643eb9f198))
+* correct the configuration inheritance relationship ([#671](https://github.com/microsoft/RD-Agent/issues/671)) ([30b1ff8](https://github.com/microsoft/RD-Agent/commit/30b1ff8e1ce59b741e0b81481962063014641c0b))
+* default emb model ([#702](https://github.com/microsoft/RD-Agent/issues/702)) ([4329a72](https://github.com/microsoft/RD-Agent/commit/4329a722832a201b3fa6f9d8f9d8d46f78110410))
+* direct_exp_gen to json_target_type in DSExpGen class ([#661](https://github.com/microsoft/RD-Agent/issues/661)) ([428b74a](https://github.com/microsoft/RD-Agent/commit/428b74a988157ea864ebb40e828bd9f67589c863))
+* docker error will trigger retry and data science runner loop set to 3 ([#602](https://github.com/microsoft/RD-Agent/issues/602)) ([ad785e0](https://github.com/microsoft/RD-Agent/commit/ad785e03d5db05d9191d5e772e184532835a787b))
+* ensure expected type ([#593](https://github.com/microsoft/RD-Agent/issues/593)) ([098a9a6](https://github.com/microsoft/RD-Agent/commit/098a9a6618f70fa8dd276b9014b9e7ba9621553b))
+* filter empty log traces in ds UI ([#533](https://github.com/microsoft/RD-Agent/issues/533)) ([1a2057c](https://github.com/microsoft/RD-Agent/commit/1a2057c9fc11edc4637f0baaa6dd226eb049c36e))
+* fix a bug in cross validation ([#618](https://github.com/microsoft/RD-Agent/issues/618)) ([05a4f10](https://github.com/microsoft/RD-Agent/commit/05a4f101e0b64b860ad03294619b2350004657e8))
+* fix a bug in ensemble test script ([#713](https://github.com/microsoft/RD-Agent/issues/713)) ([ad32100](https://github.com/microsoft/RD-Agent/commit/ad321000acbd9291d22fe03a9c60e57c70511c73))
+* fix a bug in initial tasks ([#635](https://github.com/microsoft/RD-Agent/issues/635)) ([edb552e](https://github.com/microsoft/RD-Agent/commit/edb552ed283119444f357fbd0b6170b2ad97712a))
+* fix a bug in kaggle conf ([#459](https://github.com/microsoft/RD-Agent/issues/459)) ([b4ed32b](https://github.com/microsoft/RD-Agent/commit/b4ed32b17ef07d8557450063765585a48d5fcd32))
+* fix a bug in progress_bar filter ([#712](https://github.com/microsoft/RD-Agent/issues/712)) ([ba5a84d](https://github.com/microsoft/RD-Agent/commit/ba5a84dee59c39cc2a8c0d428a82da1f899ce537))
+* fix a bug in proposal (add last loop's exception to last task desc) ([#596](https://github.com/microsoft/RD-Agent/issues/596)) ([419186f](https://github.com/microsoft/RD-Agent/commit/419186ffb985fe5a0aa0f7fe59c7a223e355492e))
+* fix a bug in regular expression exception processing ([#734](https://github.com/microsoft/RD-Agent/issues/734)) ([67d3702](https://github.com/microsoft/RD-Agent/commit/67d37027bbcd7294a5890a350fe16fe78e0dfa77))
+* fix a bug in threshold score display ([#592](https://github.com/microsoft/RD-Agent/issues/592)) ([0b0a2dc](https://github.com/microsoft/RD-Agent/commit/0b0a2dc512a5560a66464ad49de25d362d0dc17e))
+* fix a bug related to model_name in ensemble ([#692](https://github.com/microsoft/RD-Agent/issues/692)) ([c6ce473](https://github.com/microsoft/RD-Agent/commit/c6ce4733f32578298abe0b60f9d82611b793cc09))
+* fix a minor bug ([#694](https://github.com/microsoft/RD-Agent/issues/694)) ([1405d8d](https://github.com/microsoft/RD-Agent/commit/1405d8dafd99ecde6f3ba9dd76133d8830d03b47))
+* fix an error in model_coder prompt ([#690](https://github.com/microsoft/RD-Agent/issues/690)) ([4528826](https://github.com/microsoft/RD-Agent/commit/452882674e915dbd9e3399c26c70ce5bb86d012c))
+* fix combined_factors_df.pkl not loading in docker ([#697](https://github.com/microsoft/RD-Agent/issues/697)) ([3984b99](https://github.com/microsoft/RD-Agent/commit/3984b995aa74318b40de7712e100d4de5cc95b11))
+* fix docs build error ([#711](https://github.com/microsoft/RD-Agent/issues/711)) ([c9e1d32](https://github.com/microsoft/RD-Agent/commit/c9e1d32d6b63560350cc7cb799c3a908e2c04e42))
+* fix ExtendedSettingsConfigDict does not work ([#660](https://github.com/microsoft/RD-Agent/issues/660)) ([3a877f3](https://github.com/microsoft/RD-Agent/commit/3a877f383b908da8d027560714030b201946bb76))
+* fix kaggle templates path error ([#747](https://github.com/microsoft/RD-Agent/issues/747)) ([3b3f504](https://github.com/microsoft/RD-Agent/commit/3b3f5041514baf741fe2d4613fa651fb5d9c002d))
+* fix KeyError direct_exp_gen ([#735](https://github.com/microsoft/RD-Agent/issues/735)) ([7200682](https://github.com/microsoft/RD-Agent/commit/7200682ac4e60d3910c29a4f7c4a37b3d24e4224))
+* fix some bugs (ensemble output, HPO, model tuning) ([#648](https://github.com/microsoft/RD-Agent/issues/648)) ([818ee29](https://github.com/microsoft/RD-Agent/commit/818ee29f8e5d4765b9801463b85b42ee9516ec33))
+* fix some bugs in the ensemble component ([#595](https://github.com/microsoft/RD-Agent/issues/595)) ([c0990ab](https://github.com/microsoft/RD-Agent/commit/c0990abb06c73ae062d9a50f50cdfd6d04aded22))
+* fix some bugs in workflow unit test ([#624](https://github.com/microsoft/RD-Agent/issues/624)) ([f845dcc](https://github.com/microsoft/RD-Agent/commit/f845dcc0ee1b059b8b32485ad46bb90c7ae0fa78))
+* fix some description errors in direct_exp_gen ([#698](https://github.com/microsoft/RD-Agent/issues/698)) ([dfaacb6](https://github.com/microsoft/RD-Agent/commit/dfaacb6d06e5d5f55e950d7177570d1efebf958f))
+* fix some minor bugs and add AutoML & cross-validation ([#604](https://github.com/microsoft/RD-Agent/issues/604)) ([18c5ef2](https://github.com/microsoft/RD-Agent/commit/18c5ef268d40efe7bb9ee18aa0d250732bdda6fa))
+* fix submission file search and add TODO in env.py ([#544](https://github.com/microsoft/RD-Agent/issues/544)) ([54d930e](https://github.com/microsoft/RD-Agent/commit/54d930e91e629f0fc2f8bdd0d0d62fcad1e99a9c))
+* fix task return dict with wrong format ([#558](https://github.com/microsoft/RD-Agent/issues/558)) ([2008244](https://github.com/microsoft/RD-Agent/commit/20082440a249dd0e5a7026c2d98c9de0288dd400))
+* fix the errors in the coder and evaluator of the five components ([#576](https://github.com/microsoft/RD-Agent/issues/576)) ([c487f83](https://github.com/microsoft/RD-Agent/commit/c487f835b651cdc40b95bbbe4efcb9a617be9e40))
+* handle division by zero in percentage calculations ([#550](https://github.com/microsoft/RD-Agent/issues/550)) ([de16c91](https://github.com/microsoft/RD-Agent/commit/de16c915e1716ef8cee43ce41069ea1a09cf1f24))
+* handle invalid regex patterns in filter_progress_bar function ([#579](https://github.com/microsoft/RD-Agent/issues/579)) ([b0daee0](https://github.com/microsoft/RD-Agent/commit/b0daee0d90e193ca1d028e01c31ebf368af89601))
+* Handle ValueError when resolving relative path for uri ([#585](https://github.com/microsoft/RD-Agent/issues/585)) ([4c7765a](https://github.com/microsoft/RD-Agent/commit/4c7765a12bda5dcfd9af72b292853d9bc28c5baf))
+* include data information in cache key generation ([#566](https://github.com/microsoft/RD-Agent/issues/566)) ([26dda46](https://github.com/microsoft/RD-Agent/commit/26dda4682b7b643c164589057cb568a4d9e55e17))
+* keep some txt files ([#557](https://github.com/microsoft/RD-Agent/issues/557)) ([54aba85](https://github.com/microsoft/RD-Agent/commit/54aba851c9fa194e318d37700307df59e06c6c84))
+* mle_score save problem ([#674](https://github.com/microsoft/RD-Agent/issues/674)) ([ca2e478](https://github.com/microsoft/RD-Agent/commit/ca2e478cf25c2c8511d5f027e32f8a98fc8e3a07))
+* move docker timeout message to __run() ([#620](https://github.com/microsoft/RD-Agent/issues/620)) ([585f4f9](https://github.com/microsoft/RD-Agent/commit/585f4f96e09f70d00eb397c10bf49c09973111df))
+* move mlebench check into runner ([#556](https://github.com/microsoft/RD-Agent/issues/556)) ([b0f7965](https://github.com/microsoft/RD-Agent/commit/b0f7965f650638273710302efee2e5da037368a2))
+* move next_component_required logic to DSTrace class and accurate implement ([#612](https://github.com/microsoft/RD-Agent/issues/612)) ([c20d311](https://github.com/microsoft/RD-Agent/commit/c20d311792f33b2ccccb466c6ec3155ff8be3213))
+* patching weird azure deployment ([#494](https://github.com/microsoft/RD-Agent/issues/494)) ([89c50ae](https://github.com/microsoft/RD-Agent/commit/89c50aee2ec8bfd1cb23767ddf7dcdd023daac8b))
+* qlib and other scenario bugs ([#636](https://github.com/microsoft/RD-Agent/issues/636)) ([98de31d](https://github.com/microsoft/RD-Agent/commit/98de31d4e577c8c450c9694f73a755c19af571f7))
+* refine prompt to generate the most simple task in init stage ([#546](https://github.com/microsoft/RD-Agent/issues/546)) ([9d6feed](https://github.com/microsoft/RD-Agent/commit/9d6feed28ce034db48482d8d9741ef8c72f4bddc))
+* replace API call with build_cls_from_json_with_retry function ([#548](https://github.com/microsoft/RD-Agent/issues/548)) ([eb72a47](https://github.com/microsoft/RD-Agent/commit/eb72a47fbf9c88dacea9691b8d7e92610492d190))
+* replace func "len()" in ensemble test code to support various data type ([#739](https://github.com/microsoft/RD-Agent/issues/739)) ([ab9c7b9](https://github.com/microsoft/RD-Agent/commit/ab9c7b955f78c5de7ec08a6c1a012a76badbdd0e))
+* return 1D embedding if create_embedding receive a string input ([#670](https://github.com/microsoft/RD-Agent/issues/670)) ([4a9c318](https://github.com/microsoft/RD-Agent/commit/4a9c3180ae4a4b043b1b4a89f51ee69cb6843142))
+* rich.print error when some control char in output ([#684](https://github.com/microsoft/RD-Agent/issues/684)) ([ec0cb2a](https://github.com/microsoft/RD-Agent/commit/ec0cb2a032824023dcd04a3acc93202471d1f90a))
+* Runnable on first complete & Rename method to next_incomplete_component for clarity ([#615](https://github.com/microsoft/RD-Agent/issues/615)) ([93d9f63](https://github.com/microsoft/RD-Agent/commit/93d9f63369a78f78e1a67ab548923bb994d1d3b4))
+* runner COSTEER evaluator ([#693](https://github.com/microsoft/RD-Agent/issues/693)) ([6a379ec](https://github.com/microsoft/RD-Agent/commit/6a379ec9b84d4e4944f1e412347aae4f5a93d476))
+* save only one mle_score pkl for a running exp ([#675](https://github.com/microsoft/RD-Agent/issues/675)) ([f87ab67](https://github.com/microsoft/RD-Agent/commit/f87ab676b73cce82bd9f997ac779e31c571b53c4))
+* Set default value for 'entry' parameter in Env.run method ([#643](https://github.com/microsoft/RD-Agent/issues/643)) ([e50d242](https://github.com/microsoft/RD-Agent/commit/e50d2424b849e4181d6ca02e9cace90236665924))
+* sort file name for cache reproduction ([#588](https://github.com/microsoft/RD-Agent/issues/588)) ([7158410](https://github.com/microsoft/RD-Agent/commit/7158410fbfdd84052f9a69cf1e04e09ac07ca598))
+* sota comparison logic ([#608](https://github.com/microsoft/RD-Agent/issues/608)) ([3575372](https://github.com/microsoft/RD-Agent/commit/35753722c0800d62855faeab996d513e62cfe7de))
+* target json type & round ([#662](https://github.com/microsoft/RD-Agent/issues/662)) ([58cb58f](https://github.com/microsoft/RD-Agent/commit/58cb58f966a1db26f5ea9662a54ba12bc921ee24))
+* templates bug ([#456](https://github.com/microsoft/RD-Agent/issues/456)) ([434a868](https://github.com/microsoft/RD-Agent/commit/434a8687eeda77e27b4938fb19694c15858ee446))
+* trace summary df showing in dsapp ([#551](https://github.com/microsoft/RD-Agent/issues/551)) ([177096d](https://github.com/microsoft/RD-Agent/commit/177096d55fecb8c7dab9650ef8f5a31024cd4c1c))
+* unzip kaggle data ([#464](https://github.com/microsoft/RD-Agent/issues/464)) ([3a9fc8e](https://github.com/microsoft/RD-Agent/commit/3a9fc8e73337d3757267b6f4482499499a1b6792))
+
+## [0.3.0](https://github.com/microsoft/RD-Agent/compare/v0.2.1...v0.3.0) (2024-10-21)
+
+
+### Features
+
+* add a new template for kaggle ([#289](https://github.com/microsoft/RD-Agent/issues/289)) ([eee3ab5](https://github.com/microsoft/RD-Agent/commit/eee3ab5b25198224826cb7a8a17eab28bd5d1f7d))
+* add download submission.csv button for kaggle scenario ([#317](https://github.com/microsoft/RD-Agent/issues/317)) ([dcdcbe4](https://github.com/microsoft/RD-Agent/commit/dcdcbe46b4858bfb133ae3cca056e7f602d5cf63))
+* add kaggle command ([#271](https://github.com/microsoft/RD-Agent/issues/271)) ([0938394](https://github.com/microsoft/RD-Agent/commit/0938394b7084ffbf3294d8c23d2d34bf7322ca0b))
+* add kaggle tpl: feedback-prize ([#331](https://github.com/microsoft/RD-Agent/issues/331)) ([a288e39](https://github.com/microsoft/RD-Agent/commit/a288e399e6b0beec62729bd7d46b98a55de5ab79))
+* add more templates for kaggle ([#291](https://github.com/microsoft/RD-Agent/issues/291)) ([da752ec](https://github.com/microsoft/RD-Agent/commit/da752ec806e6f5f5679bc27ac1c072ed9a319251))
+* add normal rag into framework ([#360](https://github.com/microsoft/RD-Agent/issues/360)) ([91b0b1f](https://github.com/microsoft/RD-Agent/commit/91b0b1f66c3c1bf757cb64c4cfbdcaafe59eab74))
+* add qlib_factor_strategy ([#307](https://github.com/microsoft/RD-Agent/issues/307)) ([f8f59ff](https://github.com/microsoft/RD-Agent/commit/f8f59ff0a1be4428a68c8c27f220aabad0b6c9f0))
+* Add ranking in kaggle scenario ([#401](https://github.com/microsoft/RD-Agent/issues/401)) ([b16b4be](https://github.com/microsoft/RD-Agent/commit/b16b4beb402e0c27dfb39ee9d2a120f1b56d447c))
+* Add runtime measurement for each step and loop in RDLoop. ([#281](https://github.com/microsoft/RD-Agent/issues/281)) ([83058c8](https://github.com/microsoft/RD-Agent/commit/83058c864ceeec413dd29bf501030d5a7bd34679))
+* add s3e11 kaggle template ([#324](https://github.com/microsoft/RD-Agent/issues/324)) ([8c57524](https://github.com/microsoft/RD-Agent/commit/8c57524bead1c8f655a08763d608eb7a6dd5975e))
+* Added RepoAnalyzer to empower auto-summary of a workspace ([#264](https://github.com/microsoft/RD-Agent/issues/264)) ([0bd349a](https://github.com/microsoft/RD-Agent/commit/0bd349af50b9b881ba1774bdeb4d723529ef2aa9))
+* Added support for loading and storing RAG in Kaggle scenarios. ([#269](https://github.com/microsoft/RD-Agent/issues/269)) ([c4895de](https://github.com/microsoft/RD-Agent/commit/c4895de83f1ed000e563d42b3468a6bd9e5a4965))
+* announce Discord and WeChat ([#367](https://github.com/microsoft/RD-Agent/issues/367)) ([acac507](https://github.com/microsoft/RD-Agent/commit/acac5078a103b71afa6bd6c053b0766a6a7e609d))
+* auto submit result after one kaggle RDLoop ([#345](https://github.com/microsoft/RD-Agent/issues/345)) ([ab55d70](https://github.com/microsoft/RD-Agent/commit/ab55d7052b53a928b84dc5d5d0d2999d90ca9056))
+* better feedback & evaluation ([#346](https://github.com/microsoft/RD-Agent/issues/346)) ([cc9a8c1](https://github.com/microsoft/RD-Agent/commit/cc9a8c1eab3ca89f8c1e5de4a2bb4e7fcc0cc615))
+* Dynamic scenario based on task ([#392](https://github.com/microsoft/RD-Agent/issues/392)) ([665a037](https://github.com/microsoft/RD-Agent/commit/665a037e4fd7326c450e3fa0d0605eea26fd9ef3))
+* Factor Implement Search Enhancement ([#294](https://github.com/microsoft/RD-Agent/issues/294)) ([4ecf25f](https://github.com/microsoft/RD-Agent/commit/4ecf25f0acf2389a172b14d3dab20895daf2ab89))
+* Feature selection v3 to support all actions  ([#280](https://github.com/microsoft/RD-Agent/issues/280)) ([0047641](https://github.com/microsoft/RD-Agent/commit/00476413fbf00e36e71ab3ccb48d4e766b6ccf4d))
+* fix some bugs and add original features' description ([#259](https://github.com/microsoft/RD-Agent/issues/259)) ([1a5f45a](https://github.com/microsoft/RD-Agent/commit/1a5f45a40d821c017bdba14af8c93710707c5ea5))
+* get kaggle notebooks & disscussion text for RAG ([#371](https://github.com/microsoft/RD-Agent/issues/371)) ([cead345](https://github.com/microsoft/RD-Agent/commit/cead3450a14bf4b142ac988c27fa098c7656a95c))
+* Iceberge competition ([#372](https://github.com/microsoft/RD-Agent/issues/372)) ([c10ea4f](https://github.com/microsoft/RD-Agent/commit/c10ea4f5d4cc56a75b47cf23c7084ee189ba1a25))
+* implement isolated model feature selection loop ([#370](https://github.com/microsoft/RD-Agent/issues/370)) ([cf1292d](https://github.com/microsoft/RD-Agent/commit/cf1292de1a0153ca14ea64971e73a1c93f7d89e3))
+* Initial version if Graph RAG in KAGGLE scenario ([#301](https://github.com/microsoft/RD-Agent/issues/301)) ([fd3c0fd](https://github.com/microsoft/RD-Agent/commit/fd3c0fd26eff7d3be72fa4f2a234e33b9f796627))
+* Integrate RAG into the Kaggle scenarios. ([#262](https://github.com/microsoft/RD-Agent/issues/262)) ([be0e48a](https://github.com/microsoft/RD-Agent/commit/be0e48a7dfbee2b5d2947d09115db5db2e5266f1))
+* Kaggle loop update (Feature & Model) ([#241](https://github.com/microsoft/RD-Agent/issues/241)) ([4cf22a6](https://github.com/microsoft/RD-Agent/commit/4cf22a65c964123b4267569ee02c0c7094c54ca4))
+* kaggle templates related ([#287](https://github.com/microsoft/RD-Agent/issues/287)) ([785fdc1](https://github.com/microsoft/RD-Agent/commit/785fdc144d16fa8454b7c9d2e53e78fe7f22a29a))
+* Model context for tuning and selection ([#284](https://github.com/microsoft/RD-Agent/issues/284)) ([f2831e7](https://github.com/microsoft/RD-Agent/commit/f2831e7442510668b0ca75953b3359894803ef3c))
+* Modify FactorRowCountEvaluator and FactorIndexEvaluator to return the ratio ([#328](https://github.com/microsoft/RD-Agent/issues/328)) ([8f43f8e](https://github.com/microsoft/RD-Agent/commit/8f43f8e87a92e05b541e925910608606ec8f6c4b))
+* New competition - Optiver ([#356](https://github.com/microsoft/RD-Agent/issues/356)) ([3705efe](https://github.com/microsoft/RD-Agent/commit/3705efe3b923748655a57d76b7a236e54d361831))
+* random forest for s3e11 ([#347](https://github.com/microsoft/RD-Agent/issues/347)) ([b57846d](https://github.com/microsoft/RD-Agent/commit/b57846d29314e9a5967945d1b4895f0f48c0f5ce))
+* refine the code in model description and fix some bugs in feedback.py ([#288](https://github.com/microsoft/RD-Agent/issues/288)) ([5b124d7](https://github.com/microsoft/RD-Agent/commit/5b124d7372137e4c613eb2749ddcc773922cc7b6))
+* refine the template in several Kaggle competitions ([#343](https://github.com/microsoft/RD-Agent/issues/343)) ([034f238](https://github.com/microsoft/RD-Agent/commit/034f238ed5ec351486b21250eabc75114961936c))
+* Revise to support better hypothesis proposal  ([#390](https://github.com/microsoft/RD-Agent/issues/390)) ([c55ec0a](https://github.com/microsoft/RD-Agent/commit/c55ec0a0f577bbf7fc6228f7b87d2089ded83b31))
+* show workspace in demo ([#348](https://github.com/microsoft/RD-Agent/issues/348)) ([ddf567c](https://github.com/microsoft/RD-Agent/commit/ddf567c551b553788be022e9312c209ef6137d64))
+* support Multi output ([#330](https://github.com/microsoft/RD-Agent/issues/330)) ([3d36c45](https://github.com/microsoft/RD-Agent/commit/3d36c452ff0983800e5343834cc69f24a508ea70))
+* Supporting COVID-19 competition ([#374](https://github.com/microsoft/RD-Agent/issues/374)) ([a1b63db](https://github.com/microsoft/RD-Agent/commit/a1b63db79600edc9a74ba713c9d0be290214a592))
+* supporting Mnist competition ([#375](https://github.com/microsoft/RD-Agent/issues/375)) ([e958a34](https://github.com/microsoft/RD-Agent/commit/e958a34f5632a46ac43bff8e0d07d6ed020fdfc2))
+* Supporting Model Specifications ([#319](https://github.com/microsoft/RD-Agent/issues/319)) ([e126471](https://github.com/microsoft/RD-Agent/commit/e1264719e10b76158a91cd0ef331848e7c2de7c7))
+* supporting various Kaggle competitions & scenarios for RD-Agent ([#409](https://github.com/microsoft/RD-Agent/issues/409)) ([75eea22](https://github.com/microsoft/RD-Agent/commit/75eea22cc3d4e6f5a94c88cce915e27c507f8c50))
+* template for kaggle ([#308](https://github.com/microsoft/RD-Agent/issues/308)) ([ff97cf0](https://github.com/microsoft/RD-Agent/commit/ff97cf0155ab6941e4b5cf7d103575f934b70dc9))
+* use auto gen seed when using LLM cache ([#441](https://github.com/microsoft/RD-Agent/issues/441)) ([ca15365](https://github.com/microsoft/RD-Agent/commit/ca15365d23eeb094f42cf3dc8f5269b2f1c42bd3))
+* use unified pickle cacher & move llm config into a isolated config ([#424](https://github.com/microsoft/RD-Agent/issues/424)) ([2879ecf](https://github.com/microsoft/RD-Agent/commit/2879ecff816d97688b60909a79c7e568d42608a1))
+* xgboost gpu accelerate ([#359](https://github.com/microsoft/RD-Agent/issues/359)) ([56a5b8f](https://github.com/microsoft/RD-Agent/commit/56a5b8f9b2c6726cc64ec5b04b4ce7935d59b572))
+
+
+### Bug Fixes
+
+* a bug of developer& edit s4e8 template ([#338](https://github.com/microsoft/RD-Agent/issues/338)) ([f12ce72](https://github.com/microsoft/RD-Agent/commit/f12ce726e7de96d478a232a3c27f92439820f8b4))
+* actively raised errors aer also considered as negative feedback. ([#268](https://github.com/microsoft/RD-Agent/issues/268)) ([46ec908](https://github.com/microsoft/RD-Agent/commit/46ec908e3594ac5e4cdc4057268e2f8800f5ed1f))
+* bug of saving preprocess cache files ([#310](https://github.com/microsoft/RD-Agent/issues/310)) ([5fb0608](https://github.com/microsoft/RD-Agent/commit/5fb0608f39f113cc9807fb1f381284a0bd4da318))
+* cache ([#383](https://github.com/microsoft/RD-Agent/issues/383)) ([f2a6e75](https://github.com/microsoft/RD-Agent/commit/f2a6e75b36ca96f7733b9c2a7154ac67bd2d7c6f))
+* change css tag of kaggle competition info crawler ([#306](https://github.com/microsoft/RD-Agent/issues/306)) ([1e3d38b](https://github.com/microsoft/RD-Agent/commit/1e3d38bf1ca3654f3a90ff392ecba1dbb4e80224))
+* debug dsagent ([#387](https://github.com/microsoft/RD-Agent/issues/387)) ([8fe9511](https://github.com/microsoft/RD-Agent/commit/8fe9511e606ba148c66f384add6ab94857079541))
+* eval_method cannot catch run factor error ([#260](https://github.com/microsoft/RD-Agent/issues/260)) ([2aaab31](https://github.com/microsoft/RD-Agent/commit/2aaab317ccb7a0121063bcd85fc36c21c7b8a391))
+* fix a bug in competition metric evaluation ([#407](https://github.com/microsoft/RD-Agent/issues/407)) ([94c47d6](https://github.com/microsoft/RD-Agent/commit/94c47d6fd5c3e38fc786a83e6d0d05e8d04498f3))
+* fix a bug in mini case ([#389](https://github.com/microsoft/RD-Agent/issues/389)) ([e75bb57](https://github.com/microsoft/RD-Agent/commit/e75bb5746f63933b750406bbd34ee63c5ba76b9f))
+* fix a bug in model tuning feedback ([#316](https://github.com/microsoft/RD-Agent/issues/316)) ([8aa088d](https://github.com/microsoft/RD-Agent/commit/8aa088da2dc7525a3970c01d01987246f47d6238))
+* fix a bug in scenario.py ([#388](https://github.com/microsoft/RD-Agent/issues/388)) ([999a1eb](https://github.com/microsoft/RD-Agent/commit/999a1eb0eff9088e1b02419db741db4acf8d9ff7))
+* fix a bug in the format of the model input ([#327](https://github.com/microsoft/RD-Agent/issues/327)) ([8f0574e](https://github.com/microsoft/RD-Agent/commit/8f0574eaaadb245b8c38e09ad4821306996d926f))
+* fix a small bug in cache using module name and function name as unique folder name ([#429](https://github.com/microsoft/RD-Agent/issues/429)) ([4f8134a](https://github.com/microsoft/RD-Agent/commit/4f8134a697d952f7ac824d7ebeec64bbc4545ab3))
+* fix a typo ([#362](https://github.com/microsoft/RD-Agent/issues/362)) ([9fafabd](https://github.com/microsoft/RD-Agent/commit/9fafabdf321b818bdd2211a2324d50cd0ebe1c1f))
+* fix cache result logic ([#430](https://github.com/microsoft/RD-Agent/issues/430)) ([5e34263](https://github.com/microsoft/RD-Agent/commit/5e342637dcc862679fd0642c6ba9ef048c984845))
+* fix command injection ([#421](https://github.com/microsoft/RD-Agent/issues/421)) ([52f30a6](https://github.com/microsoft/RD-Agent/commit/52f30a6184af1295be15e855a80b84bc424fc75d))
+* fix json load error ([#386](https://github.com/microsoft/RD-Agent/issues/386)) ([bba55fb](https://github.com/microsoft/RD-Agent/commit/bba55fb48fe105f4847c1b9c476eedc80835f523))
+* fix some bugs in feedback.py and refine the prompt ([#292](https://github.com/microsoft/RD-Agent/issues/292)) ([d834052](https://github.com/microsoft/RD-Agent/commit/d8340527f133dcc649d599d90d6402eddd37859e))
+* fix some bugs in knowledge base ([#378](https://github.com/microsoft/RD-Agent/issues/378)) ([fa6ff8e](https://github.com/microsoft/RD-Agent/commit/fa6ff8e591cf1847df77d73116649c5623161573))
+* fix some bugs in rag ([#399](https://github.com/microsoft/RD-Agent/issues/399)) ([194215c](https://github.com/microsoft/RD-Agent/commit/194215c4559aee5b6ece18d65c95fb30968e2db6))
+* fix some bugs in the entire loop ([#274](https://github.com/microsoft/RD-Agent/issues/274)) ([8a564ec](https://github.com/microsoft/RD-Agent/commit/8a564ece1d87b27ee98b76db317935e802468965))
+* fix some errors in scenario.py, proposal.py and runner.py and several complex competition scenarios([#365](https://github.com/microsoft/RD-Agent/issues/365)) ([2e383b1](https://github.com/microsoft/RD-Agent/commit/2e383b175d8448a67cb470f4e3ae8977d8ec6b5b))
+* improve_execution_time_in_kaggle_loop ([#279](https://github.com/microsoft/RD-Agent/issues/279)) ([4c8f998](https://github.com/microsoft/RD-Agent/commit/4c8f998c76f1e983a5687d2c65d3251750f2a9a0))
+* kaggle data mount problem ([#297](https://github.com/microsoft/RD-Agent/issues/297)) ([795df31](https://github.com/microsoft/RD-Agent/commit/795df311e3f93cd2f3fb51ba5698adaf10f6bd62))
+* Optiver fixes ([#357](https://github.com/microsoft/RD-Agent/issues/357)) ([b054017](https://github.com/microsoft/RD-Agent/commit/b054017463af0d1784407030f2477d212118f341))
+* partial bug in bench ([#368](https://github.com/microsoft/RD-Agent/issues/368)) ([af9808f](https://github.com/microsoft/RD-Agent/commit/af9808f98736a2df07e121c2f6d7bfeb7b7d3581))
+* preprocess output format & some mistake in spelling ([#358](https://github.com/microsoft/RD-Agent/issues/358)) ([b8b2cd6](https://github.com/microsoft/RD-Agent/commit/b8b2cd6ccd3b27aa73de847e50899a8a53b71b8f))
+* rag save file ([#385](https://github.com/microsoft/RD-Agent/issues/385)) ([1cb01dd](https://github.com/microsoft/RD-Agent/commit/1cb01dd6fe595f2f5fb86487601326611dd1a57a))
+* raise error in demo when no Metric in a Loop ([#313](https://github.com/microsoft/RD-Agent/issues/313)) ([e46a78e](https://github.com/microsoft/RD-Agent/commit/e46a78eb69271cb19978aab2f3b976c2870ca082))
+* refactor Bench ([#302](https://github.com/microsoft/RD-Agent/issues/302)) ([78a87f6](https://github.com/microsoft/RD-Agent/commit/78a87f624780ff67c0fa995ae4692678a120f99c))
+* refine some codes ([#353](https://github.com/microsoft/RD-Agent/issues/353)) ([866c2e6](https://github.com/microsoft/RD-Agent/commit/866c2e63ffa3876a3d16ad37f96da41d0558b714))
+* refine the prompt ([#286](https://github.com/microsoft/RD-Agent/issues/286)) ([77966c4](https://github.com/microsoft/RD-Agent/commit/77966c4f5e9f492c437c5b4b78d89c0f875ef0d8))
+* refine the ucb algorithm ([#406](https://github.com/microsoft/RD-Agent/issues/406)) ([14f7d97](https://github.com/microsoft/RD-Agent/commit/14f7d976e03c92d6e727524e0cdad8a03b585016))
+* revert model and make SOTA model available to COSTEER ([#351](https://github.com/microsoft/RD-Agent/issues/351)) ([3b7437b](https://github.com/microsoft/RD-Agent/commit/3b7437b87e685188259779cd85a78a0b592de9de))
+* stop using markup in docker env print ([#336](https://github.com/microsoft/RD-Agent/issues/336)) ([3009889](https://github.com/microsoft/RD-Agent/commit/3009889b5e2605b5427c76f3084e0e58026bb5ae))
+* support seed and fix absolute path ([#278](https://github.com/microsoft/RD-Agent/issues/278)) ([26352e1](https://github.com/microsoft/RD-Agent/commit/26352e13121cad5be95c0de78bb9f5dda4330614))
+* template for kaggle foreset & s4e9 ([#334](https://github.com/microsoft/RD-Agent/issues/334)) ([2393a41](https://github.com/microsoft/RD-Agent/commit/2393a41e7237615ced2c3fdd5c49308236b9f276))
+* test kaggle method ([#296](https://github.com/microsoft/RD-Agent/issues/296)) ([91a6196](https://github.com/microsoft/RD-Agent/commit/91a619618be1d7db660ea2b413a78dfaba9417a1))
+* update code to fix a small bug in model cache md5 hash ([#303](https://github.com/microsoft/RD-Agent/issues/303)) ([b00e4dc](https://github.com/microsoft/RD-Agent/commit/b00e4dc2eff5b16029a2a12a6589eadac5cfd148))
+* update new feature engineering code format ([#272](https://github.com/microsoft/RD-Agent/issues/272)) ([7850b80](https://github.com/microsoft/RD-Agent/commit/7850b8006a7c89d22629b345b4f361b0f35bc60d))
+* Update prompts.yaml to constrain only one model type   ([#341](https://github.com/microsoft/RD-Agent/issues/341)) ([5b5dfee](https://github.com/microsoft/RD-Agent/commit/5b5dfeefbc7eb9dcbd9923544005c5d281262c03))
+* Update runner.py to fix a small bug ([#282](https://github.com/microsoft/RD-Agent/issues/282)) ([8aef3ab](https://github.com/microsoft/RD-Agent/commit/8aef3abcecd6002bd4bfeedcbe2c786d8bbfe2be))
+* Use fixed file name in model costeer & fixing cache ([#311](https://github.com/microsoft/RD-Agent/issues/311)) ([1f910a5](https://github.com/microsoft/RD-Agent/commit/1f910a5248bc576895ed66c2f7b2c3e046a2bc28))
+
+
+### Performance Improvements
+
+* some small upgrade to factor costeer to improve the performance ([#420](https://github.com/microsoft/RD-Agent/issues/420)) ([9eb931f](https://github.com/microsoft/RD-Agent/commit/9eb931ffd971f252380dbd33ad1db259a4f229fd))
+
+
+### Reverts
+
+* Revert feat: Factor Implement Search Enhancement ([#294](https://github.com/microsoft/RD-Agent/issues/294)) ([#305](https://github.com/microsoft/RD-Agent/issues/305)) ([f663cf4](https://github.com/microsoft/RD-Agent/commit/f663cf42a2f75cd52aef1c6b18be7c27f0641fed))
+
+## [0.2.1](https://github.com/microsoft/RD-Agent/compare/v0.2.0...v0.2.1) (2024-09-10)
+
+
+### Bug Fixes
+
+* default model value in config ([#256](https://github.com/microsoft/RD-Agent/issues/256)) ([c097585](https://github.com/microsoft/RD-Agent/commit/c097585f631f401c2c0966f6ad4c17286924f011))
+* fix_dotenv_error ([#257](https://github.com/microsoft/RD-Agent/issues/257)) ([923063c](https://github.com/microsoft/RD-Agent/commit/923063c1fd957c4ed42e97272c72b5e9545451dc))
+* readme ([#248](https://github.com/microsoft/RD-Agent/issues/248)) ([8cede22](https://github.com/microsoft/RD-Agent/commit/8cede2209922876490148459e1134da828e1fda0))
+
+## [0.2.0](https://github.com/microsoft/RD-Agent/compare/v0.1.0...v0.2.0) (2024-09-07)
+
+
+### Features
+
+* add collect info ([#233](https://github.com/microsoft/RD-Agent/issues/233)) ([89f4af9](https://github.com/microsoft/RD-Agent/commit/89f4af90fb4d95a0689bf9efc8ffd9326469c0aa))
+* add cross validation for kaggle scenario ([#236](https://github.com/microsoft/RD-Agent/issues/236)) ([e0b03ba](https://github.com/microsoft/RD-Agent/commit/e0b03ba6b5c3d9aa552b99d470e106d4e348e64d))
+* add progress status for docker env ([#215](https://github.com/microsoft/RD-Agent/issues/215)) ([538d4ef](https://github.com/microsoft/RD-Agent/commit/538d4ef2e52de795b90d3f75b2e1e877ab85c18d))
+* Added loop code for Kaggle scene. ([#211](https://github.com/microsoft/RD-Agent/issues/211)) ([975c327](https://github.com/microsoft/RD-Agent/commit/975c32715e51aec6b49537401f5fc59115e04a01))
+* Demo display effect and usage ([#162](https://github.com/microsoft/RD-Agent/issues/162)) ([8cf122a](https://github.com/microsoft/RD-Agent/commit/8cf122a0155f434fa4477ae7a6d616b5caecd3e0))
+* piloting of the framework ([#227](https://github.com/microsoft/RD-Agent/issues/227)) ([e9b103e](https://github.com/microsoft/RD-Agent/commit/e9b103e684fdd2b98cd1a89971a3fce2d6e884a1))
+* support more models for kaggle scenario ([#223](https://github.com/microsoft/RD-Agent/issues/223)) ([e3a9659](https://github.com/microsoft/RD-Agent/commit/e3a96598c0720fe092ec86d7ca8c195c7d6bcc72))
+* update model_experiment.py to support basic EDA ([#220](https://github.com/microsoft/RD-Agent/issues/220)) ([bf2684c](https://github.com/microsoft/RD-Agent/commit/bf2684c4d55ab8e1048ac0291695475ad53b0cd6))
+
+
+### Bug Fixes
+
+* fix some bugs in llm calling ([#217](https://github.com/microsoft/RD-Agent/issues/217)) ([7b010f8](https://github.com/microsoft/RD-Agent/commit/7b010f8b5940aba65a58f1d78192aa80bcd0e654))
+* package dependency. ([#234](https://github.com/microsoft/RD-Agent/issues/234)) ([46be295](https://github.com/microsoft/RD-Agent/commit/46be2952952af534fd8d98a656c704c688d7cbdd))
+* remove useless line ([#177](https://github.com/microsoft/RD-Agent/issues/177)) ([64e9a8e](https://github.com/microsoft/RD-Agent/commit/64e9a8e39a2072a962111db18f5b9565df5b0176))
+
+## [0.1.0](https://github.com/microsoft/RD-Agent/compare/v0.0.1...v0.1.0) (2024-08-09)
+
+
+### Features
+
+* add entry for rdagent. ([#187](https://github.com/microsoft/RD-Agent/issues/187)) ([121b6d9](https://github.com/microsoft/RD-Agent/commit/121b6d98de38cd03be30cbee47b40baf39a2b60b))
+* change ui entry ([#197](https://github.com/microsoft/RD-Agent/issues/197)) ([fa5d335](https://github.com/microsoft/RD-Agent/commit/fa5d3354d22240888f4fc4007d9834f7424632aa))
+* remove pdfs and enable online pdf readings ([#183](https://github.com/microsoft/RD-Agent/issues/183)) ([18c0501](https://github.com/microsoft/RD-Agent/commit/18c05016a23d694c7b12759cf1322562dcffc56a))
+
+
+### Bug Fixes
+
+* Fix a fail href in readme ([#189](https://github.com/microsoft/RD-Agent/issues/189)) ([1b89218](https://github.com/microsoft/RD-Agent/commit/1b89218f6bc697494f4a1b8a42ad18963002714f))
+* fix quick start problem ([#191](https://github.com/microsoft/RD-Agent/issues/191)) ([44f61bf](https://github.com/microsoft/RD-Agent/commit/44f61bfa1058a8efb59ca48b7f1417765aeea33e))
+* update command line in readme.md ([#192](https://github.com/microsoft/RD-Agent/issues/192)) ([9c45d24](https://github.com/microsoft/RD-Agent/commit/9c45d24a192da02f7d9765cb001097da1bc36c61))
+
+## 0.0.1 (2024-08-08)
+
+
+### Features
+
+* Add description for scenario experiments. ([#174](https://github.com/microsoft/RD-Agent/issues/174)) ([fbd8c6d](https://github.com/microsoft/RD-Agent/commit/fbd8c6d87e1424c08997103b8e8fbf264858c4ed))
+* Added QlibFactorFromReportScenario and improved the report-factor loop. ([#161](https://github.com/microsoft/RD-Agent/issues/161)) ([882c79b](https://github.com/microsoft/RD-Agent/commit/882c79bf11583980e646b130f71cfa20201ffc7b))
+* filter feature which is high correlation to former implemented features ([#145](https://github.com/microsoft/RD-Agent/issues/145)) ([e818326](https://github.com/microsoft/RD-Agent/commit/e818326422740e04a4863f7c3c18744dde2ad98f))
+* Remove redundant 'key steps' section in frontend scene display. ([#169](https://github.com/microsoft/RD-Agent/issues/169)) ([e767005](https://github.com/microsoft/RD-Agent/commit/e76700513bee29232c93b97414419df330d9be8d))
+* streamlit webapp demo for different scenarios ([#135](https://github.com/microsoft/RD-Agent/issues/135)) ([d8da7db](https://github.com/microsoft/RD-Agent/commit/d8da7db865e6653fc4740efee9a843b69bd79699))
+* Uploaded Documentation, Updated Prompts & Some Code for model demo ([#144](https://github.com/microsoft/RD-Agent/issues/144)) ([529f935](https://github.com/microsoft/RD-Agent/commit/529f935aa98623f0dc1dda29eecee3ef738dd446))
+
+
+### Bug Fixes
+
+* Add framework handling for task coding failure. ([#176](https://github.com/microsoft/RD-Agent/issues/176)) ([5e14fa5](https://github.com/microsoft/RD-Agent/commit/5e14fa54a9dd30a94aebe2643b8c9a3b85517a11))
+* Comprehensive update to factor extraction. ([#143](https://github.com/microsoft/RD-Agent/issues/143)) ([b5ea040](https://github.com/microsoft/RD-Agent/commit/b5ea04019fd5fa15c0f8b9a7e4f18f490f7057d4))
+* first round app folder cleaning ([#166](https://github.com/microsoft/RD-Agent/issues/166)) ([6a5a750](https://github.com/microsoft/RD-Agent/commit/6a5a75021912927deb5e8e4c7ad3ec4b51bfc788))
+* fix pickle problem ([#140](https://github.com/microsoft/RD-Agent/issues/140)) ([7ee4258](https://github.com/microsoft/RD-Agent/commit/7ee42587b60d94417f34332cee395cf210dc8a0e))
+* fix release CI ([#165](https://github.com/microsoft/RD-Agent/issues/165)) ([85d6a5e](https://github.com/microsoft/RD-Agent/commit/85d6a5ed91113fda34ae079b23c89aa24acd2cb2))
+* fix release CI error ([#160](https://github.com/microsoft/RD-Agent/issues/160)) ([1c9f8ef](https://github.com/microsoft/RD-Agent/commit/1c9f8ef287961731944acc9008496b4dddeddca7))
+* fix several bugs in data mining scenario ([#147](https://github.com/microsoft/RD-Agent/issues/147)) ([b233380](https://github.com/microsoft/RD-Agent/commit/b233380e2c66fb030db39424f0f040c86e37f5c4))
+* fix some small bugs in report-factor loop ([#152](https://github.com/microsoft/RD-Agent/issues/152)) ([a79f9f9](https://github.com/microsoft/RD-Agent/commit/a79f9f93406aff6305a76e6a6abd3852642e4c62))
+* fix_release_ci_error ([#150](https://github.com/microsoft/RD-Agent/issues/150)) ([4f82e99](https://github.com/microsoft/RD-Agent/commit/4f82e9960a2638af9d831581185ddd3bac5711fc))
+* Fixed some bugs introduced during refactoring. ([#167](https://github.com/microsoft/RD-Agent/issues/167)) ([f8f1445](https://github.com/microsoft/RD-Agent/commit/f8f1445283fb89aefeb2918243c35a219a51a56c))
+* optimize some prompts in factor loop. ([#158](https://github.com/microsoft/RD-Agent/issues/158)) ([c2c1330](https://github.com/microsoft/RD-Agent/commit/c2c13300b9ad315a663ec2d0eada414e56c6f54f))
+
+
+### Miscellaneous Chores
+
+* release 0.0.1 ([1feacd3](https://github.com/microsoft/RD-Agent/commit/1feacd39b21193de11e9bbecf880ddf96d7c261c))

CONTRIBUTING.md

@@ -0,0 +1,50 @@
+# Contributing to RD-Agent
+
+We welcome contributions and suggestions to improve RD-Agent. Whether it's solving an issue, addressing a bug, enhancing documentation, or even correcting a typo, every contribution is valuable and helps improve the project.
+
+## Getting Started
+
+To get started, you can explore the issues list or search for `TODO:` comments in the codebase by running the command:
+```sh
+grep -r "TODO:"
+```
+
+## How to Contribute
+
+1. **Fork the Repository**: Create a fork of the repository on GitHub.
+2. **Clone the Repository**: Clone your forked repository to your local machine.
+   ```sh
+   git clone https://github.com/your-username/RD-Agent.git
+   ```
+3. **Create a Branch**: Create a new branch for your changes.
+   ```sh
+   git checkout -b feature/your-feature-name
+   ```
+4. **Make Changes**: Make your changes to the codebase.
+5. **Commit Changes**: Commit your changes with a descriptive commit message.
+   ```sh
+   git commit -m "Description of your changes"
+   ```
+6. **Push Changes**: Push your changes to your forked repository.
+   ```sh
+   git push origin feature/your-feature-name
+   ```
+7. **Ensure CI Passes**: Make sure your code passes the automatic CI checks on GitHub.
+8. **Create a Pull Request**: Create a pull request from your forked repository to the main repository.
+
+## Code of Conduct
+
+Please adhere to the [Code of Conduct](CODE_OF_CONDUCT.md) in all your interactions with the project.
+
+## Reporting Issues
+
+If you encounter any issues or have suggestions for improvements, please open an issue on GitHub.
+
+## Guidelines
+
+- Ensure your code follows the project's coding standards.
+- Write clear and concise commit messages.
+- Update documentation as needed.
+- Test your changes thoroughly before submitting a pull request.
+
+Thank you for contributing to RD-Agent!

Makefile

@@ -68,6 +68,7 @@ init-qlib-env:
 
 dev:
 	$(PIPRUN) pip install -e .[docs,lint,package,test] -c $(CONSTRAINTS_FILE)
+	$(PIPRUN) pip install -U kaggle
 	if [ "$(CI)" != "true" ] && command -v pre-commit > /dev/null 2>&1; then pre-commit install --hook-type pre-push; fi
 
 # Generate constraints for current Python version.
@@ -81,31 +82,56 @@ constraints: deepclean
 
 # Check lint with black.
 black:
-	$(PIPRUN) python -m black --check . --extend-exclude test/scripts --extend-exclude git_ignore_folder -l 120
+	$(PIPRUN) python -m black --check --diff . --extend-exclude test/scripts --extend-exclude git_ignore_folder -l 120
 
 # Check lint with isort.
 isort:
 	$(PIPRUN) python -m isort --check . -s git_ignore_folder -s test/scripts
 
 # Check lint with mypy.
+# First deal with the core folder, and then gradually increase the scope of detection,
+# and eventually realize the detection of the complete project.
 mypy:
-	$(PIPRUN) python -m mypy . --exclude rdagent/scripts --exclude git_ignore_folder
+	$(PIPRUN) python -m mypy rdagent/core
 
 # Check lint with ruff.
+# First deal with the core folder, and then gradually increase the scope of detection,
+# and eventually realize the detection of the complete project.
 ruff:
-	$(PIPRUN) ruff check .  --exclude rdagent/scripts,git_ignore_folder
+	$(PIPRUN) ruff check rdagent/core --ignore FBT001,FBT002,I001   # --exclude rdagent/scripts,git_ignore_folder
 
 # Check lint with toml-sort.
 toml-sort:
 	$(PIPRUN) toml-sort --check pyproject.toml
 
 # Check lint with all linters.
-lint: mypy ruff toml-sort
+# Prioritize fixing isort, then black, otherwise you'll get weird and unfixable black errors.
+# lint: mypy ruff
+lint: mypy ruff isort black toml-sort
 
 # Run pre-commit with autofix against all files.
 pre-commit:
 	pre-commit run --all-files
 
+########################################################################################
+# Auto Lint
+########################################################################################
+
+# Auto lint with black.
+auto-black:
+	$(PIPRUN) python -m black . --extend-exclude test/scripts --extend-exclude git_ignore_folder -l 120
+
+# Auto lint with isort.
+auto-isort:
+	$(PIPRUN) python -m isort . -s git_ignore_folder -s test/scripts
+
+# Auto lint with toml-sort.
+auto-toml-sort:
+	$(PIPRUN) toml-sort pyproject.toml
+
+# Auto lint with all linters.
+auto-lint: auto-isort auto-black auto-toml-sort
+
 ########################################################################################
 # Test
 ########################################################################################
@@ -116,10 +142,21 @@ test-run:
 	$(PIPRUN) python -m coverage run --concurrency=multiprocessing -m pytest --ignore test/scripts
 	$(PIPRUN) python -m coverage combine
 
+test-run-offline:
+	# some test that does not require api calling
+	$(PIPRUN) python -m coverage erase
+	$(PIPRUN) python -m coverage run --concurrency=multiprocessing -m pytest -m "offline" --ignore test/scripts
+	$(PIPRUN) python -m coverage combine
+
 # Generate coverage report for terminal and xml.
+# TODO: we may have higher coverage rate if we have more test
 test: test-run
-	$(PIPRUN) python -m coverage report --fail-under 80
-	$(PIPRUN) python -m coverage xml --fail-under 80
+	$(PIPRUN) python -m coverage report --fail-under 20  # 80
+	$(PIPRUN) python -m coverage xml --fail-under 20  # 80
+
+test-offline: test-run-offline
+	$(PIPRUN) python -m coverage report --fail-under 20  # 80
+	$(PIPRUN) python -m coverage xml --fail-under 20  # 80
 
 ########################################################################################
 # Package
@@ -144,13 +181,19 @@ docs-autobuild:
 		--watch rdagent
 
 # Generate changelog from git commits.
+# The -c and -s arguments should match
+# If -c uses Basic (default, inherits from base class), -s optional argument: # If -c uses conventional (inherits from base class), -s optional parameter: add,fix,change,remove,merge,doc
+# If -c uses conventional (inherits from base class), -s is optional: build,chore,ci,deps,doc,docs,feat,fix,perf,ref,refactor,revert,style,test,tests
+# If -c uses angular (inherits from conventional), -s optional argument: build,chore,ci,deps,doc,docs,feat,fix,perf,ref,refactor,revert,style,test,tests
 # NOTE(xuan.hu): Need to be run before document generation to take effect.
+# $(PIPRUN) git-changelog -ETrio $(CHANGELOG_PATH) -c conventional -s build,chore,ci,docs,feat,fix,perf,refactor,revert,style,test
 changelog:
 	@if wget -q --spider $(CHANGELOG_URL); then \
 		echo "Existing Changelog found at '$(CHANGELOG_URL)', download for incremental generation."; \
 		wget -q -O $(CHANGELOG_PATH) $(CHANGELOG_URL); \
 	fi
-	$(PIPRUN) git-changelog -ETrio $(CHANGELOG_PATH) -c conventional -s build,chore,ci,docs,feat,fix,perf,refactor,revert,style,test
+	$(PIPRUN) LATEST_TAG=$$(git tag --sort=-creatordate | head -n 1); \
+	git-changelog --bump $$LATEST_TAG -Tio docs/changelog.md -c conventional -s build,chore,ci,deps,doc,docs,feat,fix,perf,ref,refactor,revert,style,test,tests
 
 # Generate release notes from changelog.
 release-notes:
@@ -158,7 +201,7 @@ release-notes:
 
 # Build documentation only from rdagent.
 docs-gen:
-	$(PIPRUN) python -m sphinx.cmd.build docs $(PUBLIC_DIR)
+	$(PIPRUN) python -m sphinx.cmd.build -W docs $(PUBLIC_DIR)
 
 # Generate mypy reports.
 docs-mypy: docs-gen

README.md

@@ -1,64 +1,422 @@
-# Project
+<h4 align="center">
+  <img src="docs/_static/logo.png" alt="RA-Agent logo" style="width:70%; ">
+  
+  <a href="https://rdagent.azurewebsites.net" target="_blank">🖥️ Live Demo</a> |
+  <a href="https://rdagent.azurewebsites.net/factor_loop" target="_blank">🎥 Demo Video</a> <a href="https://www.youtube.com/watch?v=JJ4JYO3HscM&list=PLALmKB0_N3_i52fhUmPQiL4jsO354uopR" target="_blank">▶️YouTube</a>   |
+  <a href="https://rdagent.readthedocs.io/en/latest/index.html" target="_blank">📖 Documentation</a> |
+  <a href="https://aka.ms/RD-Agent-Tech-Report" target="_blank">📄 Tech Report</a> |
+  <a href="#-paperwork-list"> 📃 Papers </a>
+</h3>
 
-> This repo has been populated by an initial template to help get you started. Please
-> make sure to update the content to build a great experience for community-building.
 
-As the maintainer of this project, please make a few updates:
+[![CI](https://github.com/microsoft/RD-Agent/actions/workflows/ci.yml/badge.svg)](https://github.com/microsoft/RD-Agent/actions/workflows/ci.yml)
+[![CodeQL](https://github.com/microsoft/RD-Agent/actions/workflows/github-code-scanning/codeql/badge.svg)](https://github.com/microsoft/RD-Agent/actions/workflows/github-code-scanning/codeql)
+[![Dependabot Updates](https://github.com/microsoft/RD-Agent/actions/workflows/dependabot/dependabot-updates/badge.svg)](https://github.com/microsoft/RD-Agent/actions/workflows/dependabot/dependabot-updates)
+[![Lint PR Title](https://github.com/microsoft/RD-Agent/actions/workflows/pr.yml/badge.svg)](https://github.com/microsoft/RD-Agent/actions/workflows/pr.yml)
+[![Release.yml](https://github.com/microsoft/RD-Agent/actions/workflows/release.yml/badge.svg)](https://github.com/microsoft/RD-Agent/actions/workflows/release.yml)
+[![Platform](https://img.shields.io/badge/platform-Linux-blue)](https://pypi.org/project/rdagent/#files)
+[![PyPI](https://img.shields.io/pypi/v/rdagent)](https://pypi.org/project/rdagent/)
+[![PyPI - Python Version](https://img.shields.io/pypi/pyversions/rdagent)](https://pypi.org/project/rdagent/)
+[![Release](https://img.shields.io/github/v/release/microsoft/RD-Agent)](https://github.com/microsoft/RD-Agent/releases)
+[![GitHub](https://img.shields.io/github/license/microsoft/RD-Agent)](https://github.com/microsoft/RD-Agent/blob/main/LICENSE)
+[![pre-commit](https://img.shields.io/badge/pre--commit-enabled-brightgreen?logo=pre-commit)](https://github.com/pre-commit/pre-commit)
+[![Checked with mypy](https://www.mypy-lang.org/static/mypy_badge.svg)](http://mypy-lang.org/)
+[![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)
+[![Chat](https://img.shields.io/badge/chat-discord-blue)](https://discord.gg/ybQ97B6Jjy)
+[![Documentation Status](https://readthedocs.org/projects/rdagent/badge/?version=latest)](https://rdagent.readthedocs.io/en/latest/?badge=latest)
+[![Readthedocs Preview](https://github.com/microsoft/RD-Agent/actions/workflows/readthedocs-preview.yml/badge.svg)](https://github.com/microsoft/RD-Agent/actions/workflows/readthedocs-preview.yml) <!-- this badge is too long, please place it in the last one to make it pretty --> 
+[![arXiv](https://img.shields.io/badge/arXiv-2505.14738-00ff00.svg)](https://arxiv.org/abs/2505.14738)
 
-- Improving this README.MD file to provide a great experience
-- Updating SUPPORT.MD with content about this project's support experience
-- Understanding the security reporting process in SECURITY.MD
-- Remove this section from the README
 
-## Configuration:
 
-You can manually source the `.env` file in your shell before running the Python script:
-Most of the workflow are controlled by the environment variables.
-```sh
-# Export each variable in the .env file; Please note that it is different from `source .env` without export
-export $(grep -v '^#' .env | xargs)
-# Run the Python script
-python your_script.py
+# 🏆 The Best Machine Learning Engineering Agent!
+
+[MLE-bench](https://github.com/openai/mle-bench) is a comprehensive benchmark evaluating the performance of AI agents on machine learning engineering tasks. Utilizing datasets from 75 Kaggle competitions, MLE-bench provides robust assessments of AI systems' capabilities in real-world ML engineering scenarios.
+
+R&D-Agent currently leads as the top-performing machine learning engineering agent on MLE-bench:
+
+| Agent | Low == Lite (%) | Medium (%) | High (%) | All (%) |
+|---------|--------|-----------|---------|----------|
+| R&D-Agent o1-preview | 48.18 ± 2.49 | 8.95 ± 2.36 | 18.67 ± 2.98 | 22.4 ± 1.1 |
+| R&D-Agent o3(R)+GPT-4.1(D) | 51.52 ± 6.21 | 7.89 ± 3.33 | 16.67 ± 3.65 | 22.45 ± 2.45 |
+| AIDE o1-preview | 34.3 ± 2.4 | 8.8 ± 1.1 | 10.0 ± 1.9 | 16.9 ± 1.1 |
+
+**Notes:**
+- **O3(R)+GPT-4.1(D)**: This version is designed to both reduce average time per loop and leverage a cost-effective combination of backend LLMs by seamlessly integrating Research Agent (o3) with Development Agent (GPT-4.1).
+- **AIDE o1-preview**: Represents the previously best public result on MLE-bench as reported in the original MLE-bench paper.
+- Average and standard deviation results for R&D-Agent o1-preview is based on a independent of 5 seeds and for R&D-Agent o3(R)+GPT-4.1(D) is based on 6 seeds.
+- According to MLE-Bench, the 75 competitions are categorized into three levels of complexity: **Low==Lite** if we estimate that an experienced ML engineer can produce a sensible solution in under 2 hours, excluding the time taken to train any models; **Medium** if it takes between 2 and 10 hours; and **High** if it takes more than 10 hours.
+
+You can inspect the detailed runs of the above results online.
+- [R&D-Agent o1-preview detailed runs](https://aka.ms/RD-Agent_MLE-Bench_O1-preview)
+- [R&D-Agent o3(R)+GPT-4.1(D) detailed runs](https://aka.ms/RD-Agent_MLE-Bench_O3_GPT41)
+
+For running R&D-Agent on MLE-bench, refer to **[MLE-bench Guide: Running ML Engineering via MLE-bench](https://rdagent.readthedocs.io/en/latest/scens/data_science.html)**
+
+# 🥇 The First Data-Centric Quant Multi-Agent Framework!
+
+R&D-Agent for Quantitative Finance, in short **RD-Agent(Q)**, is the first data-centric, multi-agent framework designed to automate the full-stack research and development of quantitative strategies via coordinated factor-model co-optimization.
+
+![image](https://github.com/user-attachments/assets/3198bc10-47ba-4ee0-8a8e-46d5ce44f45d)
+
+Extensive experiments in real stock markets show that, at a cost under $10, RD-Agent(Q) achieves approximately 2× higher ARR than benchmark factor libraries while using over 70% fewer factors. It also surpasses state-of-the-art deep time-series models under smaller resource budgets. Its alternating factor–model optimization further delivers excellent trade-off between predictive accuracy and strategy robustness.
+
+You can learn more details about **RD-Agent(Q)** through the [paper](https://arxiv.org/abs/2505.15155) and reproduce it through the [documentation](https://rdagent.readthedocs.io/en/latest/scens/quant_agent_fin.html).
+
+# 📰 News
+| 🗞️ News        | 📝 Description                 |
+| --            | ------      |
+| [Technical Report Release](#overall-technical-report) | Overall framework description and results on MLE-bench | 
+| [R&D-Agent-Quant Release](#deep-application-in-diverse-scenarios) | Apply R&D-Agent to quant trading | 
+| MLE-Bench Results Released | R&D-Agent currently leads as the [top-performing machine learning engineering agent](#-the-best-machine-learning-engineering-agent) on MLE-bench |
+| Support LiteLLM Backend | We now fully support **[LiteLLM](https://github.com/BerriAI/litellm)** as a backend for integration with multiple LLM providers. |
+| General Data Science Agent | [Data Science Agent](https://rdagent.readthedocs.io/en/latest/scens/data_science.html) |
+| Kaggle Scenario release | We release **[Kaggle Agent](https://rdagent.readthedocs.io/en/latest/scens/data_science.html)**, try the new features!                  |
+| Official WeChat group release  | We created a WeChat group, welcome to join! (🗪[QR Code](https://github.com/microsoft/RD-Agent/issues/880)) |
+| Official Discord release  | We launch our first chatting channel in Discord (🗪[![Chat](https://img.shields.io/badge/chat-discord-blue)](https://discord.gg/ybQ97B6Jjy)) |
+| First release | **R&D-Agent** is released on GitHub |
+
+
+
+# Data Science Agent Preview
+Check out our demo video showcasing the current progress of our Data Science Agent under development:
+
+https://github.com/user-attachments/assets/3eccbecb-34a4-4c81-bce4-d3f8862f7305
+
+# 🌟 Introduction
+<div align="center">
+      <img src="docs/_static/scen.png" alt="Our focused scenario" style="width:80%; ">
+</div>
+
+R&D-Agent aims to automate the most critical and valuable aspects of the industrial R&D process, and we begin with focusing on the data-driven scenarios to streamline the development of models and data. 
+Methodologically, we have identified a framework with two key components: 'R' for proposing new ideas and 'D' for implementing them.
+We believe that the automatic evolution of R&D will lead to solutions of significant industrial value.
+
+
+<!-- Tag Cloud -->
+R&D is a very general scenario. The advent of R&D-Agent can be your
+- 💰 **Automatic Quant Factory** ([🎥Demo Video](https://rdagent.azurewebsites.net/factor_loop)|[▶️YouTube](https://www.youtube.com/watch?v=X4DK2QZKaKY&t=6s))
+- 🤖 **Data Mining Agent:** Iteratively proposing data & models ([🎥Demo Video 1](https://rdagent.azurewebsites.net/model_loop)|[▶️YouTube](https://www.youtube.com/watch?v=dm0dWL49Bc0&t=104s)) ([🎥Demo Video 2](https://rdagent.azurewebsites.net/dmm)|[▶️YouTube](https://www.youtube.com/watch?v=VIaSTZuoZg4))  and implementing them by gaining knowledge from data.
+- 🦾 **Research Copilot:** Auto read research papers ([🎥Demo Video](https://rdagent.azurewebsites.net/report_model)|[▶️YouTube](https://www.youtube.com/watch?v=BiA2SfdKQ7o)) / financial reports ([🎥Demo Video](https://rdagent.azurewebsites.net/report_factor)|[▶️YouTube](https://www.youtube.com/watch?v=ECLTXVcSx-c)) and implement model structures or building datasets.
+- 🤖 **Kaggle Agent:** Auto Model Tuning and Feature Engineering([🎥Demo Video Coming Soon...]()) and implementing them to achieve more in competitions.
+- ...
+
+You can click the links above to view the demo. We're continuously adding more methods and scenarios to the project to enhance your R&D processes and boost productivity. 
+
+Additionally, you can take a closer look at the examples in our **[🖥️ Live Demo](https://rdagent.azurewebsites.net/)**.
+
+<div align="center">
+    <a href="https://rdagent.azurewebsites.net/" target="_blank">
+        <img src="docs/_static/demo.png" alt="Watch the demo" width="80%">
+    </a>
+</div>
+
+
+# ⚡ Quick start
+
+You can try above demos by running the following command:
+
+### 🐳 Docker installation.
+Users must ensure Docker is installed before attempting most scenarios. Please refer to the [official 🐳Docker page](https://docs.docker.com/engine/install/) for installation instructions.
+Ensure the current user can run Docker commands **without using sudo**. You can verify this by executing `docker run hello-world`.
+
+### 🐍 Create a Conda Environment
+- Create a new conda environment with Python (3.10 and 3.11 are well-tested in our CI):
+  ```sh
+  conda create -n rdagent python=3.10
+  ```
+- Activate the environment:
+  ```sh
+  conda activate rdagent
+  ```
+
+### 🛠️ Install the R&D-Agent
+- You can directly install the R&D-Agent package from PyPI:
+  ```sh
+  pip install rdagent
+  ```
+
+### 💊 Health check
+- rdagent provides a health check that currently checks two things.
+  - whether the docker installation was successful.
+  - whether the default port used by the [rdagent ui](https://github.com/microsoft/RD-Agent?tab=readme-ov-file#%EF%B8%8F-monitor-the-application-results) is occupied.
+  ```sh
+  rdagent health_check
+  ```
+
+
+### ⚙️ Configuration
+- The demos requires following ability:
+  - ChatCompletion
+  - json_mode
+  - embedding query
+
+  You can set your Chat Model and Embedding Model in the following ways:
+
+- **Using LiteLLM (Default)**: We now support LiteLLM as a backend for integration with multiple LLM providers. You can configure in two ways:
+
+  **Option 1: Unified API base for both models**
+  ```bash
+  cat << EOF  > .env
+  # Set to any model supported by LiteLLM.
+  CHAT_MODEL=gpt-4o 
+  EMBEDDING_MODEL=text-embedding-3-small
+  # Configure unified API base
+  OPENAI_API_BASE=<your_unified_api_base>
+  OPENAI_API_KEY=<replace_with_your_openai_api_key>
+  ```
+
+  **Option 2: Separate API bases for Chat and Embedding models**
+  ```bash
+  cat << EOF  > .env
+  # Set to any model supported by LiteLLM.
+  # Configure separate API bases for chat and embedding
+  
+  # CHAT MODEL:
+  CHAT_MODEL=gpt-4o 
+  OPENAI_API_BASE=<your_chat_api_base>
+  OPENAI_API_KEY=<replace_with_your_openai_api_key>
+
+  # EMBEDDING MODEL:
+  # TAKE siliconflow as an example, you can use other providers.
+  # Note: embedding requires litellm_proxy prefix
+  EMBEDDING_MODEL=litellm_proxy/BAAI/bge-large-en-v1.5
+  LITELLM_PROXY_API_KEY=<replace_with_your_siliconflow_api_key>
+  LITELLM_PROXY_API_BASE=https://api.siliconflow.cn/v1
+  ```
+
+  Notice: If you are using reasoning models that include thought processes in their responses (such as \<think> tags), you need to set the following environment variable:
+  ```bash
+  REASONING_THINK_RM=True
+  ```
+
+- You can also use a deprecated backend if you only use `OpenAI API` or `Azure OpenAI` directly. For this deprecated setting and more configuration information, please refer to the [documentation](https://rdagent.readthedocs.io/en/latest/installation_and_configuration.html).
+
+### 🚀 Run the Application
+
+The **[🖥️ Live Demo](https://rdagent.azurewebsites.net/)** is implemented by the following commands(each item represents one demo, you can select the one you prefer):
+
+- Run the **Automated Quantitative Trading & Iterative Factors Model Joint Evolution**:  [Qlib](http://github.com/microsoft/qlib) self-loop factor & model proposal and implementation application
+  ```sh
+  rdagent fin_quant
+  ```
+
+- Run the **Automated Quantitative Trading & Iterative Factors Evolution**:  [Qlib](http://github.com/microsoft/qlib) self-loop factor proposal and implementation application
+  ```sh
+  rdagent fin_factor
+  ```
+
+- Run the **Automated Quantitative Trading & Iterative Model Evolution**: [Qlib](http://github.com/microsoft/qlib) self-loop model proposal and implementation application
+  ```sh
+  rdagent fin_model
+  ```
+
+- Run the **Automated Quantitative Trading & Factors Extraction from Financial Reports**:  Run the [Qlib](http://github.com/microsoft/qlib) factor extraction and implementation application based on financial reports
+  ```sh
+  # 1. Generally, you can run this scenario using the following command:
+  rdagent fin_factor_report --report_folder=<Your financial reports folder path>
+
+  # 2. Specifically, you need to prepare some financial reports first. You can follow this concrete example:
+  wget https://github.com/SunsetWolf/rdagent_resource/releases/download/reports/all_reports.zip
+  unzip all_reports.zip -d git_ignore_folder/reports
+  rdagent fin_factor_report --report_folder=git_ignore_folder/reports
+  ```
+
+- Run the **Automated Model Research & Development Copilot**: model extraction and implementation application
+  ```sh
+  # 1. Generally, you can run your own papers/reports with the following command:
+  rdagent general_model <Your paper URL>
+
+  # 2. Specifically, you can do it like this. For more details and additional paper examples, use `rdagent general_model -h`:
+  rdagent general_model  "https://arxiv.org/pdf/2210.09789"
+  ```
+
+- Run the **Automated Kaggle Model Tuning & Feature Engineering**:  self-loop model proposal and feature engineering implementation application <br />
+  > Using **sf-crime** *(San Francisco Crime Classification)* as an example. <br />
+  > 1. Register and login on the [Kaggle](https://www.kaggle.com/) website. <br />
+  > 2. Configuring the Kaggle API. <br />
+  > (1) Click on the avatar (usually in the top right corner of the page) -> `Settings` -> `Create New Token`, A file called `kaggle.json` will be downloaded. <br />
+  > (2) Move `kaggle.json` to `~/.config/kaggle/` <br />
+  > (3) Modify the permissions of the kaggle.json file. Reference command: `chmod 600 ~/.config/kaggle/kaggle.json` <br />
+  > 3. Join the competition: Click `Join the competition` -> `I Understand and Accept` at the bottom of the [competition details page](https://www.kaggle.com/competitions/sf-crime/data).
+  ```bash
+  # Generally, you can run the Kaggle competition program with the following command:
+  rdagent data_science --competition <your competition name>
+
+  # Specifically, you need to create a folder for storing competition files (e.g., competition description file, competition datasets, etc.), and configure the path to the folder in your environment. In addition, you need to use chromedriver when you download the competition descriptors, which you can follow for this specific example:
+  
+  # 1. Install chromedriver.
+
+  # 2. Add the competition description file path to the `.env` file.
+  mkdir -p ./git_ignore_folder/kaggle_data
+  dotenv set DS_LOCAL_DATA_PATH "$(pwd)/git_ignore_folder/kaggle_data"
+  dotenv set DS_IF_USING_MLE_DATA True
+
+  # 3. run the application
+  rdagent data_science --competition sf-crime
+  ```
+
+### 🖥️ Monitor the Application Results
+- You can run the following command for our demo program to see the run logs.
+
+  ```sh
+  rdagent ui --port 19899 --log_dir <your log folder like "log/">
+  ```
+
+  **Note:** Although port 19899 is not commonly used, but before you run this demo, you need to check if port 19899 is occupied. If it is, please change it to another port that is not occupied.
+
+  You can check if a port is occupied by running the following command.
+
+  ```sh
+  rdagent health_check
+  ```
+
+# 🏭 Scenarios
+
+We have applied R&D-Agent to multiple valuable data-driven industrial scenarios.
+
+
+## 🎯 Goal: Agent for Data-driven R&D
+
+In this project, we are aiming to build an Agent to automate Data-Driven R\&D that can
++ 📄 Read real-world material (reports, papers, etc.) and **extract** key formulas, descriptions of interested **features** and **models**, which are the key components of data-driven R&D .
++ 🛠️ **Implement** the extracted formulas (e.g., features, factors, and models) in runnable codes.
+   + Due to the limited ability of LLM in implementing at once, build an evolving process for the agent to improve performance by learning from feedback and knowledge.
++ 💡 Propose **new ideas** based on current knowledge and observations.
+
+<!-- ![Data-Centric R&D Overview](docs/_static/overview.png) -->
+
+## 📈 Scenarios/Demos
+
+In the two key areas of data-driven scenarios, model implementation and data building, our system aims to serve two main roles: 🦾Copilot and 🤖Agent. 
+- The 🦾Copilot follows human instructions to automate repetitive tasks. 
+- The 🤖Agent, being more autonomous, actively proposes ideas for better results in the future.
+
+The supported scenarios are listed below:
+
+| Scenario/Target | Model Implementation                   | Data Building                                                                      |
+| --              | --                                     | --                                                                                 |
+| **💹 Finance**      | 🤖 [Iteratively Proposing Ideas & Evolving](https://rdagent.azurewebsites.net/model_loop)[▶️YouTube](https://www.youtube.com/watch?v=dm0dWL49Bc0&t=104s) |  🤖 [Iteratively Proposing Ideas & Evolving](https://rdagent.azurewebsites.net/factor_loop) [▶️YouTube](https://www.youtube.com/watch?v=X4DK2QZKaKY&t=6s) <br/>   🦾 [Auto reports reading & implementation](https://rdagent.azurewebsites.net/report_factor)[▶️YouTube](https://www.youtube.com/watch?v=ECLTXVcSx-c)  |
+| **🩺 Medical**      | 🤖 [Iteratively Proposing Ideas & Evolving](https://rdagent.azurewebsites.net/dmm)[▶️YouTube](https://www.youtube.com/watch?v=VIaSTZuoZg4) | -                                                                                  |
+| **🏭 General**      | 🦾 [Auto paper reading & implementation](https://rdagent.azurewebsites.net/report_model)[▶️YouTube](https://www.youtube.com/watch?v=BiA2SfdKQ7o) <br/> 🤖 Auto Kaggle Model Tuning   | 🤖Auto Kaggle feature Engineering |
+
+- **[RoadMap](https://rdagent.readthedocs.io/en/latest/scens/data_science.html#roadmap)**: Currently, we are working hard to add new features to the Kaggle scenario.
+
+Different scenarios vary in entrance and configuration. Please check the detailed setup tutorial in the scenarios documents.
+
+Here is a gallery of [successful explorations](https://github.com/SunsetWolf/rdagent_resource/releases/download/demo_traces/demo_traces.zip) (5 traces showed in **[🖥️ Live Demo](https://rdagent.azurewebsites.net/)**). You can download and view the execution trace using [this command](https://github.com/microsoft/RD-Agent?tab=readme-ov-file#%EF%B8%8F-monitor-the-application-results) from the documentation.
+
+Please refer to **[📖readthedocs_scen](https://rdagent.readthedocs.io/en/latest/scens/catalog.html)** for more details of the scenarios.
+
+# ⚙️ Framework
+
+<div align="center">
+    <img src="docs/_static/Framework-RDAgent.png" alt="Framework-RDAgent" width="85%">
+</div>
+
+
+Automating the R&D process in data science is a highly valuable yet underexplored area in industry. We propose a framework to push the boundaries of this important research field.
+
+The research questions within this framework can be divided into three main categories:
+| Research Area | Paper/Work List |
+|--------------------|-----------------|
+| **Benchmark the R&D abilities** | [Benchmark](#benchmark) |
+| **Idea proposal:** Explore new ideas or refine existing ones | [Research](#research) |
+| **Ability to realize ideas:** Implement and execute ideas | [Development](#development) |
+
+We believe that the key to delivering high-quality solutions lies in the ability to evolve R&D capabilities. Agents should learn like human experts, continuously improving their R&D skills.
+
+More documents can be found in the **[📖 readthedocs](https://rdagent.readthedocs.io/)**.
+
+# 📃 Paper/Work list
+
+## Overall Technical Report
+- [R&D-Agent: Automating Data-Driven AI Solution Building Through LLM-Powered Automated Research, Development, and Evolution](https://arxiv.org/abs/2505.14738)
+```BibTeX
+@misc{yang2024rdagent,
+    title={R\&D-Agent: Automating Data-Driven AI Solution Building Through LLM-Powered Automated Research, Development, and Evolution},
+    author={Xu Yang and Xiao Yang and Shikai Fang and Bowen Xian and Yuante Li and Jian Wang and Minrui Xu and Haoran Pan and Xinpeng Hong and Weiqing Liu and Yelong Shen and Weizhu Chen and Jiang Bian},
+    year={2025},
+    eprint={2505.14738},
+    archivePrefix={arXiv},
+    primaryClass={cs.AI},
+    url={https://arxiv.org/abs/2505.14738}
+}
 ```
+![image](https://github.com/user-attachments/assets/28b0488d-a546-4fef-8dc5-563ed64a9b4d)
 
-## Naming convention
+## 📊 Benchmark
+- [Towards Data-Centric Automatic R&D](https://arxiv.org/abs/2404.11276)
+```BibTeX
+@misc{chen2024datacentric,
+    title={Towards Data-Centric Automatic R&D},
+    author={Haotian Chen and Xinjie Shen and Zeqi Ye and Wenjun Feng and Haoxue Wang and Xiao Yang and Xu Yang and Weiqing Liu and Jiang Bian},
+    year={2024},
+    eprint={2404.11276},
+    archivePrefix={arXiv},
+    primaryClass={cs.AI}
+}
+```
+![image](https://github.com/user-attachments/assets/494f55d3-de9e-4e73-ba3d-a787e8f9e841)
 
-### File naming convention
+## 🔍 Research
 
-| Name      | Description       |
-| --        | --                |
-| `conf.py` | The configuration for the module & app & project  | 
+In a data mining expert's daily research and development process, they propose a hypothesis (e.g., a model structure like RNN can capture patterns in time-series data), design experiments (e.g., finance data contains time-series and we can verify the hypothesis in this scenario), implement the experiment as code (e.g., Pytorch model structure), and then execute the code to get feedback (e.g., metrics, loss curve, etc.). The experts learn from the feedback and improve in the next iteration.
 
-<!-- TODO: renaming files -->
- 
+Based on the principles above, we have established a basic method framework that continuously proposes hypotheses, verifies them, and gets feedback from the real-world practice. This is the first scientific research automation framework that supports linking with real-world verification.
 
-## Contributing
+For more detail, please refer to our **[🖥️ Live Demo page](https://rdagent.azurewebsites.net)**.
 
-### Guidance
+## 🛠️ Development
+
+- [Collaborative Evolving Strategy for Automatic Data-Centric Development](https://arxiv.org/abs/2407.18690)
+```BibTeX
+@misc{yang2024collaborative,
+    title={Collaborative Evolving Strategy for Automatic Data-Centric Development},
+    author={Xu Yang and Haotian Chen and Wenjun Feng and Haoxue Wang and Zeqi Ye and Xinjie Shen and Xiao Yang and Shizhao Sun and Weiqing Liu and Jiang Bian},
+    year={2024},
+    eprint={2407.18690},
+    archivePrefix={arXiv},
+    primaryClass={cs.AI}
+}
+```
+![image](https://github.com/user-attachments/assets/75d9769b-0edd-4caf-9d45-57d1e577054b)
+
+## Deep Application in Diverse Scenarios
+
+- [R&D-Agent-Quant: A Multi-Agent Framework for Data-Centric Factors and Model Joint Optimization](https://arxiv.org/abs/2505.15155)
+```BibTeX
+@misc{li2025rdagentquant,
+    title={R\&D-Agent-Quant: A Multi-Agent Framework for Data-Centric Factors and Model Joint Optimization},
+    author={Yuante Li and Xu Yang and Xiao Yang and Minrui Xu and Xisen Wang and Weiqing Liu and Jiang Bian},
+    year={2025},
+    eprint={2505.15155},
+    archivePrefix={arXiv},
+    primaryClass={cs.AI}
+}
+```
+![image](https://github.com/user-attachments/assets/3186f67a-c2f8-4b6b-8bb9-a9b959c13866)
+
+
+# 🤝 Contributing
+
+We welcome contributions and suggestions to improve R&D-Agent. Please refer to the [Contributing Guide](CONTRIBUTING.md) for more details on how to contribute.
+
+Before submitting a pull request, ensure that your code passes the automatic CI checks.
+
+## 📝 Guidelines
 This project welcomes contributions and suggestions.
-You can find issues in the issues list or simply running `grep -r "TODO:"`.
+Contributing to this project is straightforward and rewarding. Whether it's solving an issue, addressing a bug, enhancing documentation, or even correcting a typo, every contribution is valuable and helps improve R&D-Agent.
 
-Making contributions is not a hard thing. Solving an issue(maybe just answering a question raised in issues list ), fixing/issuing a bug, improving the documents and even fixing a typo are important contributions to RDAgent.
+To get started, you can explore the issues list, or search for `TODO:` comments in the codebase by running the command `grep -r "TODO:"`.
 
+<img src="https://img.shields.io/github/contributors-anon/microsoft/RD-Agent"/>
 
-### Policy
+<a href="https://github.com/microsoft/RD-Agent/graphs/contributors">
+  <img src="https://contrib.rocks/image?repo=microsoft/RD-Agent&max=100&columns=15" />
+</a>
 
-This project welcomes contributions and suggestions.  Most contributions require you to agree to a
-Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us
-the rights to use your contribution. For details, visit https://cla.opensource.microsoft.com.
+Before we released R&D-Agent as an open-source project on GitHub, it was an internal project within our group. Unfortunately, the internal commit history was not preserved when we removed some confidential code. As a result, some contributions from our group members, including Haotian Chen, Wenjun Feng, Haoxue Wang, Zeqi Ye, Xinjie Shen, and Jinhui Li, were not included in the public commits.
 
-When you submit a pull request, a CLA bot will automatically determine whether you need to provide
-a CLA and decorate the PR appropriately (e.g., status check, comment). Simply follow the instructions
-provided by the bot. You will only need to do this once across all repos using our CLA.
-
-This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/).
-For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or
-contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments.
-
-## Trademarks
-
-This project may contain trademarks or logos for projects, products, or services. Authorized use of Microsoft 
-trademarks or logos is subject to and must follow 
-[Microsoft's Trademark & Brand Guidelines](https://www.microsoft.com/en-us/legal/intellectualproperty/trademarks/usage/general).
-Use of Microsoft trademarks or logos in modified versions of this project must not cause confusion or imply Microsoft sponsorship.
-Any use of third-party trademarks or logos are subject to those third-party's policies.
+# ⚖️ Legal disclaimer
+<p style="line-height: 1; font-style: italic;">The RD-agent is provided “as is”, without warranty of any kind, express or implied, including but not limited to the warranties of merchantability, fitness for a particular purpose and noninfringement. The RD-agent is aimed to facilitate research and development process in the financial industry and not ready-to-use for any financial investment or advice. Users shall independently assess and test the risks of the RD-agent in a specific use scenario, ensure the responsible use of AI technology, including but not limited to developing and integrating risk mitigation measures, and comply with all applicable laws and regulations in all applicable jurisdictions. The RD-agent does not provide financial opinions or reflect the opinions of Microsoft, nor is it designed to replace the role of qualified financial professionals in formulating, assessing, and approving finance products. The inputs and outputs of the RD-agent belong to the users and users shall assume all liability under any theory of liability, whether in contract, torts, regulatory, negligence, products liability, or otherwise, associated with use of the RD-agent and any inputs and outputs thereof.</p>

TODO.md

@@ -0,0 +1,10 @@
+We encourage to set the TODOs in code. But some TODOs are more global.
+So we place it here.
+
+
+- [ ] Aligning the naming of files in components & scenarios.
+  - We would like to have the same logic for naming convention in components(reusable components for all scenarios) and scenarios (componets for specific scenario).
+  - But now we have following mismatch
+    - `coder` in `components` & `developer` in `components`
+- [ ] The name of the folders mismatch with the content in them.
+  - Why are scenarios in experiments?

constraints/3.10.txt

@@ -1,153 +1,8 @@
-aiohttp==3.9.1
-aiosignal==1.3.1
-alabaster==0.7.13
-annotated-types==0.6.0
-anyio==4.2.0
-appdirs==1.4.4
-async-timeout==4.0.3
-attrs==23.2.0
-autodoc-pydantic==2.0.1
-azure-ai-formrecognizer==3.3.2
-azure-common==1.1.28
-azure-core==1.29.6
-Babel==2.14.0
-beautifulsoup4==4.12.2
-black==23.12.1
-build==1.0.3
-certifi==2023.11.17
-cffi==1.16.0
-charset-normalizer==3.3.2
-click==8.1.7
-colorama==0.4.6
-coverage==7.4.0
-cryptography==41.0.7
-Cython==3.0.7
-dataclasses-json==0.6.3
-distro==1.9.0
-docutils==0.20.1
-exceptiongroup==1.2.0
-filelock==3.13.1
-fire==0.5.0
-frozenlist==1.4.1
-fsspec==2023.12.2
-furo==2023.9.10
-fuzzywuzzy==0.18.0
-git-changelog==2.4.0
-greenlet==3.0.3
-h11==0.14.0
-httpcore==1.0.2
-httpx==0.26.0
-idna==3.6
-imagesize==1.4.1
-importlib-metadata==7.0.1
-iniconfig==2.0.0
-isodate==0.6.1
-isort==5.13.2
-jaraco.classes==3.3.0
-jeepney==0.8.0
-Jinja2==3.1.2
-jsonpatch==1.33
-jsonpointer==2.4
-keyring==24.3.0
-langchain==0.0.353
-langchain-community==0.0.7
-langchain-core==0.1.4
-langsmith==0.0.75
-livereload==2.6.3
-loguru==0.7.2
-lxml==5.0.0
-markdown-it-py==3.0.0
-MarkupSafe==2.1.3
-marshmallow==3.20.1
-mdit-py-plugins==0.4.0
-mdurl==0.1.2
-more-itertools==10.1.0
-mpmath==1.3.0
-msrest==0.7.1
-multidict==6.0.4
-mypy==1.10.0
-mypy-extensions==1.0.0
-myst-parser==2.0.0
-networkx==3.2.1
-nh3==0.2.15
-numpy==1.26.2
-nvidia-cublas-cu12==12.1.3.1
-nvidia-cuda-cupti-cu12==12.1.105
-nvidia-cuda-nvrtc-cu12==12.1.105
-nvidia-cuda-runtime-cu12==12.1.105
-nvidia-cudnn-cu12==8.9.2.26
-nvidia-cufft-cu12==11.0.2.54
-nvidia-curand-cu12==10.3.2.106
-nvidia-cusolver-cu12==11.4.5.107
-nvidia-cusparse-cu12==12.1.0.106
-nvidia-nccl-cu12==2.18.1
-nvidia-nvjitlink-cu12==12.3.101
-nvidia-nvtx-cu12==12.1.105
-oauthlib==3.2.2
-openai==1.6.1
-packaging==23.2
-pandas==2.1.4
-pathspec==0.12.1
-pkginfo==1.9.6
-platformdirs==4.1.0
-pluggy==1.3.0
-pycparser==2.21
-pydantic==2.5.3
-pydantic-settings==2.1.0
-pydantic_core==2.14.6
-Pygments==2.17.2
-pypdf==3.17.4
-pyproject_hooks==1.0.0
-pytest==7.4.4
-python-dateutil==2.8.2
-python-dotenv==1.0.0
-pytz==2023.3.post1
-PyYAML==6.0.1
-readme-renderer==42.0
-requests==2.31.0
-requests-oauthlib==1.3.1
-requests-toolbelt==1.0.0
-rfc3986==2.0.0
-rich==13.7.0
-ruamel.yaml==0.18.5
-ruamel.yaml.clib==0.2.8
-ruff==0.4.5
-scipy==1.11.4
-SecretStorage==3.3.3
-semver==3.0.2
-setuptools-scm==8.0.4
-shellingham==1.5.4
-six==1.16.0
-sniffio==1.3.0
-snowballstemmer==2.2.0
-soupsieve==2.5
-Sphinx==7.2.6
-sphinx-autobuild==2021.3.14
-sphinx-basic-ng==1.0.0b2
-sphinx-click==5.1.0
-sphinx-togglebutton==0.3.2
-sphinxcontrib-applehelp==1.0.7
-sphinxcontrib-devhelp==1.0.5
-sphinxcontrib-htmlhelp==2.0.4
-sphinxcontrib-jsmath==1.0.1
-sphinxcontrib-qthelp==1.0.6
-sphinxcontrib-serializinghtml==1.1.9
-SQLAlchemy==2.0.24
-sympy==1.12
-tenacity==8.2.3
-termcolor==2.4.0
-toml-sort==0.23.1
-tomli==2.0.1
-tomlkit==0.12.3
-torch==2.1.2
-tornado==6.4
-tqdm==4.66.1
-triton==2.1.0
-twine==4.0.2
-typer==0.9.0
-typing-inspect==0.9.0
-typing_extensions==4.9.0
-tzdata==2023.4
-urllib3==2.1.0
-yarl==1.9.4
-zipp==3.17.0
+azure-identity==1.17.1
+dill==0.3.9
+pillow==10.4.0
+psutil==6.1.0
+rich==13.9.2
+scipy==1.14.1
+tqdm==4.66.5
+litellm==1.72.4

constraints/3.11.txt

@@ -1,150 +1,8 @@
-aiohttp==3.9.1
-aiosignal==1.3.1
-alabaster==0.7.13
-annotated-types==0.6.0
-anyio==4.2.0
-appdirs==1.4.4
-attrs==23.2.0
-autodoc-pydantic==2.0.1
-azure-ai-formrecognizer==3.3.2
-azure-common==1.1.28
-azure-core==1.29.6
-Babel==2.14.0
-beautifulsoup4==4.12.2
-black==23.12.1
-build==1.0.3
-certifi==2023.11.17
-cffi==1.16.0
-charset-normalizer==3.3.2
-click==8.1.7
-colorama==0.4.6
-coverage==7.4.0
-cryptography==41.0.7
-Cython==3.0.7
-dataclasses-json==0.6.3
-distro==1.9.0
-docutils==0.20.1
-filelock==3.13.1
-fire==0.5.0
-frozenlist==1.4.1
-fsspec==2023.12.2
-furo==2023.9.10
-fuzzywuzzy==0.18.0
-git-changelog==2.4.0
-greenlet==3.0.3
-h11==0.14.0
-httpcore==1.0.2
-httpx==0.26.0
-idna==3.6
-imagesize==1.4.1
-importlib-metadata==7.0.1
-iniconfig==2.0.0
-isodate==0.6.1
-isort==5.13.2
-jaraco.classes==3.3.0
-jeepney==0.8.0
-Jinja2==3.1.2
-jsonpatch==1.33
-jsonpointer==2.4
-keyring==24.3.0
-langchain==0.0.353
-langchain-community==0.0.7
-langchain-core==0.1.4
-langsmith==0.0.75
-livereload==2.6.3
-loguru==0.7.2
-lxml==5.0.0
-markdown-it-py==3.0.0
-MarkupSafe==2.1.3
-marshmallow==3.20.1
-mdit-py-plugins==0.4.0
-mdurl==0.1.2
-more-itertools==10.1.0
-mpmath==1.3.0
-msrest==0.7.1
-multidict==6.0.4
-mypy==1.10.0
-mypy-extensions==1.0.0
-myst-parser==2.0.0
-networkx==3.2.1
-nh3==0.2.15
-numpy==1.26.2
-nvidia-cublas-cu12==12.1.3.1
-nvidia-cuda-cupti-cu12==12.1.105
-nvidia-cuda-nvrtc-cu12==12.1.105
-nvidia-cuda-runtime-cu12==12.1.105
-nvidia-cudnn-cu12==8.9.2.26
-nvidia-cufft-cu12==11.0.2.54
-nvidia-curand-cu12==10.3.2.106
-nvidia-cusolver-cu12==11.4.5.107
-nvidia-cusparse-cu12==12.1.0.106
-nvidia-nccl-cu12==2.18.1
-nvidia-nvjitlink-cu12==12.3.101
-nvidia-nvtx-cu12==12.1.105
-oauthlib==3.2.2
-openai==1.6.1
-packaging==23.2
-pandas==2.1.4
-pathspec==0.12.1
-pkginfo==1.9.6
-platformdirs==4.1.0
-pluggy==1.3.0
-pycparser==2.21
-pydantic==2.5.3
-pydantic-settings==2.1.0
-pydantic_core==2.14.6
-Pygments==2.17.2
-pypdf==3.17.4
-pyproject_hooks==1.0.0
-pytest==7.4.4
-python-dateutil==2.8.2
-python-dotenv==1.0.0
-pytz==2023.3.post1
-PyYAML==6.0.1
-readme-renderer==42.0
-requests==2.31.0
-requests-oauthlib==1.3.1
-requests-toolbelt==1.0.0
-rfc3986==2.0.0
-rich==13.7.0
-ruamel.yaml==0.18.5
-ruamel.yaml.clib==0.2.8
-ruff==0.4.5
-scipy==1.11.4
-SecretStorage==3.3.3
-semver==3.0.2
-setuptools-scm==8.0.4
-shellingham==1.5.4
-six==1.16.0
-sniffio==1.3.0
-snowballstemmer==2.2.0
-soupsieve==2.5
-Sphinx==7.2.6
-sphinx-autobuild==2021.3.14
-sphinx-basic-ng==1.0.0b2
-sphinx-click==5.1.0
-sphinx-togglebutton==0.3.2
-sphinxcontrib-applehelp==1.0.7
-sphinxcontrib-devhelp==1.0.5
-sphinxcontrib-htmlhelp==2.0.4
-sphinxcontrib-jsmath==1.0.1
-sphinxcontrib-qthelp==1.0.6
-sphinxcontrib-serializinghtml==1.1.9
-SQLAlchemy==2.0.24
-sympy==1.12
-tenacity==8.2.3
-termcolor==2.4.0
-toml-sort==0.23.1
-tomlkit==0.12.3
-torch==2.1.2
-tornado==6.4
-tqdm==4.66.1
-triton==2.1.0
-twine==4.0.2
-typer==0.9.0
-typing-inspect==0.9.0
-typing_extensions==4.9.0
-tzdata==2023.4
-urllib3==2.1.0
-yarl==1.9.4
-zipp==3.17.0
+azure-identity==1.17.1
+dill==0.3.9
+pillow==10.4.0
+psutil==6.1.0
+rich==13.9.2
+scipy==1.14.1
+tqdm==4.66.5
+litellm==1.72.4

constraints/3.8.txt

@@ -1,154 +0,0 @@
-aiohttp==3.9.1
-aiosignal==1.3.1
-alabaster==0.7.13
-annotated-types==0.6.0
-anyio==4.2.0
-appdirs==1.4.4
-async-timeout==4.0.3
-attrs==23.2.0
-autodoc-pydantic==2.0.1
-azure-ai-formrecognizer==3.3.2
-azure-common==1.1.28
-azure-core==1.29.6
-Babel==2.14.0
-beautifulsoup4==4.12.2
-black==23.12.1
-build==1.0.3
-certifi==2023.11.17
-cffi==1.16.0
-charset-normalizer==3.3.2
-click==8.1.7
-colorama==0.4.6
-coverage==7.4.0
-cryptography==41.0.7
-Cython==3.0.7
-dataclasses-json==0.6.3
-distro==1.9.0
-docutils==0.20.1
-exceptiongroup==1.2.0
-filelock==3.13.1
-fire==0.5.0
-frozenlist==1.4.1
-fsspec==2023.12.2
-furo==2023.9.10
-fuzzywuzzy==0.18.0
-git-changelog==2.4.0
-greenlet==3.0.3
-h11==0.14.0
-httpcore==1.0.2
-httpx==0.26.0
-idna==3.6
-imagesize==1.4.1
-importlib-metadata==7.0.1
-importlib-resources==6.1.1
-iniconfig==2.0.0
-isodate==0.6.1
-isort==5.13.2
-jaraco.classes==3.3.0
-jeepney==0.8.0
-Jinja2==3.1.2
-jsonpatch==1.33
-jsonpointer==2.4
-keyring==24.3.0
-langchain==0.0.353
-langchain-community==0.0.7
-langchain-core==0.1.4
-langsmith==0.0.75
-livereload==2.6.3
-loguru==0.7.2
-lxml==5.0.0
-markdown-it-py==3.0.0
-MarkupSafe==2.1.3
-marshmallow==3.20.1
-mdit-py-plugins==0.4.0
-mdurl==0.1.2
-more-itertools==10.1.0
-mpmath==1.3.0
-msrest==0.7.1
-multidict==6.0.4
-mypy==1.10.0
-mypy-extensions==1.0.0
-myst-parser==2.0.0
-networkx==3.1
-nh3==0.2.15
-numpy==1.24.4
-nvidia-cublas-cu12==12.1.3.1
-nvidia-cuda-cupti-cu12==12.1.105
-nvidia-cuda-nvrtc-cu12==12.1.105
-nvidia-cuda-runtime-cu12==12.1.105
-nvidia-cudnn-cu12==8.9.2.26
-nvidia-cufft-cu12==11.0.2.54
-nvidia-curand-cu12==10.3.2.106
-nvidia-cusolver-cu12==11.4.5.107
-nvidia-cusparse-cu12==12.1.0.106
-nvidia-nccl-cu12==2.18.1
-nvidia-nvjitlink-cu12==12.3.101
-nvidia-nvtx-cu12==12.1.105
-oauthlib==3.2.2
-openai==1.6.1
-packaging==23.2
-pandas==2.0.3
-pathspec==0.12.1
-pkginfo==1.9.6
-platformdirs==4.1.0
-pluggy==1.3.0
-pycparser==2.21
-pydantic==2.5.3
-pydantic-settings==2.1.0
-pydantic_core==2.14.6
-Pygments==2.17.2
-pypdf==3.17.4
-pyproject_hooks==1.0.0
-pytest==7.4.4
-python-dateutil==2.8.2
-python-dotenv==1.0.0
-pytz==2023.3.post1
-PyYAML==6.0.1
-readme-renderer==42.0
-requests==2.31.0
-requests-oauthlib==1.3.1
-requests-toolbelt==1.0.0
-rfc3986==2.0.0
-rich==13.7.0
-ruamel.yaml==0.18.5
-ruamel.yaml.clib==0.2.8
-ruff==0.4.5
-scipy==1.10.1
-SecretStorage==3.3.3
-semver==3.0.2
-setuptools-scm==8.0.4
-shellingham==1.5.4
-six==1.16.0
-sniffio==1.3.0
-snowballstemmer==2.2.0
-soupsieve==2.5
-Sphinx==7.1.2
-sphinx-autobuild==2021.3.14
-sphinx-basic-ng==1.0.0b2
-sphinx-click==5.1.0
-sphinx-togglebutton==0.3.2
-sphinxcontrib-applehelp==1.0.4
-sphinxcontrib-devhelp==1.0.2
-sphinxcontrib-htmlhelp==2.0.1
-sphinxcontrib-jsmath==1.0.1
-sphinxcontrib-qthelp==1.0.3
-sphinxcontrib-serializinghtml==1.1.5
-SQLAlchemy==2.0.24
-sympy==1.12
-tenacity==8.2.3
-termcolor==2.4.0
-toml-sort==0.23.1
-tomli==2.0.1
-tomlkit==0.12.3
-torch==2.1.2
-tornado==6.4
-tqdm==4.66.1
-triton==2.1.0
-twine==4.0.2
-typer==0.9.0
-typing-inspect==0.9.0
-typing_extensions==4.9.0
-tzdata==2023.4
-urllib3==2.1.0
-yarl==1.9.4
-zipp==3.17.0

constraints/3.9.txt

@@ -1,153 +0,0 @@
-aiohttp==3.9.1
-aiosignal==1.3.1
-alabaster==0.7.13
-annotated-types==0.6.0
-anyio==4.2.0
-appdirs==1.4.4
-async-timeout==4.0.3
-attrs==23.2.0
-autodoc-pydantic==2.0.1
-azure-ai-formrecognizer==3.3.2
-azure-common==1.1.28
-azure-core==1.29.6
-Babel==2.14.0
-beautifulsoup4==4.12.2
-black==23.12.1
-build==1.0.3
-certifi==2023.11.17
-cffi==1.16.0
-charset-normalizer==3.3.2
-click==8.1.7
-colorama==0.4.6
-coverage==7.4.0
-cryptography==41.0.7
-Cython==3.0.7
-dataclasses-json==0.6.3
-distro==1.9.0
-docutils==0.20.1
-exceptiongroup==1.2.0
-filelock==3.13.1
-fire==0.5.0
-frozenlist==1.4.1
-fsspec==2023.12.2
-furo==2023.9.10
-fuzzywuzzy==0.18.0
-git-changelog==2.4.0
-greenlet==3.0.3
-h11==0.14.0
-httpcore==1.0.2
-httpx==0.26.0
-idna==3.6
-imagesize==1.4.1
-importlib-metadata==7.0.1
-iniconfig==2.0.0
-isodate==0.6.1
-isort==5.13.2
-jaraco.classes==3.3.0
-jeepney==0.8.0
-Jinja2==3.1.2
-jsonpatch==1.33
-jsonpointer==2.4
-keyring==24.3.0
-langchain==0.0.353
-langchain-community==0.0.7
-langchain-core==0.1.4
-langsmith==0.0.75
-livereload==2.6.3
-loguru==0.7.2
-lxml==5.0.0
-markdown-it-py==3.0.0
-MarkupSafe==2.1.3
-marshmallow==3.20.1
-mdit-py-plugins==0.4.0
-mdurl==0.1.2
-more-itertools==10.1.0
-mpmath==1.3.0
-msrest==0.7.1
-multidict==6.0.4
-mypy==1.10.0
-mypy-extensions==1.0.0
-myst-parser==2.0.0
-networkx==3.2.1
-nh3==0.2.15
-numpy==1.26.2
-nvidia-cublas-cu12==12.1.3.1
-nvidia-cuda-cupti-cu12==12.1.105
-nvidia-cuda-nvrtc-cu12==12.1.105
-nvidia-cuda-runtime-cu12==12.1.105
-nvidia-cudnn-cu12==8.9.2.26
-nvidia-cufft-cu12==11.0.2.54
-nvidia-curand-cu12==10.3.2.106
-nvidia-cusolver-cu12==11.4.5.107
-nvidia-cusparse-cu12==12.1.0.106
-nvidia-nccl-cu12==2.18.1
-nvidia-nvjitlink-cu12==12.3.101
-nvidia-nvtx-cu12==12.1.105
-oauthlib==3.2.2
-openai==1.6.1
-packaging==23.2
-pandas==2.1.4
-pathspec==0.12.1
-pkginfo==1.9.6
-platformdirs==4.1.0
-pluggy==1.3.0
-pycparser==2.21
-pydantic==2.5.3
-pydantic-settings==2.1.0
-pydantic_core==2.14.6
-Pygments==2.17.2
-pypdf==3.17.4
-pyproject_hooks==1.0.0
-pytest==7.4.4
-python-dateutil==2.8.2
-python-dotenv==1.0.0
-pytz==2023.3.post1
-PyYAML==6.0.1
-readme-renderer==42.0
-requests==2.31.0
-requests-oauthlib==1.3.1
-requests-toolbelt==1.0.0
-rfc3986==2.0.0
-rich==13.7.0
-ruamel.yaml==0.18.5
-ruamel.yaml.clib==0.2.8
-ruff==0.4.5
-scipy==1.11.4
-SecretStorage==3.3.3
-semver==3.0.2
-setuptools-scm==8.0.4
-shellingham==1.5.4
-six==1.16.0
-sniffio==1.3.0
-snowballstemmer==2.2.0
-soupsieve==2.5
-Sphinx==7.2.6
-sphinx-autobuild==2021.3.14
-sphinx-basic-ng==1.0.0b2
-sphinx-click==5.1.0
-sphinx-togglebutton==0.3.2
-sphinxcontrib-applehelp==1.0.7
-sphinxcontrib-devhelp==1.0.5
-sphinxcontrib-htmlhelp==2.0.4
-sphinxcontrib-jsmath==1.0.1
-sphinxcontrib-qthelp==1.0.6
-sphinxcontrib-serializinghtml==1.1.9
-SQLAlchemy==2.0.24
-sympy==1.12
-tenacity==8.2.3
-termcolor==2.4.0
-toml-sort==0.23.1
-tomli==2.0.1
-tomlkit==0.12.3
-torch==2.1.2
-tornado==6.4
-tqdm==4.66.1
-triton==2.1.0
-twine==4.0.2
-typer==0.9.0
-typing-inspect==0.9.0
-typing_extensions==4.9.0
-tzdata==2023.4
-urllib3==2.1.0
-yarl==1.9.4
-zipp==3.17.0

docs/Makefile

@@ -0,0 +1,20 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS    ?=
+SPHINXBUILD   ?= sphinx-build
+SOURCEDIR     = .
+BUILDDIR      = build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

docs/_static/RD2bench.json

@@ -0,0 +1,332 @@
+{
+  "alpha053_15": {
+    "description": "Reversal class factor, negative delta of a ratio involving close, low, and high prices over 15 days.",
+    "formulation": "-1 times Deltaleft(frac{(text{close} - text{low}) - (text{high} - text{close})}{text{close} - text{low}}, 15right)",
+    "variables": {
+      "Delta(x, d)": "Change in 'x' over 'd' days.",
+      "text{close}": "Closing price of the stock.",
+      "text{low}": "Lowest price of the stock for the day.",
+      "text{high}": "Highest price of the stock for the day."
+    },
+    "Category": "Volume&Price",
+    "Difficulty": "Easy",
+    "gt_code": "import pandas as pd\ndata_pv = pd.read_hdf('daily_pv.h5')\nnew_df= data_pv.reset_index()\n# Calculate Alpha053\nnew_df['ratio'] =  (new_df['$close'] - new_df['$low'] - (new_df['$high'] - new_df['$close'])) / (new_df['$close'] - new_df['$low'])\n# the change of ratio in new_df over the 15 days\nnew_df['result']=-new_df['ratio'].diff(15)\n# transfer the result to series\nresult=pd.DataFrame(new_df['result']).set_index(data_pv.index)\nresult=result['result']\nresult.to_hdf('result.h5', key='data')"
+  },
+  "liquidity_imbalance": {
+    "description": "liquidity_imbalance=std(minute trading liquidity_imbalance)/mean(minute trading liquidity_imbalance).",
+    "formulation": "liquidity_imbalance = frac{text{std}(text{minute trading liquidity_imbalance})}{text{mean}(text{minute liquidity_imbalance})}",
+    "variables": {
+      "std(minute liquidity_imbalance)": "Standard deviation of trading liquidity_imbalance for each minute of the trading day.",
+      "mean(minute liquidity_imbalance)": "Mean of trading liquidity_imbalance for each minute of the trading day.",
+      "liquidity_imbalance": "(bid_size-ask_size)/(bid_size+ask_size), we use something like bidV for the size"
+    },
+    "Category": "High-Frequency",
+    "Difficulty": "Medium",
+    "gt_code": "import pandas as pd\ndata_hf = pd.read_hdf('high_freq.h5')\nsample_df= data_hf.reset_index()\n# Convert 'datetime' column to datetime and extract date for grouping\nsample_df['date'] = sample_df['datetime'].dt.date\nsample_df['liquidity_imbalance']=(sample_df['bidV']-sample_df['askV'])/(sample_df['bidV']+sample_df['askV'])\n# Group by instrument and date\ngrouped = sample_df.groupby(['date','instrument'])['liquidity_imbalance']\n# Calculate mean and standard deviation of the volume for each group\nstats = grouped.agg(['mean', 'std'])\n# Calculate Z value for each instrument per day\nstats['liquidity_imbalance'] = stats['std'] / stats['mean']\n# Display the calculated Z values\nresult=stats['liquidity_imbalance']\nresult.index.names = ['datetime','instrument']\n# result = result.swaplevel().sort_index()\nresult.to_hdf('result.h5', key='data')"
+  },
+  "liquidity_imbalance_2": {
+    "description": "liquidity_imbalance=std(minute trading liquidity_imbalance)/mean(minute trading liquidity_imbalance).",
+    "formulation": "liquidity_imbalance = frac{text{std}(text{minute trading liquidity_imbalance})}{text{mean}(text{minute liquidity_imbalance})}",
+    "variables": {
+      "std(minute liquidity_imbalance)": "Standard deviation of trading liquidity_imbalance for each minute of the trading day.",
+      "mean(minute liquidity_imbalance)": "Mean of trading liquidity_imbalance for each minute of the trading day.",
+      "liquidity_imbalance": "(bid_size-ask_size)/2*(bid_size+ask_size), we use something like bidV for the size"
+    },
+    "Category": "High-Frequency",
+    "Difficulty": "Medium",
+    "gt_code": "import pandas as pd\ndata_hf = pd.read_hdf('high_freq.h5')\nsample_df= data_hf.reset_index()\n# Convert 'datetime' column to datetime and extract date for grouping\nsample_df['date'] = sample_df['datetime'].dt.date\nsample_df['liquidity_imbalance']=(sample_df['bidV']-sample_df['askV'])/((sample_df['bidV']+sample_df['askV'])*2)\n# Group by instrument and date\ngrouped = sample_df.groupby(['date','instrument'])['liquidity_imbalance']\n# Calculate mean and standard deviation of the volume for each group\nstats = grouped.agg(['mean', 'std'])\n# Calculate Z value for each instrument per day\nstats['liquidity_imbalance'] = stats['std'] / stats['mean']\n# Display the calculated Z values\nresult=stats['liquidity_imbalance']\nresult.index.names = ['datetime','instrument']\n# result = result.swaplevel().sort_index()\nresult.to_hdf('result.h5', key='data')"
+  },
+  "liquidity_imbalance_3": {
+    "description": "liquidity_imbalance=std(minute trading liquidity_imbalance)/mean(minute trading liquidity_imbalance).",
+    "formulation": "liquidity_imbalance = frac{text{std}(text{minute trading liquidity_imbalance})}{text{mean}(text{minute liquidity_imbalance})}",
+    "variables": {
+      "std(minute liquidity_imbalance)": "Standard deviation of trading liquidity_imbalance for each minute of the trading day.",
+      "mean(minute liquidity_imbalance)": "Mean of trading liquidity_imbalance for each minute of the trading day.",
+      "liquidity_imbalance": "(bid_size-ask_size)/3*(bid_size+ask_size), we use something like bidV for the size"
+    },
+    "Category": "High-Frequency",
+    "Difficulty": "Medium",
+    "gt_code": "import pandas as pd\ndata_hf = pd.read_hdf('high_freq.h5')\nsample_df= data_hf.reset_index()\n# Convert 'datetime' column to datetime and extract date for grouping\nsample_df['date'] = sample_df['datetime'].dt.date\nsample_df['liquidity_imbalance']=(sample_df['bidV']-sample_df['askV'])/((sample_df['bidV']+sample_df['askV'])*3)\n# Group by instrument and date\ngrouped = sample_df.groupby(['date','instrument'])['liquidity_imbalance']\n# Calculate mean and standard deviation of the volume for each group\nstats = grouped.agg(['mean', 'std'])\n# Calculate Z value for each instrument per day\nstats['liquidity_imbalance'] = stats['std'] / stats['mean']\n# Display the calculated Z values\nresult=stats['liquidity_imbalance']\nresult.index.names = ['datetime','instrument']\n# result = result.swaplevel().sort_index()\nresult.to_hdf('result.h5', key='data')"
+  },
+  "micro_price": {
+    "description": "micro_price=std(minute trading micro_price)/mean(minute trading micro_price).",
+    "formulation": "micro_price = frac{text{std}(text{minute trading micro_price})}{text{mean}(text{minute micro_price})}",
+    "variables": {
+      "std(minute micro_price)": "Standard deviation of trading micro_price for each minute of the trading day.",
+      "mean(minute micro_price)": "Mean of trading micro_price for each minute of the trading day.",
+      "micro_price": "((df['bid_price'] * df['ask_size']) + (df['ask_price'] * df['bid_size'])) / (df['bid_size'] + df['ask_size'])"
+    },
+    "Category": "High-Frequency",
+    "Difficulty": "Hard",
+    "gt_code": "import pandas as pd\ndata_hf = pd.read_hdf('high_freq.h5')\nsample_df= data_hf.reset_index()\n# Convert 'datetime' column to datetime and extract date for grouping\nsample_df['date'] = sample_df['datetime'].dt.date\nsample_df['micro_price']=(sample_df['bid']*sample_df['askV']+sample_df['ask']*sample_df['bidV'])/(sample_df['bidV']+sample_df['askV'])\n# Group by instrument and date\ngrouped = sample_df.groupby(['date','instrument'])['micro_price']\n# Calculate mean and standard deviation of the volume for each group\nstats = grouped.agg(['mean', 'std'])\n# Calculate Z value for each instrument per day\nstats['micro_price'] = stats['std'] / stats['mean']\n# Display the calculated Z values\nresult=stats['micro_price']\nresult.index.names = ['datetime','instrument']\n# result = result.swaplevel().sort_index()\nresult.to_hdf('result.h5', key='data')"
+  },
+  "micro_price_2": {
+    "description": "micro_price_2=std(minute trading micro_price)/mean(minute trading micro_price).",
+    "formulation": "micro_price_2 = frac{text{std}(text{minute trading micro_price})}{text{mean}(text{minute micro_price})}",
+    "variables": {
+      "std(minute micro_price)": "Standard deviation of trading micro_price for each minute of the trading day.",
+      "mean(minute micro_price)": "Mean of trading micro_price for each minute of the trading day.",
+      "micro_price": "((df['bid_price'] * df['ask_size']) + (df['ask_price'] * df['bid_size'])) / 2*(df['bid_size'] + df['ask_size']), we use something like bidV for the size"
+    },
+    "Category": "High-Frequency",
+    "Difficulty": "Hard",
+    "gt_code": "import pandas as pd\ndata_hf = pd.read_hdf('high_freq.h5')\nsample_df= data_hf.reset_index()\n# Convert 'datetime' column to datetime and extract date for grouping\nsample_df['date'] = sample_df['datetime'].dt.date\nsample_df['micro_price']=(sample_df['bid']*sample_df['askV']+sample_df['ask']*sample_df['bidV'])/((sample_df['bidV']+sample_df['askV'])*2)\n# Group by instrument and date\ngrouped = sample_df.groupby(['date','instrument'])['micro_price']\n# Calculate mean and standard deviation of the volume for each group\nstats = grouped.agg(['mean', 'std'])\n# Calculate Z value for each instrument per day\nstats['micro_price'] = stats['std'] / stats['mean']\n# Display the calculated Z values\nresult=stats['micro_price']\nresult.index.names = ['datetime','instrument']\n# result = result.swaplevel().sort_index()\nresult.to_hdf('result.h5', key='data')"
+  },
+  "micro_price_3": {
+    "description": "micro_price_3=std(minute trading micro_price)/mean(minute trading micro_price).",
+    "formulation": "micro_price_3 = frac{text{std}(text{minute trading micro_price})}{text{mean}(text{minute micro_price})}",
+    "variables": {
+      "std(minute micro_price)": "Standard deviation of trading micro_price for each minute of the trading day.",
+      "mean(minute micro_price)": "Mean of trading micro_price for each minute of the trading day.",
+      "micro_price": "((df['bid_price'] * df['ask_size']) + (df['ask_price'] * df['bid_size'])) / 3*(df['bid_size'] + df['ask_size']), we use something like bidV for the size"
+    },
+    "Category": "High-Frequency",
+    "Difficulty": "Hard",
+    "gt_code": "import pandas as pd\ndata_hf = pd.read_hdf('high_freq.h5')\nsample_df= data_hf.reset_index()\n# Convert 'datetime' column to datetime and extract date for grouping\nsample_df['date'] = sample_df['datetime'].dt.date\nsample_df['micro_price']=(sample_df['bid']*sample_df['askV']+sample_df['ask']*sample_df['bidV'])/((sample_df['bidV']+sample_df['askV'])*3)\n# Group by instrument and date\ngrouped = sample_df.groupby(['date','instrument'])['micro_price']\n# Calculate mean and standard deviation of the volume for each group\nstats = grouped.agg(['mean', 'std'])\n# Calculate Z value for each instrument per day\nstats['micro_price'] = stats['std'] / stats['mean']\n# Display the calculated Z values\nresult=stats['micro_price']\nresult.index.names = ['datetime','instrument']\n# result = result.swaplevel().sort_index()\nresult.to_hdf('result.h5', key='data')"
+  },
+  "mid_price": {
+    "description": "mid_price=std(minute trading mid_price)/mean(minute trading mid_price).",
+    "formulation": "mid_price = frac{text{std}(text{minute trading mid price})}{text{mean}(text{minute mid price})}",
+    "variables": {
+      "std(minute mid_price)": "Standard deviation of trading mid_price for each minute of the trading day.",
+      "mean(minute mid_price)": "Mean of trading mid_price for each minute of the trading day.",
+      "mid_price": "The average of the bid and ask prices."
+    },
+    "Category": "High-Frequency",
+    "Difficulty": "Easy",
+    "gt_code": "import pandas as pd\ndata_hf = pd.read_hdf('high_freq.h5')\nsample_df= data_hf.reset_index()\n# Convert 'datetime' column to datetime and extract date for grouping\nsample_df['date'] = sample_df['datetime'].dt.date\nsample_df['mid_price']=(sample_df['bid']+sample_df['ask'])/2\n# Group by instrument and date\ngrouped = sample_df.groupby(['date','instrument'])['mid_price']\n# Calculate mean and standard deviation of the volume for each group\nstats = grouped.agg(['mean', 'std'])\nstats['mid_price'] = stats['std'] / stats['mean']\nresult=stats['mid_price']\nresult.index.names = ['datetime','instrument']\n# result = result.swaplevel().sort_index()\nresult.to_hdf('result.h5', key='data')"
+  },
+  "mid_price_2": {
+    "description": "mid_price=std(minute trading mid_price)/mean(minute trading mid_price).",
+    "formulation": "mid_price = frac{text{std}(text{minute trading mid price})}{text{mean}(text{minute mid price})}",
+    "variables": {
+      "std(minute mid_price)": "Standard deviation of trading mid_price for each minute of the trading day.",
+      "mean(minute mid_price)": "Mean of trading mid_price for each minute of the trading day.",
+      "mid_price_2": "the average of the bid and ask prices plus the the average of the bid and ask size (bidV and askV)."
+    },
+    "Category": "High-Frequency",
+    "Difficulty": "Easy",
+    "gt_code": "import pandas as pd\ndata_hf = pd.read_hdf('high_freq.h5')\nsample_df= data_hf.reset_index()\n# Convert 'datetime' column to datetime and extract date for grouping\nsample_df['date'] = sample_df['datetime'].dt.date\nsample_df['mid_price']=(sample_df['bid']+sample_df['ask'])/2+(sample_df['bidV']+sample_df['askV'])/2\n# Group by instrument and date\ngrouped = sample_df.groupby(['date','instrument'])['mid_price']\n# Calculate mean and standard deviation of the volume for each group\nstats = grouped.agg(['mean', 'std'])\nstats['mid_price'] = stats['std'] / stats['mean']\nresult=stats['mid_price']\nresult.index.names = ['datetime','instrument']\n# result = result.swaplevel().sort_index()\nresult.to_hdf('result.h5', key='data')"
+  },
+  "mid_price_3": {
+    "description": "mid_price=std(minute trading mid_price)/mean(minute trading mid_price).",
+    "formulation": "mid_price = frac{text{std}(text{minute trading mid price})}{text{mean}(text{minute mid price})}",
+    "variables": {
+      "std(minute mid_price)": "Standard deviation of trading mid_price for each minute of the trading day.",
+      "mean(minute mid_price)": "Mean of trading mid_price for each minute of the trading day.",
+      "mid_price_3": "The coefficient of variation (CV) of the mid-price for each minute of the trading day, calculated as the standard deviation of the mid-price divided by the mean mid-price."
+    },
+    "Category": "High-Frequency",
+    "Difficulty": "Easy",
+    "gt_code": "import pandas as pd\ndata_hf = pd.read_hdf('high_freq.h5')\nsample_df= data_hf.reset_index()\n# Convert 'datetime' column to datetime and extract date for grouping\nsample_df['date'] = sample_df['datetime'].dt.date\nsample_df['mid_price']=(sample_df['bid']+sample_df['ask'])/3\n# Group by instrument and date\ngrouped = sample_df.groupby(['date','instrument'])['mid_price']\n# Calculate mean and standard deviation of the volume for each group\nstats = grouped.agg(['mean', 'std'])\nstats['mid_price'] = stats['std'] / stats['mean']\nresult=stats['mid_price']\nresult.index.names = ['datetime','instrument']\n# result = result.swaplevel().sort_index()\nresult.to_hdf('result.h5', key='data')"
+  },
+  "PB_ROE": {
+    "description": "Constructed using the ranking difference between PB and ROE, with regression versions of PB and ROE replacing original PB and ROE to obtain reconstructed factor values.",
+    "formulation": "text{rank}(PB_t) - rank(ROE_t)",
+    "variables": {
+      "text{rank}(PB_t)": "Ranking of regression version PB on cross-section at time t.",
+      "text{rank}(ROE_t)": "Ranking of regression version single-quarter ROE on cross-section at time t."
+    },
+    "Category": "Fundamentals",
+    "Difficulty": "Easy",
+    "gt_code": "import pandas as pd\ndata_f = pd.read_hdf('daily_f.h5')\ndata = data_f.reset_index()\n# Calculate the rank of PB and ROE\ndata['PB_rank'] = data.groupby('datetime')['B/P'].rank()\ndata['ROE_rank'] = data.groupby('datetime')['ROE'].rank()\n# Calculate the difference between the ranks\ndata['PB_ROE'] = data['PB_rank'] - data['ROE_rank']\n# set the datetime and instrument as index and drop the original index\nresult=pd.DataFrame(data['PB_ROE']).set_index(data_f.index)\n# transfer the result to series\nresult=result['PB_ROE']\nresult.to_hdf('result.h5', key='data')"
+  },
+  "PB_ROE_2": {
+    "description": "Constructed using the ranking difference between PB/2 and ROE, with regression versions of PB and ROE replacing original PB and ROE to obtain reconstructed factor values.",
+    "formulation": "text{rank}(PB_t)/2 - rank(ROE_t)",
+    "variables": {
+      "text{rank}(PB_t)": "Ranking of regression version PB on cross-section at time t.",
+      "text{rank}(ROE_t)": "Ranking of regression version single-quarter ROE on cross-section at time t."
+    },
+    "Category": "Fundamentals",
+    "Difficulty": "Easy",
+    "gt_code": "import pandas as pd\ndata_f = pd.read_hdf('daily_f.h5')\ndata = data_f.reset_index()\n# Calculate the rank of PB and ROE\ndata['PB_rank'] = data.groupby('datetime')['B/P'].rank()\ndata['ROE_rank'] = data.groupby('datetime')['ROE'].rank()\n# Calculate the difference between the ranks\ndata['PB_ROE'] = data['PB_rank']/2 - data['ROE_rank']\n# set the datetime and instrument as index and drop the original index\nresult=pd.DataFrame(data['PB_ROE']).set_index(data_f.index)\n# transfer the result to series\nresult=result['PB_ROE']\nresult.to_hdf('result.h5', key='data')"
+  },
+  "PB_ROE_3": {
+    "description": "Constructed using the ranking difference between PB/3 and ROE, with regression versions of PB and ROE replacing original PB and ROE to obtain reconstructed factor values.",
+    "formulation": "text{rank}(PB_t)/3 - rank(ROE_t)",
+    "variables": {
+      "text{rank}(PB_t)": "Ranking of regression version PB on cross-section at time t.",
+      "text{rank}(ROE_t)": "Ranking of regression version single-quarter ROE on cross-section at time t."
+    },
+    "Category": "Fundamentals",
+    "Difficulty": "Easy",
+    "gt_code": "import pandas as pd\ndata_f = pd.read_hdf('daily_f.h5')\ndata = data_f.reset_index()\n# Calculate the rank of PB and ROE\ndata['PB_rank'] = data.groupby('datetime')['B/P'].rank()\ndata['ROE_rank'] = data.groupby('datetime')['ROE'].rank()\n# Calculate the difference between the ranks\ndata['PB_ROE'] = data['PB_rank']/3 - data['ROE_rank']\n# set the datetime and instrument as index and drop the original index\nresult=pd.DataFrame(data['PB_ROE']).set_index(data_f.index)\n# transfer the result to series\nresult=result['PB_ROE']\nresult.to_hdf('result.h5', key='data')"
+  },
+  "PB_ROE_movement": {
+    "description": "PB_ROE_movement=five day PB_ROE movement indicator(-1 and 1 or 0).",
+    "formulation": "PB_ROE_movement = 5_day_movement(PB_ROE), PB_ROE = text{rank}(PB_t) - rank(ROE_t)",
+    "variables": {
+      "PB_ROE": "the ranking difference between PB and ROE.",
+      "5_day_PB_ROE_movement": "1 if PB_ROE is higher than the PB_ROE 5 days ago, -1 if PB_ROE is lower than the PB_ROE 5 days ago, 0 if PB_ROE is the same as the PB_ROE 5 days ago.",
+      "text{rank}(PB_t)": "Ranking of regression version PB on cross-section at time t.",
+      "text{rank}(ROE_t)": "Ranking of regression version single-quarter ROE on cross-section at time t."
+    },
+    "Category": "Fundamentals",
+    "Difficulty": "Hard",
+    "gt_code": "import pandas as pd\ndata_f = pd.read_hdf('daily_f.h5')\nsample_df = data_f.reset_index()\n# Calculate the rank of PB and ROE\nsample_df['PB_rank'] = sample_df.groupby('datetime')['B/P'].rank()\nsample_df['ROE_rank'] = sample_df.groupby('datetime')['ROE'].rank()\nsample_df['PB_ROE'] = sample_df['PB_rank'] - sample_df['ROE_rank']\n# Group by instrument and date\nsample_df['PB_ROE_movement'] = sample_df['PB_ROE'].diff(periods=5).apply(lambda x: 1 if x > 0 else (-1 if x < 0 else 0))\n#calculate the mid_price_movement ratio for each day\n# set the datetime and instrument as index and drop the original index\nresult=pd.DataFrame(sample_df['PB_ROE_movement']).set_index(data_f.index)\n# transfer the result to series\nresult=result['PB_ROE_movement']\nresult.to_hdf('result.h5', key='data')"
+  },
+  "PB_ROE_movement_10": {
+    "description": "PB_ROE_movement=10 days PB_ROE movement indicator(-1 and 1 or 0).",
+    "formulation": "PB_ROE_movement = 10_day_movement(PB_ROE), PB_ROE = text{rank}(PB_t) - rank(ROE_t)",
+    "variables": {
+      "PB_ROE": "the ranking difference between PB and ROE.",
+      "10_day_PB_ROE_movement": "1 if PB_ROE is higher than the PB_ROE 10 days ago, -1 if PB_ROE is lower than the PB_ROE 10 days ago, 0 if PB_ROE is the same as the PB_ROE 10 days ago.",
+      "text{rank}(PB_t)": "Ranking of regression version PB on cross-section at time t.",
+      "text{rank}(ROE_t)": "Ranking of regression version single-quarter ROE on cross-section at time t."
+    },
+    "Category": "Fundamentals",
+    "Difficulty": "Hard",
+    "gt_code": "import pandas as pd\ndata_f = pd.read_hdf('daily_f.h5')\nsample_df = data_f.reset_index()\n# Calculate the rank of PB and ROE\nsample_df['PB_rank'] = sample_df.groupby('datetime')['B/P'].rank()\nsample_df['ROE_rank'] = sample_df.groupby('datetime')['ROE'].rank()\nsample_df['PB_ROE'] = sample_df['PB_rank'] - sample_df['ROE_rank']\n# Group by instrument and date\nsample_df['PB_ROE_movement'] = sample_df['PB_ROE'].diff(periods=10).apply(lambda x: 1 if x > 0 else (-1 if x < 0 else 0))\n#calculate the mid_price_movement ratio for each day\n# set the datetime and instrument as index and drop the original index\nresult=pd.DataFrame(sample_df['PB_ROE_movement']).set_index(data_f.index)\n# transfer the result to series\nresult=result['PB_ROE_movement']\nresult.to_hdf('result.h5', key='data')"
+  },
+  "PB_ROE_movement_20": {
+    "description": "PB_ROE_movement=20 days PB_ROE movement indicator(-1 and 1 or 0).",
+    "formulation": "PB_ROE_movement = 20_day_movement(PB_ROE), PB_ROE = text{rank}(PB_t) - rank(ROE_t)",
+    "variables": {
+      "PB_ROE": "the ranking difference between PB and ROE.",
+      "20_day_PB_ROE_movement": "1 if PB_ROE is higher than the PB_ROE 20 days ago, -1 if PB_ROE is lower than the PB_ROE 20 days ago, 0 if PB_ROE is the same as the PB_ROE 20 days ago.",
+      "text{rank}(PB_t)": "Ranking of regression version PB on cross-section at time t.",
+      "text{rank}(ROE_t)": "Ranking of regression version single-quarter ROE on cross-section at time t."
+    },
+    "Category": "Fundamentals",
+    "Difficulty": "Hard",
+    "gt_code": "import pandas as pd\ndata_f = pd.read_hdf('daily_f.h5')\nsample_df = data_f.reset_index()\n# Calculate the rank of PB and ROE\nsample_df['PB_rank'] = sample_df.groupby('datetime')['B/P'].rank()\nsample_df['ROE_rank'] = sample_df.groupby('datetime')['ROE'].rank()\nsample_df['PB_ROE'] = sample_df['PB_rank'] - sample_df['ROE_rank']\n# Group by instrument and date\nsample_df['PB_ROE_movement'] = sample_df['PB_ROE'].diff(periods=20).apply(lambda x: 1 if x > 0 else (-1 if x < 0 else 0))\n#calculate the mid_price_movement ratio for each day\n# set the datetime and instrument as index and drop the original index\nresult=pd.DataFrame(sample_df['PB_ROE_movement']).set_index(data_f.index)\n# transfer the result to series\nresult=result['PB_ROE_movement']\nresult.to_hdf('result.h5', key='data')"
+  },
+  "ROE_movement": {
+    "description": "ROE_movement=five day ROE movement indicator(-1 and 1 or 0).",
+    "formulation": "ROE_movement = 5_day_movement(ROE)",
+    "variables": {
+      "ROE": "ROE in fundamental statistics.",
+      "5_day_ROE_movement": "1 if ROE is higher than the ROE 5 days ago, -1 if ROE is lower than the ROE 5 days ago, 0 if ROE is the same as the ROE 5 days ago."
+    },
+    "Category": "Fundamentals",
+    "Difficulty": "Medium",
+    "gt_code": "import pandas as pd\ndata_f = pd.read_hdf('daily_f.h5')\nsample_df = data_f.reset_index()\n# Group by instrument and date\nsample_df['ROE_movement'] = sample_df['ROE'].diff(periods=5).apply(lambda x: 1 if x > 0 else (-1 if x < 0 else 0))\n#calculate the mid_price_movement ratio for each day\n# set the datetime and instrument as index and drop the original index\nresult=pd.DataFrame(sample_df['ROE_movement']).set_index(data_f.index)\n# transfer the result to series\nresult=result['ROE_movement']\nresult.to_hdf('result.h5', key='data')"
+  },
+  "ROE_movement_10": {
+    "description": "ROE_movement_10=ten day ROE movement indicator(-1 and 1 or 0).",
+    "formulation": "ROE_movement = 10_day_movement(ROE)",
+    "variables": {
+      "ROE": "ROE in fundamental statistics.",
+      "10_day_ROE_movement": "1 if ROE is higher than the ROE 10 days ago, -1 if ROE is lower than the ROE 10 days ago, 0 if ROE is the same as the ROE 10 days ago."
+    },
+    "Category": "Fundamentals",
+    "Difficulty": "Medium",
+    "gt_code": "import pandas as pd\ndata_f = pd.read_hdf('daily_f.h5')\nsample_df = data_f.reset_index()\n# Group by instrument and date\nsample_df['ROE_movement'] = sample_df['ROE'].diff(periods=10).apply(lambda x: 1 if x > 0 else (-1 if x < 0 else 0))\n#calculate the mid_price_movement ratio for each day\n# set the datetime and instrument as index and drop the original index\nresult=pd.DataFrame(sample_df['ROE_movement']).set_index(data_f.index)\n# transfer the result to series\nresult=result['ROE_movement']\nresult.to_hdf('result.h5', key='data')"
+  },
+  "ROE_movement_20": {
+    "description": "ROE_movement_20=20 day ROE movement indicator(-1 and 1 or 0).",
+    "formulation": "ROE_movement_20 = 20_day_movement(ROE)",
+    "variables": {
+      "ROE": "ROE in fundamental statistics.",
+      "20_day_ROE_movement": "1 if ROE is higher than the ROE 20 days ago, -1 if ROE is lower than the ROE 20 days ago, 0 if ROE is the same as the ROE 20 days ago."
+    },
+    "Category": "Fundamentals",
+    "Difficulty": "Medium",
+    "gt_code": "import pandas as pd\ndata_f = pd.read_hdf('daily_f.h5')\nsample_df = data_f.reset_index()\n# Group by instrument and date\nsample_df['ROE_movement'] = sample_df['ROE'].diff(periods=20).apply(lambda x: 1 if x > 0 else (-1 if x < 0 else 0))\n#calculate the mid_price_movement ratio for each day\n# set the datetime and instrument as index and drop the original index\nresult=pd.DataFrame(sample_df['ROE_movement']).set_index(data_f.index)\n# transfer the result to series\nresult=result['ROE_movement']\nresult.to_hdf('result.h5', key='data')"
+  },
+  "alpha_pv_diff": {
+    "description": "alpha_pv_diff is defined as the ratio of the difference between close prices 10 days change and open prices 10 days change to the sum of the highest minus lowest prices plus a small constant.",
+    "formulation": "frac{(text{close_diff10} - text{open_diff10})}{(text{high} - text{low} + 0.001)}",
+    "variables": {
+      "close": "Closing price of the stock",
+      "open": "Opening price of the stock",
+      "high": "Highest price of the stock during the day",
+      "low": "Lowest price of the stock during the day"
+    },
+    "Category": "Volume&Price",
+    "Difficulty": "Medium",
+    "gt_code": "import pandas as pd\ndata_pv = pd.read_hdf('daily_pv.h5')\nnew_df= data_pv.reset_index()\n# Calculate Alpha101\nnew_df['result'] = (new_df['$close'].diff(10) - new_df['$open'].diff(10)) / (new_df['$high'] - new_df['$low'] + 0.001)\n# keep the index of the original dataframe\nresult=pd.DataFrame(new_df['result']).set_index(data_pv.index)\n# transfer the result to series\nresult=result['result']\nresult.to_hdf('result.h5', key='data')"
+  },
+  "alpha_pv_diff_15": {
+    "description": "alpha_pv_diff is defined as the ratio of the difference between close prices 15 days change and open prices 15 days change to the sum of the highest minus lowest prices plus a small constant.",
+    "formulation": "frac{(text{close_diff15} - text{open_diff15})}{(text{high} - text{low} + 0.001)}",
+    "variables": {
+      "close": "Closing price of the stock",
+      "open": "Opening price of the stock",
+      "high": "Highest price of the stock during the day",
+      "low": "Lowest price of the stock during the day"
+    },
+    "Category": "Volume&Price",
+    "Difficulty": "Medium",
+    "gt_code": "import pandas as pd\ndata_pv = pd.read_hdf('daily_pv.h5')\nnew_df= data_pv.reset_index()\n# Calculate Alpha101\nnew_df['result'] = (new_df['$close'].diff(15) - new_df['$open'].diff(15)) / (new_df['$high'] - new_df['$low'] + 0.001)\n# keep the index of the original dataframe\nresult=pd.DataFrame(new_df['result']).set_index(data_pv.index)\n# transfer the result to series\nresult=result['result']\nresult.to_hdf('result.h5', key='data')"
+  },
+  "alpha_pv_diff_20": {
+    "description": "alpha_pv_diff is defined as the ratio of the difference between close prices 20 days change and open prices 20 days change to the sum of the highest minus lowest prices plus a small constant.",
+    "formulation": "frac{(text{close_diff20} - text{open_diff20})}{(text{high} - text{low} + 0.001)}",
+    "variables": {
+      "close": "Closing price of the stock",
+      "open": "Opening price of the stock",
+      "high": "Highest price of the stock during the day",
+      "low": "Lowest price of the stock during the day"
+    },
+    "Category": "Volume&Price",
+    "Difficulty": "Medium",
+    "gt_code": "import pandas as pd\ndata_pv = pd.read_hdf('daily_pv.h5')\nnew_df= data_pv.reset_index()\n# Calculate Alpha101\nnew_df['result'] = (new_df['$close'].diff(20) - new_df['$open'].diff(20)) / (new_df['$high'] - new_df['$low'] + 0.001)\n# keep the index of the original dataframe\nresult=pd.DataFrame(new_df['result']).set_index(data_pv.index)\n# transfer the result to series\nresult=result['result']\nresult.to_hdf('result.h5', key='data')"
+  },
+  "alpha_pv_diff_pct": {
+    "description": "alpha_pv is defined as the ratio of the difference between close prices 10 days change and open prices 10 days change to the sum of the highest prices 10 days change ratio minus lowest prices 10 days change ratio plus a small constant.",
+    "formulation": "frac{(text{close_diff10} - text{open_diff10})}{(text{high_pct10} - text{low_pct10} + 0.001)}",
+    "variables": {
+      "close": "Closing price of the stock",
+      "open": "Opening price of the stock",
+      "high": "Highest price of the stock during the day",
+      "low": "Lowest price of the stock during the day"
+    },
+    "Category": "Volume&Price",
+    "Difficulty": "Hard",
+    "gt_code": "import pandas as pd\ndata_pv = pd.read_hdf('daily_pv.h5')\nnew_df= data_pv.reset_index()\n# Calculate Alpha101\nnew_df['result'] = (new_df['$close'].diff(10) - new_df['$open'].diff(10)) / (new_df['$high'].pct_change(10) - new_df['$low'].pct_change(10) + 0.001)\n# keep the index of the original dataframe\nresult=pd.DataFrame(new_df['result']).set_index(data_pv.index)\n# transfer the result to series\nresult=result['result']\nresult.to_hdf('result.h5', key='data')"
+  },
+  "alpha_pv_diff_pct_15": {
+    "description": "alpha_pv is defined as the ratio of the difference between close prices 15 days change and open prices 15 days change to the sum of the highest prices 10 days change ratio minus lowest prices 10 days change ratio plus a small constant.",
+    "formulation": "frac{(text{close_diff15} - text{open_diff15})}{(text{high_pct10} - text{low_pct10} + 0.001)}",
+    "variables": {
+      "close": "Closing price of the stock",
+      "open": "Opening price of the stock",
+      "high": "Highest price of the stock during the day",
+      "low": "Lowest price of the stock during the day"
+    },
+    "Category": "Volume&Price",
+    "Difficulty": "Hard",
+    "gt_code": "import pandas as pd\ndata_pv = pd.read_hdf('daily_pv.h5')\nnew_df= data_pv.reset_index()\n# Calculate Alpha101\nnew_df['result'] = (new_df['$close'].diff(15) - new_df['$open'].diff(15)) / (new_df['$high'].pct_change(10) - new_df['$low'].pct_change(10) + 0.001)\n# keep the index of the original dataframe\nresult=pd.DataFrame(new_df['result']).set_index(data_pv.index)\n# transfer the result to series\nresult=result['result']\nresult.to_hdf('result.h5', key='data')"
+  },
+  "alpha_pv_diff_pct_20": {
+    "description": "alpha_pv is defined as the ratio of the difference between close prices 20 days change and open prices 20 days change to the sum of the highest prices 10 days change ratio minus lowest prices 10 days change ratio plus a small constant.",
+    "formulation": "frac{(text{close_diff20} - text{open_diff20})}{(text{high_pct10} - text{low_pct10} + 0.001)}",
+    "variables": {
+      "close": "Closing price of the stock",
+      "open": "Opening price of the stock",
+      "high": "Highest price of the stock during the day",
+      "low": "Lowest price of the stock during the day"
+    },
+    "Category": "Volume&Price",
+    "Difficulty": "Hard",
+    "gt_code": "import pandas as pd\ndata_pv = pd.read_hdf('daily_pv.h5')\nnew_df= data_pv.reset_index()\n# Calculate Alpha101\nnew_df['result'] = (new_df['$close'].diff(20) - new_df['$open'].diff(20)) / (new_df['$high'].pct_change(10) - new_df['$low'].pct_change(10) + 0.001)\n# keep the index of the original dataframe\nresult=pd.DataFrame(new_df['result']).set_index(data_pv.index)\n# transfer the result to series\nresult=result['result']\nresult.to_hdf('result.h5', key='data')"
+  },
+  "alpha053": {
+    "description": "Reversal class factor, negative delta of a ratio involving close, low, and high prices over 9 days.",
+    "formulation": "-1 times Deltaleft(frac{(text{close} - text{low}) - (text{high} - text{close})}{text{close} - text{low}}, 9right)",
+    "variables": {
+      "Delta(x, d)": "Change in 'x' over 'd' days.",
+      "text{close}": "Closing price of the stock.",
+      "text{low}": "Lowest price of the stock for the day.",
+      "text{high}": "Highest price of the stock for the day."
+    },
+    "Category": "Volume&Price",
+    "Difficulty": "Easy",
+    "gt_code": "import pandas as pd\ndata_pv = pd.read_hdf('daily_pv.h5')\nnew_df= data_pv.reset_index()\n# Calculate Alpha053\nnew_df['ratio'] =  (new_df['$close'] - new_df['$low'] - (new_df['$high'] - new_df['$close'])) / (new_df['$close'] - new_df['$low'])\n# the change of ratio in new_df over the 9 days\nnew_df['result']=-new_df['ratio'].diff(9)\n# transfer the result to series\nresult=pd.DataFrame(new_df['result']).set_index(data_pv.index)\nresult=result['result']\nresult.to_hdf('result.h5', key='data')"
+  },
+  "alpha053_5": {
+    "description": "Reversal class factor, negative delta of a ratio involving close, low, and high prices over 5 days.",
+    "formulation": "-1 times Deltaleft(frac{(text{close} - text{low}) - (text{high} - text{close})}{text{close} - text{low}}, 5right)",
+    "variables": {
+      "Delta(x, d)": "Change in 'x' over 'd' days.",
+      "text{close}": "Closing price of the stock.",
+      "text{low}": "Lowest price of the stock for the day.",
+      "text{high}": "Highest price of the stock for the day."
+    },
+    "Category": "Volume&Price",
+    "Difficulty": "Easy",
+    "gt_code": "import pandas as pd\ndata_pv = pd.read_hdf('daily_pv.h5')\nnew_df= data_pv.reset_index()\n# Calculate Alpha053\nnew_df['ratio'] =  (new_df['$close'] - new_df['$low'] - (new_df['$high'] - new_df['$close'])) / (new_df['$close'] - new_df['$low'])\n# the change of ratio in new_df over the 5 days\nnew_df['result']=-new_df['ratio'].diff(5)\n# transfer the result to series\nresult=pd.DataFrame(new_df['result']).set_index(data_pv.index)\nresult=result['result']\nresult.to_hdf('result.h5', key='data')"
+  }
+}

docs/api_reference.rst

@@ -0,0 +1,15 @@
+=============
+API Reference
+=============
+
+Here you can find all ``RDAgent``'s interfaces.
+
+
+RD Loop
+=======
+
+Research
+--------
+
+.. automodule:: rdagent.core.proposal
+    :members:

docs/changelog.md

@@ -0,0 +1,4 @@
+# Changelog
+
+## [Unreleased]
+<!-- insertion marker -->

docs/conf.py

@@ -0,0 +1,72 @@
+# Configuration file for the Sphinx documentation builder.
+#
+# For the full list of built-in configuration values, see the documentation:
+# https://www.sphinx-doc.org/en/master/usage/configuration.html
+
+# -- Project information -----------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information
+
+import subprocess
+
+latest_tag = subprocess.check_output(["git", "describe", "--tags", "--abbrev=0"], text=True).strip()
+
+project = "RDAgent"
+copyright = "2024, Microsoft"
+author = "Microsoft"
+
+# -- General configuration ---------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration
+
+extensions = ["sphinx.ext.autodoc", "sphinxcontrib.autodoc_pydantic"]
+
+autodoc_member_order = "bysource"
+
+# The suffix of source filenames.
+source_suffix = {".rst": "restructuredtext"}
+
+# The encoding of source files.
+source_encoding = "utf-8"
+
+# The main toctree document.
+master_doc = "index"
+
+# The version info for the project you're documenting, acts as replacement for
+# |version| and |release|, also used in various other places throughout the
+# built documents.
+#
+# The short X.Y version.
+version = latest_tag
+release = latest_tag
+
+# The language for content autogenerated by Sphinx. Refer to documentation for
+# a list of supported languages.
+language = "en"
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+exclude_patterns = ["build"]
+
+# -- Options for HTML output -------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = "sphinx"
+
+try:
+    import furo
+
+    html_theme = "furo"
+    html_theme_options = {
+        "navigation_with_keys": True,
+    }
+except ImportError:
+    html_theme = "default"
+
+html_logo = "_static/logo.png"
+html_static_path = ["_static"]
+html_favicon = "_static/favicon.ico"
+
+html_theme_options = {
+    "source_repository": "https://github.com/microsoft/RD-Agent",
+    "source_branch": "main",
+    "source_directory": "docs/",
+}

docs/development.rst

@@ -0,0 +1,85 @@
+=========================
+For Development
+=========================
+
+If you want to try the latest version or contribute to RD-Agent. You can install it from the source and follow the commands in this page.
+
+   .. code-block:: bash
+
+      git clone https://github.com/microsoft/RD-Agent
+
+
+🔧Prepare for development
+=========================
+
+- Set up the development environment.
+
+   .. code-block:: bash
+
+      make dev
+
+- Run linting and checking.
+
+   .. code-block:: bash
+
+      make lint
+
+
+- Some linting issues can be fixed automatically. We have added a command in the Makefile for easy use.
+
+   .. code-block:: bash
+
+      make auto-lint
+
+
+
+Code Structure
+=========================
+
+.. code-block:: text
+
+    📂 src
+    ➥ 📂 <project name>: avoid namespace conflict
+      ➥ 📁 core
+      ➥ 📁 components/A
+      ➥ 📁 components/B
+      ➥ 📁 components/C
+      ➥ 📁 scenarios/X
+      ➥ 📁 scenarios/Y
+      ➥ 📂 app
+    ➥ 📁 scripts
+
+.. list-table::
+   :header-rows: 1
+
+   * - Folder Name
+     - Description
+   * - 📁 core
+     - The core framework of the system. All classes should be abstract and usually can't be used directly.
+   * - 📁 component/A
+     - Useful components that can be used by others (e.g., scenarios). Many subclasses of core classes are located here.
+   * - 📁 scenarios/X
+     - Concrete features for specific scenarios (usually built based on components or core). These modules are often unreusable across scenarios.
+   * - 📁 app
+     - Applications for specific scenarios (usually built based on components or scenarios). Removing any of them does not affect the system's completeness or other scenarios.
+   * - 📁 scripts
+     - Quick and dirty things. These are candidates for core, components, scenarios, and apps.
+
+
+
+Conventions
+===========
+
+
+File Naming Convention
+----------------------
+
+.. list-table::
+   :header-rows: 1
+
+   * - Name
+     - Description
+   * - `conf.py`
+     - The configuration for the module, app, and project.
+
+.. <!-- TODO: renaming files -->

docs/index.rst

@@ -0,0 +1,34 @@
+.. RDAgent documentation master file, created by
+   sphinx-quickstart on Mon Jul 15 04:27:50 2024.
+   You can adapt this file completely to your liking, but it should at least
+   contain the root `toctree` directive.
+
+Welcome to RDAgent's documentation!
+===================================
+
+.. image:: _static/logo.png
+   :alt: RD-Agent Logo
+
+.. toctree::
+   :maxdepth: 3
+   :caption: Doctree:
+
+   introduction
+   installation_and_configuration
+   scens/catalog
+   project_framework_introduction
+   ui
+   research/catalog
+   development
+   api_reference
+   policy
+
+   GitHub <https://github.com/microsoft/RD-Agent>
+
+
+Indices and tables
+==================
+
+* :ref:`genindex`
+* :ref:`modindex`
+* :ref:`search`

docs/installation_and_configuration.rst

@@ -0,0 +1,244 @@
+==============================
+Installation and Configuration
+==============================
+
+Installation
+============
+
+**Install RDAgent**: For different scenarios
+
+- for purely users: please use ``pip install rdagent`` to install RDAgent
+- for dev users: `See development <development.html>`_
+
+**Install Docker**: RDAgent is designed for research and development, acting like a human researcher and developer. It can write and run code in various environments, primarily using Docker for code execution. This keeps the remaining dependencies simple. Users must ensure Docker is installed before attempting most scenarios. Please refer to the `official 🐳Docker page <https://docs.docker.com/engine/install/>`_ for installation instructions.
+Ensure the current user can run Docker commands **without using sudo**. You can verify this by executing `docker run hello-world`.
+
+LiteLLM Backend Configuration (Default)
+=======================================
+
+Option 1: Unified API base for both models
+------------------------------------------
+
+   .. code-block:: Properties
+
+      # Set to any model supported by LiteLLM.
+      CHAT_MODEL=gpt-4o 
+      EMBEDDING_MODEL=text-embedding-3-small
+      # Configure unified API base
+      # The backend api_key fully follows the convention of litellm.
+      OPENAI_API_BASE=<your_unified_api_base>
+      OPENAI_API_KEY=<replace_with_your_openai_api_key>
+
+Option 2: Separate API bases for Chat and Embedding models
+----------------------------------------------------------
+
+   .. code-block:: Properties
+
+      # Set to any model supported by LiteLLM.
+      
+      # CHAT MODEL:
+      CHAT_MODEL=gpt-4o 
+      OPENAI_API_BASE=<your_chat_api_base>
+      OPENAI_API_KEY=<replace_with_your_openai_api_key>
+
+      # EMBEDDING MODEL:
+      # TAKE siliconflow as an example, you can use other providers.
+      # Note: embedding requires litellm_proxy prefix
+      EMBEDDING_MODEL=litellm_proxy/BAAI/bge-large-en-v1.5
+      LITELLM_PROXY_API_KEY=<replace_with_your_siliconflow_api_key>
+      LITELLM_PROXY_API_BASE=https://api.siliconflow.cn/v1
+
+Necessary parameters include:
+
+- `CHAT_MODEL`: The model name of the chat model.
+
+- `EMBEDDING_MODEL`: The model name of the embedding model.
+
+- `OPENAI_API_BASE`: The base URL of the API. If `EMBEDDING_MODEL` does not start with `litellm_proxy/`, this is used for both chat and embedding models; otherwise, it is used for `CHAT_MODEL` only.
+
+Optional parameters (required if your embedding model is provided by a different provider than `CHAT_MODEL`):
+
+- `LITELLM_PROXY_API_KEY`: The API key for the embedding model, required if `EMBEDDING_MODEL` starts with `litellm_proxy/`.
+
+- `LITELLM_PROXY_API_BASE`: The base URL for the embedding model, required if `EMBEDDING_MODEL` starts with `litellm_proxy/`.
+
+**Note:** If you are using an embedding model from a provider different from the chat model, remember to add the `litellm_proxy/` prefix to the `EMBEDDING_MODEL` name.
+
+
+The `CHAT_MODEL` and `EMBEDDING_MODEL` parameters will be passed into LiteLLM's completion function. 
+
+Therefore, when utilizing models provided by different providers, first review the interface configuration of LiteLLM. The model names must match those allowed by LiteLLM.
+
+Additionally, you need to set up the the additional parameters for the respective model provider, and the parameter names must align with those required by LiteLLM.
+
+For example, if you are using a DeepSeek model, you need to set as follows:
+
+   .. code-block:: Properties
+
+      # For some models LiteLLM requires a prefix to the model name.
+      CHAT_MODEL=deepseek/deepseek-chat
+      DEEPSEEK_API_KEY=<replace_with_your_deepseek_api_key>
+
+Besides, when you are using reasoning models, the response might include the thought process. For this case, you need to set the following environment variable:
+   
+   .. code-block:: Properties
+      
+      REASONING_THINK_RM=True
+
+For more details on LiteLLM requirements, refer to the `official LiteLLM documentation <https://docs.litellm.ai/docs>`_.
+
+
+Configuration(deprecated)
+=========================
+
+To run the application, please create a `.env` file in the root directory of the project and add environment variables according to your requirements.
+
+If you are using this deprecated version,  you should set `BACKEND` to `rdagent.oai.backend.DeprecBackend`.
+
+Here are some other configuration options that you can use:
+
+OpenAI API
+------------
+
+Here is a standard configuration for the user using the OpenAI API.
+
+   .. code-block:: Properties
+
+      OPENAI_API_KEY=<your_api_key>
+      EMBEDDING_MODEL=text-embedding-3-small
+      CHAT_MODEL=gpt-4-turbo
+
+Azure OpenAI
+------------
+
+The following environment variables are standard configuration options for the user using the OpenAI API.
+
+   .. code-block:: Properties
+
+      USE_AZURE=True
+
+      EMBEDDING_OPENAI_API_KEY=<replace_with_your_azure_openai_api_key>
+      EMBEDDING_AZURE_API_BASE=  # The endpoint for the Azure OpenAI API.
+      EMBEDDING_AZURE_API_VERSION=  # The version of the Azure OpenAI API.
+      EMBEDDING_MODEL=text-embedding-3-small
+
+      CHAT_OPENAI_API_KEY=<replace_with_your_azure_openai_api_key>
+      CHAT_AZURE_API_BASE=  # The endpoint for the Azure OpenAI API.
+      CHAT_AZURE_API_VERSION=  # The version of the Azure OpenAI API.
+      CHAT_MODEL=  # The model name of the Azure OpenAI API.
+
+Use Azure Token Provider
+------------------------
+
+If you are using the Azure token provider, you need to set the `CHAT_USE_AZURE_TOKEN_PROVIDER` and `EMBEDDING_USE_AZURE_TOKEN_PROVIDER` environment variable to `True`. then 
+use the environment variables provided in the `Azure Configuration section <installation_and_configuration.html#azure-openai>`_.
+
+
+☁️ Azure Configuration
+- Install Azure CLI:
+
+   ```sh
+   curl -L https://aka.ms/InstallAzureCli | bash
+   ```
+
+- Log in to Azure:
+
+   ```sh
+   az login --use-device-code
+   ```
+
+- `exit` and re-login to your environment (this step may not be necessary).
+
+
+Configuration List
+------------------
+
+.. TODO: use `autodoc-pydantic` .
+
+- OpenAI API Setting
+
++-----------------------------------+-----------------------------------------------------------------+-------------------------+
+| Configuration Option              | Meaning                                                         | Default Value           |
++===================================+=================================================================+=========================+
+| OPENAI_API_KEY                    | API key for both chat and embedding models                      | None                    |
++-----------------------------------+-----------------------------------------------------------------+-------------------------+
+| EMBEDDING_OPENAI_API_KEY          | Use a different API key for embedding model                     | None                    |
++-----------------------------------+-----------------------------------------------------------------+-------------------------+
+| CHAT_OPENAI_API_KEY               | Set to use a different API key for chat model                   | None                    |
++-----------------------------------+-----------------------------------------------------------------+-------------------------+
+| EMBEDDING_MODEL                   | Name of the embedding model                                     | text-embedding-3-small  |
++-----------------------------------+-----------------------------------------------------------------+-------------------------+
+| CHAT_MODEL                        | Name of the chat model                                          | gpt-4-turbo             |
++-----------------------------------+-----------------------------------------------------------------+-------------------------+
+| EMBEDDING_AZURE_API_BASE          | Base URL for the Azure OpenAI API                               | None                    |
++-----------------------------------+-----------------------------------------------------------------+-------------------------+
+| EMBEDDING_AZURE_API_VERSION       | Version of the Azure OpenAI API                                 | None                    |
++-----------------------------------+-----------------------------------------------------------------+-------------------------+
+| CHAT_AZURE_API_BASE               | Base URL for the Azure OpenAI API                               | None                    |
++-----------------------------------+-----------------------------------------------------------------+-------------------------+
+| CHAT_AZURE_API_VERSION            | Version of the Azure OpenAI API                                 | None                    |
++-----------------------------------+-----------------------------------------------------------------+-------------------------+
+| USE_AZURE                         | True if you are using Azure OpenAI                              | False                   |
++-----------------------------------+-----------------------------------------------------------------+-------------------------+
+| CHAT_USE_AZURE_TOKEN_PROVIDER     | True if you are using an Azure Token Provider in chat model     | False                   |
++-----------------------------------+-----------------------------------------------------------------+-------------------------+
+| EMBEDDING_USE_AZURE_TOKEN_PROVIDER| True if you are using an Azure Token Provider in embedding model| False                   |
++-----------------------------------+-----------------------------------------------------------------+-------------------------+
+
+- Globol Setting
+
++-----------------------------+--------------------------------------------------+-------------------------+
+| Configuration Option        | Meaning                                          | Default Value           |
++=============================+==================================================+=========================+
+| max_retry                   | Maximum number of times to retry                 | 10                      |
++-----------------------------+--------------------------------------------------+-------------------------+
+| retry_wait_seconds          | Number of seconds to wait before retrying        | 1                       |
++-----------------------------+--------------------------------------------------+-------------------------+
++ log_trace_path              | Path to log trace file                           | None                    |
++-----------------------------+--------------------------------------------------+-------------------------+
++ log_llm_chat_content        | Flag to indicate if chat content is logged       | True                    |
++-----------------------------+--------------------------------------------------+-------------------------+
+
+
+- Cache Setting
+
+.. TODO: update Meaning for caches
+
++------------------------------+--------------------------------------------------+-------------------------+
+| Configuration Option         | Meaning                                          | Default Value           |
++==============================+==================================================+=========================+
+| dump_chat_cache              | Flag to indicate if chat cache is dumped         | False                   |
++------------------------------+--------------------------------------------------+-------------------------+
+| dump_embedding_cache         | Flag to indicate if embedding cache is dumped    | False                   |
++------------------------------+--------------------------------------------------+-------------------------+
+| use_chat_cache               | Flag to indicate if chat cache is used           | False                   |
++------------------------------+--------------------------------------------------+-------------------------+
+| use_embedding_cache          | Flag to indicate if embedding cache is used      | False                   |
++------------------------------+--------------------------------------------------+-------------------------+
+| prompt_cache_path            | Path to prompt cache                             | ./prompt_cache.db       |
++------------------------------+--------------------------------------------------+-------------------------+
+| max_past_message_include     | Maximum number of past messages to include       | 10                      |
++------------------------------+--------------------------------------------------+-------------------------+
+
+
+
+
+Loading Configuration
+---------------------
+
+For users' convenience, we provide a CLI interface called `rdagent`, which automatically runs `load_dotenv()` to load environment variables from the `.env` file.
+However, this feature is not enabled by default for other scripts. We recommend users load the environment with the following steps:
+
+
+- ⚙️ Environment Configuration
+    - Place the `.env` file in the same directory as the `.env.example` file.
+        - The `.env.example` file contains the environment variables required for users using the OpenAI API (Please note that `.env.example` is an example file. `.env` is the one that will be finally used.)
+
+    - Export each variable in the .env file:
+
+      .. code-block:: sh
+
+          export $(grep -v '^#' .env | xargs)
+    
+    - If you want to change the default environment variables, you can refer to the above configuration and edith the `.env` file.
+

docs/introduction.rst

@@ -0,0 +1,18 @@
+=========================
+Introduction
+=========================
+
+
+
+In modern industry, research and development (R&D) is crucial for the enhancement of industrial productivity, especially in the AI era, where the core aspects of R&D are mainly focused on data and models. We are committed to automate these high-value generic R&D processes through our open source R&D automation tool RDAgent, which let AI drive data-driven AI.
+
+.. image:: _static/scen.png
+   :alt: Our focused scenario
+
+
+Our RDAgent is designed to automate the most critical industrial R&D processes, focusing first on data-driven scenarios, to greatly boost the development productivity of models and data. 
+
+Methodologically, we propose an autonomous agent framework that consists of two key parts: (R)esearch stands for actively exploring by proposing new ideas, and (D)evelopment stands for realizing these ideas. The effectiveness of these two components will ultimately get feedbacks through practice, and both research and development capabilities can continuously learn and grow in the process.
+
+
+For a quick start, visit `our GitHub home page <https://github.com/microsoft/RD-Agent>`_ ⚡. If you've already checked it out and want more details, please keep reading.

docs/make.bat

@@ -0,0 +1,35 @@
+@ECHO OFF
+
+pushd %~dp0
+
+REM Command file for Sphinx documentation
+
+if "%SPHINXBUILD%" == "" (
+	set SPHINXBUILD=sphinx-build
+)
+set SOURCEDIR=.
+set BUILDDIR=build
+
+%SPHINXBUILD% >NUL 2>NUL
+if errorlevel 9009 (
+	echo.
+	echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
+	echo.installed, then set the SPHINXBUILD environment variable to point
+	echo.to the full path of the 'sphinx-build' executable. Alternatively you
+	echo.may add the Sphinx directory to PATH.
+	echo.
+	echo.If you don't have Sphinx installed, grab it from
+	echo.https://www.sphinx-doc.org/
+	exit /b 1
+)
+
+if "%1" == "" goto help
+
+%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+goto end
+
+:help
+%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+
+:end
+popd

docs/policy.rst

@@ -0,0 +1,24 @@
+======
+Policy
+======
+
+This project welcomes contributions and suggestions.  Most contributions require you to agree to a
+Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us
+the rights to use your contribution. For details, visit https://cla.opensource.microsoft.com.
+
+When you submit a pull request, a CLA bot will automatically determine whether you need to provide
+a CLA and decorate the PR appropriately (e.g., status check, comment). Simply follow the instructions
+provided by the bot. You will only need to do this once across all repos using our CLA.
+
+This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/).
+For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or
+contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments.
+
+Trademarks
+==========
+
+This project may contain trademarks or logos for projects, products, or services. Authorized use of Microsoft 
+trademarks or logos is subject to and must follow 
+[Microsoft's Trademark & Brand Guidelines](https://www.microsoft.com/en-us/legal/intellectualproperty/trademarks/usage/general).
+Use of Microsoft trademarks or logos in modified versions of this project must not cause confusion or imply Microsoft sponsorship.
+Any use of third-party trademarks or logos are subject to those third-party's policies.

docs/project_framework_introduction.rst

@@ -0,0 +1,27 @@
+===============================
+Framework Design & Components
+===============================
+
+Framework & Components
+=========================
+
+.. NOTE: This depends on the correctness of `c-v` of github.
+
+.. image:: _static/Framework-RDAgent.png
+    :alt: Components & Feature Level
+
+The image above shows the overall framework of RDAgent.
+
+In a data mining expert's daily research and development process, they propose a hypothesis (e.g., a model structure like RNN can capture patterns in time-series data), design experiments (e.g., finance data contains time-series and we can verify the hypothesis in this scenario), implement the experiment as code (e.g., Pytorch model structure), and then execute the code to get feedback (e.g., metrics, loss curve, etc.). The experts learn from the feedback and improve in the next iteration.
+
+We have established a basic method framework that continuously proposes hypotheses, verifies them, and gets feedback from the real world. This is the first scientific research automation framework that supports linking with real-world verification.
+
+
+.. image:: https://github.com/user-attachments/assets/60cc2712-c32a-4492-a137-8aec59cdc66e
+    :alt: Class Level Figure
+
+The figure above shows the main classes and how they fit into the workflow for those interested in the detailed code.
+
+
+.. Detailed Design
+.. ===============

docs/requirements.txt

@@ -0,0 +1,4 @@
+sphinx
+sphinx_rtd_theme
+furo
+importlib.metadata

docs/research/benchmark.rst

@@ -0,0 +1,109 @@
+==============================
+Benchmark
+==============================
+
+Introduction
+=============
+
+Benchmarking the capabilities of R&D is a crucial research problem in this area. We are continuously exploring methods to benchmark these capabilities. The current benchmarks are listed on this page.
+
+Development Capability Benchmarking
+===================================
+
+Benchmarking is used to evaluate the effectiveness of factors with fixed data. It mainly includes the following steps:
+
+1. :ref:`read and prepare the eval_data <data>`
+
+2. :ref:`declare the method to be tested and pass the arguments <config>`
+
+3. :ref:`declare the eval method and pass the arguments <config>`
+
+4. :ref:`run the eval <run>`
+
+5. :ref:`save and show the result <show>`
+
+Configuration
+-------------
+.. _config:
+
+.. autopydantic_settings:: rdagent.components.benchmark.conf.BenchmarkSettings
+
+Example
++++++++
+.. _example:
+
+The default value for ``bench_test_round`` is 10, which takes about 2 hours to run. To modify it from ``10`` to ``2``, adjust the environment variables in the .env file as shown below.
+
+.. code-block:: Properties
+
+      BENCHMARK_BENCH_TEST_ROUND=2
+
+Data Format
+-------------
+.. _data:
+
+The sample data in ``bench_data_path`` is a dictionary where each key represents a factor name. The value associated with each key is factor data containing the following information:
+
+- **description**: A textual description of the factor.
+- **formulation**: A LaTeX formula representing the model's formulation.
+- **variables**: A dictionary of variables involved in the factor.
+- **Category**: The category or classification of the factor.
+- **Difficulty**: The difficulty level of implementing or understanding the factor.
+- **gt_code**: A piece of code associated with the factor.
+
+Here is an example of this data format:
+
+.. literalinclude:: ../../rdagent/components/benchmark/example.json
+   :language: json
+
+Ensure the data is placed in the ``FACTOR_COSTEER_SETTINGS.data_folder_debug``. The data files should be in ``.h5`` or ``.md`` format and must not be stored in any subfolders. LLM-Agents will review the file content and implement the tasks.
+
+.. TODO: Add a script to automatically generate the data in the `rdagent/app/quant_factor_benchmark/data` folder.
+
+Run Benchmark
+-------------
+.. _run:
+
+Start the benchmark after completing the :doc:`../installation_and_configuration`.
+
+.. code-block:: Properties
+
+      dotenv run -- python rdagent/app/benchmark/factor/eval.py
+
+Once completed, a pkl file will be generated, and its path will be printed on the last line of the console.
+
+Show Result
+-------------
+.. _show:
+
+The ``analysis.py`` script reads data from the pkl file and converts it to an image. Modify the Python code in ``rdagent/app/quant_factor_benchmark/analysis.py`` to specify the path to the pkl file and the output path for the png file.
+
+.. code-block:: Properties
+
+      dotenv run -- python rdagent/app/benchmark/factor/analysis.py <log/path to.pkl>
+
+A png file will be saved to the designated path as shown below.
+
+.. image:: ../_static/benchmark.png
+
+Related Paper
+-------------
+
+- `Towards Data-Centric Automatic R&D <https://arxiv.org/abs/2404.11276>`_:
+  We have developed a comprehensive benchmark called RD2Bench to assess data and model R&D capabilities. This benchmark includes a series of tasks that outline the features or structures of models. These tasks are used to evaluate the ability of LLM-Agents to implement them.
+
+.. code-block:: bibtex
+
+    @misc{chen2024datacentric,
+        title={Towards Data-Centric Automatic R&D},
+        author={Haotian Chen and Xinjie Shen and Zeqi Ye and Wenjun Feng and Haoxue Wang and Xiao Yang and Xu Yang and Weiqing Liu and Jiang Bian},
+        year={2024},
+        eprint={2404.11276},
+        archivePrefix={arXiv},
+        primaryClass={cs.AI}
+    }
+
+.. image:: https://github.com/user-attachments/assets/494f55d3-de9e-4e73-ba3d-a787e8f9e841
+
+To replicate the benchmark detailed in the paper, please consult the factors listed in the following file: `RD2bench.json <../_static/RD2bench.json>`_.
+Please note use ``only_correct_format=False`` when evaluating the results.

docs/research/catalog.rst

@@ -0,0 +1,34 @@
+===========
+Research
+===========
+
+To achieve the good effects and improve R&D capabilities, we face multiple challenges, the most important of which is the continuous evolution capability. Existing large language models (LLMs) find it difficult to continue growing their capabilities after training is completed. Moreover, the training process of LLMs focuses more on general knowledge, and the lack of depth in more specialized knowledge becomes an obstacle to solving professional R&D problems within the industry. This specialized knowledge needs to be learned and acquired from in-depth industry practice.
+
+
+Our RD-Agent, on the other hand, can continuously acquire in-depth domain knowledge through deep exploration during the R&D phase, allowing its R&D capabilities to keep growing.
+
+To address these key challenges and achieve industrial value, a series of research work needs to be completed.
+
+
+.. list-table:: Research Areas and Descriptions
+   :header-rows: 1
+
+   * - Research Area
+     - Description
+   * - :doc:`Benchmark <benchmark>`
+     - Benchmark the R&D abilities
+   * - Research
+     - Idea proposal: Explore new ideas or refine existing ones
+   * - :doc:`Development <dev>`
+     - Ability to realize ideas: Implement and execute ideas
+
+
+
+
+.. toctree::
+   :maxdepth: 1
+   :caption: Doctree:
+   :hidden:
+
+   benchmark
+   dev

docs/research/dev.rst

@@ -0,0 +1,25 @@
+==============================
+Development
+==============================
+
+
+Related Paper
+-------------
+
+- `Collaborative Evolving Strategy for Automatic Data-Centric Development <https://arxiv.org/abs/2407.18690>`_
+  Co-STEER is a method to tackle data-centric development (AD2) tasks and highlight its main challenges, which need expert-like implementation (i.e., learning domain knowledge from practice) and task scheduling capability (e.g., starting with easier tasks for better overall efficiency), areas that previous work has largely overlooked. Our Co-STEER agent enhances its domain knowledge through our evolving strategy and improves both its scheduling and implementation skills by gathering and using domain-specific practical experience. With a better schedule, implementation becomes faster. At the same time, as implementation feedback becomes more detailed, scheduling accuracy improves. These two capabilities grow together through practical feedback, enabling a collaborative evolution process.
+
+.. code-block:: bibtex
+
+    @misc{yang2024collaborative,
+        title={Collaborative Evolving Strategy for Automatic Data-Centric Development},
+        author={Xu Yang and Haotian Chen and Wenjun Feng and Haoxue Wang and Zeqi Ye and Xinjie Shen and Xiao Yang and Shizhao Sun and Weiqing Liu and Jiang Bian},
+        year={2024},
+        eprint={2407.18690},
+        archivePrefix={arXiv},
+        primaryClass={cs.AI}
+    }
+
+.. image:: https://github.com/user-attachments/assets/75d9769b-0edd-4caf-9d45-57d1e577054b
+   :alt: Collaborative Evolving Strategy for Automatic Data-Centric Development
+

docs/scens/catalog.rst

@@ -0,0 +1,45 @@
+=========================
+Scenarios
+=========================
+
+Scenario lists
+=========================
+
+In the two key areas of data-driven scenarios, model implementation and data building, our system aims to serve two main roles: 🦾copilot and 🤖agent.
+
+- The 🦾copilot follows human instructions to automate repetitive tasks.
+- The 🤖agent, being more autonomous, actively proposes ideas for better results in the future.
+
+The supported scenarios are listed below:
+
+
+.. list-table::
+    :header-rows: 1
+
+    * - Scenario/Target
+      - Model Implementation
+      - Data Building
+    * - 💹 Finance
+      - :ref:`🥇The First Data-Centric Quant Multi-Agent Framework <quant_agent_fin>`
+      - :ref:`🤖Iteratively Proposing Ideas & Evolving <model_agent_fin>`  
+        
+        :ref:`🦾Auto reports reading & implementation <data_copilot_fin>`  
+        
+        :ref:`🤖Iteratively Proposing Ideas & Evolving <data_agent_fin>`
+    * - 🏭 General
+      - :ref:`🦾Auto paper reading & implementation <model_copilot_general>`  
+        
+      - :ref:`🤖 Data Science <data_science_agent>`
+
+
+.. toctree::
+    :maxdepth: 1
+    :caption: Doctree:
+    :hidden:
+
+    quant_agent_fin
+    data_agent_fin
+    data_copilot_fin
+    model_agent_fin
+    model_copilot_general
+    data_science

docs/scens/data_agent_fin.rst

@@ -0,0 +1,138 @@
+.. _data_agent_fin:
+
+=====================
+Finance Data Agent
+=====================
+
+
+**🤖 Automated Quantitative Trading & Iterative Factors Evolution**
+-------------------------------------------------------------------
+
+📖 Background
+~~~~~~~~~~~~~~
+In the dynamic world of quantitative trading, **factors** serve as the strategic tools that enable traders to exploit market inefficiencies. 
+These factors—ranging from simple metrics like price-to-earnings ratios to complex models like discounted cash flows—are the key to predicting stock prices with a high degree of accuracy.
+
+By leveraging these factors, quantitative traders can develop sophisticated strategies that not only identify market patterns but also significantly enhance trading efficiency and precision. 
+The ability to systematically analyze and apply these factors is what separates ordinary trading from truly strategic market outmaneuvering.
+And this is where the **Finance Model Agent** comes into play.
+
+🎥 `Demo <https://rdagent.azurewebsites.net/factor_loop>`_
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. raw:: html
+
+    <div style="display: flex; justify-content: center; align-items: center;">
+      <video width="600" controls>
+        <source src="https://rdagent.azurewebsites.net/media/65bb598f1372c1857ccbf09b2acf5d55830911625048c03102291098.mp4" type="video/mp4">
+        Your browser does not support the video tag.
+      </video>
+    </div>
+
+
+🌟 Introduction
+~~~~~~~~~~~~~~~~
+In this scenario, our agent illustrates the iterative process of hypothesis generation, knowledge construction, and decision-making. 
+
+It highlights how financial factors evolve through continuous feedback and refinement. 
+
+Here's an enhanced outline of the steps:
+
+**Step 1 : Hypothesis Generation 🔍**
+
+- Generate and propose initial hypotheses based on previous experiment analysis and domain expertise, with thorough reasoning and financial justification.
+
+**Step 2 : Factor Creation ✨**
+
+- Based on the hypothesis, divide the tasks.
+- Each task involves developing, defining, and implementing a new financial factor, including its name, description, formulation, and variables.
+
+**Step 3 : Factor Implementation 👨‍💻**
+
+- Implement the factor code based on the description, evolving it as a developer would.
+- Quantitatively validate the newly created factors.
+
+**Step 4 : Backtesting with Qlib 📉**
+
+- Integrate the full dataset into the factor implementation code and prepare the factor library.
+- Conduct backtesting using the Alpha158 plus newly developed factors and LGBModel in Qlib to evaluate the new factors' effectiveness and performance.
+
++----------------+------------+----------------+----------------------------------------------------+
+| Dataset        | Model      | Factors        | Data Split                                         |
++================+============+================+====================================================+
+| CSI300         | LGBModel   | Alpha158 Plus  | +-----------+--------------------------+           |
+|                |            |                | | Train     | 2008-01-01 to 2014-12-31 |           |
+|                |            |                | +-----------+--------------------------+           |
+|                |            |                | | Valid     | 2015-01-01 to 2016-12-31 |           |
+|                |            |                | +-----------+--------------------------+           |
+|                |            |                | | Test      | 2017-01-01 to 2020-08-01 |           |
+|                |            |                | +-----------+--------------------------+           |
++----------------+------------+----------------+----------------------------------------------------+
+
+
+**Step 5 : Feedback Analysis 🔍**
+
+- Analyze backtest results to assess performance.
+- Incorporate feedback to refine hypotheses and improve the model.
+
+**Step 6 :Hypothesis Refinement ♻️**
+
+- Refine hypotheses based on feedback from backtesting.
+- Repeat the process to continuously improve the model.
+
+⚡ Quick Start
+~~~~~~~~~~~~~~~~~
+
+Please refer to the installation part in :doc:`../installation_and_configuration` to prepare your system dependency.
+
+You can try our demo by running the following command:
+
+- 🐍 Create a Conda Environment
+
+  - Create a new conda environment with Python (3.10 and 3.11 are well tested in our CI):
+
+    .. code-block:: sh
+
+          conda create -n rdagent python=3.10
+
+  - Activate the environment:
+
+    .. code-block:: sh
+
+        conda activate rdagent
+
+- 📦 Install the RDAgent
+  
+  - You can install the RDAgent package from PyPI:
+
+    .. code-block:: sh
+
+        pip install rdagent
+
+- 🚀 Run the Application
+    
+  - You can directly run the application by using the following command:
+    
+    .. code-block:: sh
+
+        rdagent fin_factor
+
+
+🛠️ Usage of modules
+~~~~~~~~~~~~~~~~~~~~~
+
+.. _Env Config: 
+
+- **Env Config**
+
+The following environment variables can be set in the `.env` file to customize the application's behavior:
+
+.. autopydantic_settings:: rdagent.app.qlib_rd_loop.conf.FactorBasePropSetting
+    :settings-show-field-summary: False
+    :exclude-members: Config
+
+.. autopydantic_settings:: rdagent.components.coder.factor_coder.config.FactorCoSTEERSettings
+    :settings-show-field-summary: False
+    :members: coder_use_cache, data_folder, data_folder_debug, file_based_execution_timeout, select_method, max_loop, knowledge_base_path, new_knowledge_base_path
+    :exclude-members: Config, fail_task_trial_limit, v1_query_former_trace_limit, v1_query_similar_success_limit, v2_query_component_limit, v2_query_error_limit, v2_query_former_trace_limit, v2_error_summary, v2_knowledge_sampler
+    :no-index:

docs/scens/data_copilot_fin.rst

@@ -0,0 +1,164 @@
+.. _data_copilot_fin:
+
+=====================
+Finance Data Copilot
+=====================
+
+
+**🤖 Automated Quantitative Trading & Factors Extraction from Financial Reports**
+---------------------------------------------------------------------------------
+
+📖 Background
+~~~~~~~~~~~~~~
+**Research reports** are treasure troves of insights, often unveiling potential **factors** that can drive successful quantitative trading strategies. 
+Yet, with the sheer volume of reports available, extracting the most valuable insights efficiently becomes a daunting task.
+
+Furthermore, rather than hastily replicating factors from a report, it's essential to delve into the underlying logic of their construction. 
+Does the factor capture the essential market dynamics? How unique is it compared to the factors already in your library?
+
+Therefore, there is an urgent need for a systematic approach to design a framework that can effectively manage this process. 
+And this is where the **Finance Data Copilot** steps in.
+
+
+🎥 `Demo <https://rdagent.azurewebsites.net/report_factor>`_
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. raw:: html
+
+    <div style="display: flex; justify-content: center; align-items: center;">
+      <video width="600" controls>
+        <source src="https://rdagent.azurewebsites.net/media/7b14b2bd3d8771da9cf7eb799b6d96729cec3d35c8d4f68060f3e2fd.mp4" type="video/mp4">
+        Your browser does not support the video tag.
+      </video>
+    </div>
+
+
+🌟 Introduction
+~~~~~~~~~~~~~~~~
+In this scenario, RDAgent demonstrates the process of extracting factors from financial research reports, implementing these factors, and analyzing their performance through Qlib backtesting. 
+This process continually expands and refines the factor library.
+
+Here's an enhanced outline of the steps:
+
+**Step 1 : Hypothesis Generation 🔍**
+
+- Generate and propose initial hypotheses based on insights from financial reports with thorough reasoning and financial justification.
+
+**Step 2 : Factor Creation ✨**
+
+- Based on the hypothesis and financial reports, divide the tasks. 
+- Each task involves developing, defining, and implementing a new financial factor, including its name, description, formulation, and variables.
+
+**Step 3 : Factor Implementation 👨‍💻**
+
+- Implement the factor code based on the description, evolving it as a developer would.
+- Quantitatively validate the newly created factors.
+
+**Step 4 : Backtesting with Qlib 📉**
+
+- Integrate the full dataset into the factor implementation code and prepare the factor library.
+- Conduct backtesting using the Alpha158 plus newly developed factors and LGBModel in Qlib to evaluate the new factors' effectiveness and performance.
+
++----------------+------------+----------------+----------------------------------------------------+
+| Dataset        | Model      | Factors        | Data Split                                         |
++================+============+================+====================================================+
+| CSI300         | LGBModel   | Alpha158 Plus  | +-----------+--------------------------+           |
+|                |            |                | | Train     | 2008-01-01 to 2014-12-31 |           |
+|                |            |                | +-----------+--------------------------+           |
+|                |            |                | | Valid     | 2015-01-01 to 2016-12-31 |           |
+|                |            |                | +-----------+--------------------------+           |
+|                |            |                | | Test      | 2017-01-01 to 2020-08-01 |           |
+|                |            |                | +-----------+--------------------------+           |
++----------------+------------+----------------+----------------------------------------------------+
+
+**Step 5 : Feedback Analysis 🔍**
+
+- Analyze backtest results to assess performance.
+- Incorporate feedback to refine hypotheses and improve the model.
+
+**Step 6 :Hypothesis Refinement ♻️**
+
+- Refine hypotheses based on feedback from backtesting.
+- Repeat the process to continuously improve the model.
+
+⚡ Quick Start
+~~~~~~~~~~~~~~~~~
+
+Please refer to the installation part in :doc:`../installation_and_configuration` to prepare your system dependency.
+
+You can try our demo by running the following command:
+
+- 🐍 Create a Conda Environment
+    
+  - Create a new conda environment with Python (3.10 and 3.11 are well tested in our CI):
+    
+    .. code-block:: sh
+    
+        conda create -n rdagent python=3.10
+
+  - Activate the environment:
+
+    .. code-block:: sh
+
+        conda activate rdagent
+
+- 📦 Install the RDAgent
+  
+  - You can install the RDAgent package from PyPI:
+
+    .. code-block:: sh
+
+        pip install rdagent
+
+- 🚀 Run the Application
+    
+  - Download the financial reports you wish to extract factors from and store them in your preferred folder.
+
+  - Specifically, you can follow this example, or use your own method:
+
+    .. code-block:: sh
+
+        wget https://github.com/SunsetWolf/rdagent_resource/releases/download/reports/all_reports.zip
+        unzip all_reports.zip -d git_ignore_folder/reports
+
+  - Run the application with the following command:
+
+    .. code-block:: sh
+
+        rdagent fin_factor_report --report_folder=git_ignore_folder/reports
+
+  - Alternatively, you can store the paths of the reports in `report_result_json_file_path`. The format should be:
+
+    .. code-block:: json
+
+        [
+            "git_ignore_folder/report/fin_report1.pdf",
+            "git_ignore_folder/report/fin_report2.pdf",
+            "git_ignore_folder/report/fin_report3.pdf"
+        ]
+
+  - Then, run the application using the following command:
+
+    .. code-block:: sh
+
+        rdagent fin_factor_report
+
+🛠️ Usage of modules
+~~~~~~~~~~~~~~~~~~~~~
+
+.. _Env Config: 
+
+- **Env Config**
+
+The following environment variables can be set in the `.env` file to customize the application's behavior:
+
+.. autopydantic_settings:: rdagent.app.qlib_rd_loop.conf.FactorFromReportPropSetting
+    :settings-show-field-summary: False
+    :show-inheritance:
+    :exclude-members: Config
+
+.. autopydantic_settings:: rdagent.components.coder.factor_coder.config.FactorCoSTEERSettings
+    :settings-show-field-summary: False
+    :members: coder_use_cache, data_folder, data_folder_debug, file_based_execution_timeout, select_method, max_loop, knowledge_base_path, new_knowledge_base_path
+    :exclude-members: Config, python_bin, fail_task_trial_limit, v1_query_former_trace_limit, v1_query_similar_success_limit, v2_query_component_limit, v2_query_error_limit, v2_query_former_trace_limit, v2_error_summary, v2_knowledge_sampler
+    :no-index:

docs/scens/data_science.rst

@@ -0,0 +1,184 @@
+.. _data_science_agent:
+
+=======================
+Data Science Agent
+=======================
+
+**🤖 Automated Feature Engineering & Model Tuning Evolution**
+------------------------------------------------------------------------------------------
+The Data Science Agent is an agent that can automatically perform feature engineering and model tuning. It can be used to solve various data science problems, such as image classification, time series forecasting, and text classification.
+
+🧭 Example Guide
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+- 🔧 **Set up RD-Agent Environment**
+
+  - Before you start, please make sure you have installed RD-Agent and configured the environment for RD-Agent correctly. If you want to know how to install and configure the RD-Agent, please refer to the `documentation <../installation_and_configuration.html>`_.
+
+- 🔩 **Setting the Environment variables at .env file**
+
+  - Determine the path where the data will be stored and add it to the ``.env`` file.
+
+  .. code-block:: sh
+
+    dotenv set DS_LOCAL_DATA_PATH <your local directory>/ds_data
+    dotenv set DS_SCEN rdagent.scenarios.data_science.scen.DataScienceScen
+
+- 📥 **Prepare Competition Data**
+
+  - Data Science competition data typically consists of three components: a competition description file (in Markdown format), the competition dataset, and evaluation scripts. For reference, an example of a custom user-defined dataset is provided in ``rdagent/scenarios/data_science/example``.
+
+    - **Correct directory structure (Here is an example of competition data with id custom_data)**
+
+      .. code-block:: text
+
+        ds_data
+        └── eval
+        | └── custom_data
+        |    └── grade.py
+        |    └── valid.py
+        |    └── test.csv
+        └── custom_data
+          └── train.csv
+          └── test.csv
+          └── sample_submission.csv
+          └── description.md
+          └── sample.py
+        
+      - ``ds_data/custom_data/train.csv:`` Necessary training data in csv or parquet format, or training images.
+
+      - ``ds_data/custom_data/description.md:`` (Optional) Competition description file.
+
+      - ``ds_data/custom_data/sample_submission.csv:`` (Optional) Competition sample submission file.
+
+      - ``ds_data/custom_data/sample.py:`` (Optional) Sample code for generating debug data from the competition dataset. If not provided, R&D-Agent will use its default sampling logic. For details, see the ``create_debug_data`` function in ``rdagent/scenarios/data_science/debug/data.py``.
+
+      - ``ds_data/eval/custom_data/grade.py:`` (Optional) Competition grade script, in order to calculate the score for the submission.
+
+      - ``ds_data/eval/custom_data/valid.py:`` (Optional) Competition validation script, in order to check if the submission format is correct.
+
+      - ``ds_data/eval/custom_data/submission_test.csv:`` (Optional) Competition test label file.
+
+- 🔧 **Set up Environment for Custom User-defined Dataset**
+
+  .. code-block:: sh
+
+      dotenv set DS_SCEN rdagent.scenarios.data_science.scen.DataScienceScen
+      dotenv set DS_LOCAL_DATA_PATH rdagent/scenarios/data_science/example
+      dotenv set DS_IF_USING_MLE_DATA False
+      dotenv set DS_CODER_ON_WHOLE_PIPELINE True
+      dotenv set DS_CODER_COSTEER_ENV_TYPE docker
+
+- 🚀 **Run the Application**
+
+  - You can directly run the application by using the following command:
+    
+    .. code-block:: sh
+
+        rdagent data_science --competition <Competition ID>
+
+  - Then, you can run the test set score corresponding to each round of the loop.
+
+    .. code-block:: sh
+
+        dotenv run -- python rdagent/log/mle_summary.py grade <url_to_log>
+
+    Here, <url_to_log> refers to the parent directory of the log folder generated during the run.
+
+- 📥 **Visualize the R&D Process**
+
+  - We provide a web UI to visualize the log. You just need to run:
+
+    .. code-block:: sh
+
+        streamlit run rdagent/log/ui/dsapp.py
+
+  - Then you can input the log path and visualize the R&D process.
+
+🔍 MLE-bench Guide: Running ML Engineering via MLE-bench
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+- 📝 **MLE-bench Overview**
+
+  - MLE-bench is a comprehensive benchmark designed to evaluate the ML engineering capabilities of AI systems using real-world scenarios. The dataset comprises 75 Kaggle competitions. Since Kaggle does not provide held-out test sets for these competitions, the benchmark includes preparation scripts that split the publicly available training data into new training and test sets, and grading scripts are provided for each competition to accurately evaluate submission scores.
+
+- 🔧 **Set up Environment for MLE-bench**
+
+  - Running R&D-Agent on MLE-bench is designed for full automation. There is no need for manual downloads and data preparation. Simply set the environment variable ``DS_IF_USING_MLE_DATA`` to True.  
+
+  - At runtime, R&D-Agent will automatically build the Docker image specified at ``rdagent/scenarios/kaggle/docker/mle_bench_docker/Dockerfile``. This image is responsible for downloading the required datasets and grading files for MLE-bench.  
+  
+  - Note: The first run may take longer than subsequent runs as the Docker image and data are being downloaded and set up for the first time.
+
+    .. code-block:: sh
+
+        dotenv set DS_LOCAL_DATA_PATH <your local directory>/ds_data
+        dotenv set DS_IF_USING_MLE_DATA True
+
+- 🔨 **Configuring the Kaggle API**
+
+  - Downloading Kaggle competition data requires the Kaggle API. You can set up the Kaggle API by following these steps:
+  
+    - Register and login on the `Kaggle <https://www.kaggle.com/>`_ website.
+
+    - Click on the avatar (usually in the top right corner of the page) -> ``Settings`` -> ``Create New Token``, A file called ``kaggle.json`` will be downloaded.
+
+    - Move ``kaggle.json`` to ``~/.config/kaggle/``
+
+    - Modify the permissions of the ``kaggle.json`` file.
+
+      .. code-block:: sh
+
+        chmod 600 ~/.config/kaggle/kaggle.json
+
+  - For more information about Kaggle API Settings, refer to the `Kaggle API <https://github.com/Kaggle/kaggle-api>`_.
+
+
+- 🔩 **Setting the Environment Variables for MLE-bench**
+
+  - In addition to auto-downloading the benchmark data, you must also configure the runtime environment for executing the competition code.  
+  - Use the environment variable ``DS_CODER_COSTEER_ENV_TYPE`` to select the execution mode:
+    
+    • When set to docker (the default), RD-Agent utilizes the official Kaggle Docker image (``gcr.io/kaggle-gpu-images/python:latest``) to ensure that all required packages are available.  
+    • If you prefer to use a custom Docker setup, you can modify the configuration using ``DS_DOCKER_IMAGE`` or ``DS_DOCKERFILE_FOLDER_PATH``.  
+    • Alternatively, if your competition work only demands basic libraries, you may set ``DS_CODER_COSTEER_ENV_TYPE`` to conda. In this mode, you must create a local conda environment named “kaggle” and pre-install the necessary packages. RD-Agent will execute the competition code within this “kaggle” conda environment.
+
+    .. code-block:: sh
+
+      # Configure the runtime environment: choice between 'docker' (default) or 'conda'
+      dotenv set DS_CODER_COSTEER_ENV_TYPE docker
+
+- 🚀 **Run the Application**
+
+  - You can directly run the application by using the following command:
+    
+    .. code-block:: sh
+
+        rdagent data_science --competition <Competition ID>
+
+- 📥 **Visualize the R&D Process**
+
+  - We provide a web UI to visualize the log. You just need to run:
+
+    .. code-block:: sh
+
+        streamlit run rdagent/log/ui/dsapp.py
+
+  - Then you can input the log path and visualize the R&D process.
+
+- **Additional Guidance**
+
+  - **Combine different LLM Models at R&D Stage**
+
+    - You can combine different LLM models at the R&D stage. 
+
+    - By default, when you set environment variable ``CHAT_MODEL``, it covers both R&D stages. When customizing the model for the development stage, you can set:
+    
+    .. code-block:: sh
+
+      # This example sets the model to "o3-mini". For some models, the reasoning effort shoule be set to "None".
+      dotenv set LITELLM_CHAT_MODEL_MAP '{"coding":{"model":"o3-mini","reasoning_effort":"high"},"running":{"model":"o3-mini","reasoning_effort":"high"}}'
+
+
+
+

docs/scens/model_agent_fin.rst

@@ -0,0 +1,150 @@
+.. _model_agent_fin:
+
+=======================
+Finance Model Agent
+=======================
+
+**🤖 Automated Quantitative Trading & Iterative Model Evolution**
+------------------------------------------------------------------------------------------
+
+📖 Background
+~~~~~~~~~~~~~~
+In the realm of quantitative finance, both factor discovery and model development play crucial roles in driving performance. 
+While much attention is often given to the discovery of new financial factors, the **models** that leverage these factors are equally important. 
+The effectiveness of a quantitative strategy depends not only on the factors used but also on how well these factors are integrated into robust, predictive models.
+
+However, the process of developing and optimizing these models can be labor-intensive and complex, requiring continuous refinement and adaptation to ever-changing market conditions. 
+And this is where the **Finance Model Agent** steps in.
+
+
+🎥 `Demo <https://rdagent.azurewebsites.net/model_loop>`_
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. raw:: html
+
+    <div style="display: flex; justify-content: center; align-items: center;">
+      <video width="600" controls>
+        <source src="https://rdagent.azurewebsites.net/media/d85e8cab1da1cd3501d69ce837452f53a971a24911eae7bfa9237137.mp4" type="video/mp4">
+        Your browser does not support the video tag.
+      </video>
+    </div>
+
+
+🌟 Introduction
+~~~~~~~~~~~~~~~~
+
+In this scenario, our automated system proposes hypothesis, constructs model, implements code, conducts back-testing, and utilizes feedback in a continuous, iterative process.
+
+The goal is to automatically optimize performance metrics within the Qlib library, ultimately discovering the most efficient code through autonomous research and development.
+
+Here's an enhanced outline of the steps:
+
+**Step 1 : Hypothesis Generation 🔍**
+
+- Generate and propose initial hypotheses based on previous experiment analysis and domain expertise, with thorough reasoning and financial justification.
+
+**Step 2 : Model Creation ✨**
+
+- Transform the hypothesis into a task.
+- Develop, define, and implement a quantitative model, including its name, description, and formulation.
+
+**Step 3 : Model Implementation 👨‍💻**
+
+- Implement the model code based on the detailed description.
+- Evolve the model iteratively as a developer would, ensuring accuracy and efficiency.
+
+**Step 4 : Backtesting with Qlib 📉**
+
+- Conduct backtesting using the newly developed model and 20 factors extracted from Alpha158 in Qlib.
+- Evaluate the model's effectiveness and performance.
+
++----------------+------------+------------------------+----------------------------------------------------+
+| Dataset        | Model      | Factors                | Data Split                                         |
++================+============+========================+====================================================+
+| CSI300         | RDAgent-dev| 20 factors (Alpha158)  | +-----------+--------------------------+           |
+|                |            |                        | | Train     | 2008-01-01 to 2014-12-31 |           |
+|                |            |                        | +-----------+--------------------------+           |
+|                |            |                        | | Valid     | 2015-01-01 to 2016-12-31 |           |
+|                |            |                        | +-----------+--------------------------+           |
+|                |            |                        | | Test      | 2017-01-01 to 2020-08-01 |           |
+|                |            |                        | +-----------+--------------------------+           |
++----------------+------------+------------------------+----------------------------------------------------+
+
+**Step 5 : Feedback Analysis 🔍**
+
+- Analyze backtest results to assess performance.
+- Incorporate feedback to refine hypotheses and improve the model.
+
+**Step 6 :Hypothesis Refinement ♻️**
+
+- Refine hypotheses based on feedback from backtesting.
+- Repeat the process to continuously improve the model.
+
+⚡ Quick Start
+~~~~~~~~~~~~~~~~~
+
+Please refer to the installation part in :doc:`../installation_and_configuration` to prepare your system dependency.
+
+You can try our demo by running the following command:
+
+- 🐍 Create a Conda Environment
+
+  - Create a new conda environment with Python (3.10 and 3.11 are well tested in our CI):
+
+    .. code-block:: sh
+    
+        conda create -n rdagent python=3.10
+
+  - Activate the environment:
+
+    .. code-block:: sh
+
+        conda activate rdagent
+
+- 📦 Install the RDAgent
+    
+  - You can install the RDAgent package from PyPI:
+
+    .. code-block:: sh
+
+        pip install rdagent
+
+- 🚀 Run the Application
+    
+  - You can directly run the application by using the following command:
+    
+    .. code-block:: sh
+
+        rdagent fin_model
+
+🛠️ Usage of modules
+~~~~~~~~~~~~~~~~~~~~~
+
+.. _Env Config: 
+
+- **Env Config**
+
+The following environment variables can be set in the `.env` file to customize the application's behavior:
+
+.. autopydantic_settings:: rdagent.app.qlib_rd_loop.conf.ModelBasePropSetting
+    :settings-show-field-summary: False
+    :exclude-members: Config
+
+- **Qlib Config**
+    - The `config.yaml` file located in the `model_template` folder contains the relevant configurations for running the developed model in Qlib. The default settings include key information such as:
+        - **market**: Specifies the market, which is set to `csi300`.
+        - **fields_group**: Defines the fields group, with the value `feature`.
+        - **col_list**: A list of columns used, including various indicators such as `RESI5`, `WVMA5`, `RSQR5`, and others.
+        - **start_time**: The start date for the data, set to `2008-01-01`.
+        - **end_time**: The end date for the data, set to `2020-08-01`.
+        - **fit_start_time**: The start date for fitting the model, set to `2008-01-01`.
+        - **fit_end_time**: The end date for fitting the model, set to `2014-12-31`.
+
+    - The default hyperparameters used in the configuration are as follows:
+        - **n_epochs**: The number of epochs, set to `100`.
+        - **lr**: The learning rate, set to `1e-3`.
+        - **early_stop**: The early stopping criterion, set to `10`.
+        - **batch_size**: The batch size, set to `2000`.
+        - **metric**: The evaluation metric, set to `loss`.
+        - **loss**: The loss function, set to `mse`.
+        - **n_jobs**: The number of parallel jobs, set to `20`.

docs/scens/model_copilot_general.rst

@@ -0,0 +1,99 @@
+.. _model_copilot_general:
+
+======================
+General Model Copilot
+======================
+
+**🤖 Automated Model Research & Development Co-Pilot**
+--------------------------------------------------------
+
+📖 Background
+~~~~~~~~~~~~~~
+In the fast-paced field of artificial intelligence, the number of academic papers published each year is skyrocketing. 
+These papers introduce new models, techniques, and approaches that can significantly advance the state of the art. 
+However, reproducing and implementing these models can be a daunting task, requiring substantial time and expertise. 
+Researchers often face challenges in extracting the essential details from these papers and converting them into functional code.
+And this is where the **General Model Copilot** steps in.
+
+🎥 `Demo <https://rdagent.azurewebsites.net/report_model>`_
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. raw:: html
+
+    <div style="display: flex; justify-content: center; align-items: center;">
+      <video width="600" controls>
+        <source src="https://rdagent.azurewebsites.net/media/b35f904765b05099b0fcddbebe041a04f4d7bde239657e5fc24bf0cc.mp4" type="video/mp4">
+        Your browser does not support the video tag.
+      </video>
+    </div>
+
+🌟 Introduction
+~~~~~~~~~~~~~~~~
+In this scenario, our automated system proposes hypotheses, constructs models, implements code, performs back-testing, and uses feedback to iterate continuously. The system aims to automatically optimize performance metrics from the Qlib library, finding the best code through autonomous research and development.
+
+Model R&D CoPilot Scenario
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+**Overview**
+
+This demo automates the extraction and iterative development of models from academic papers, ensuring functionality and correctness. This scenario automates the development of PyTorch models by reading academic papers or other sources. It supports various data types, including tabular, time-series, and graph data. The primary workflow involves two main components: the Reader and the Coder.
+
+**Workflow Components**
+
+1. **Reader**
+   - Parses and extracts relevant model information from academic papers or sources, including architectures, parameters, and implementation details.
+   - Uses Large Language Models to convert content into a structured format for the Coder.
+
+2. **Evolving Coder**
+   - Translates structured information from the Reader into executable PyTorch code.
+   - Utilizes an evolving coding mechanism to ensure correct tensor shapes, verified with sample input tensors.
+   - Iteratively refines the code to align with source material specifications.
+
+**Supported Data Types**
+
+- **Tabular Data:** Structured data with rows and columns, such as spreadsheets or databases.
+- **Time-Series Data:** Sequential data points indexed in time order, useful for forecasting and temporal pattern recognition.
+- **Graph Data:** Data structured as nodes and edges, suitable for network analysis and relational tasks.
+
+⚡ Quick Start
+~~~~~~~~~~~~~~~~~
+
+Please refer to the installation part in :doc:`../installation_and_configuration` to prepare your system dependency.
+
+You can try our demo by running the following command:
+
+- 🐍 Create a Conda Environment
+  
+  - Create a new conda environment with Python (3.10 and 3.11 are well tested in our CI):
+
+    .. code-block:: sh
+    
+        conda create -n rdagent python=3.10
+
+  - Activate the environment:
+
+    .. code-block:: sh
+
+        conda activate rdagent
+
+- 📦 Install the RDAgent
+    
+  - You can install the RDAgent package from PyPI:
+
+    .. code-block:: sh
+
+        pip install rdagent
+
+
+- 🚀 Run the Application
+    
+  - Prepare relevant files (in pdf format) by uploading papers to the directory below and copy the path as report_file_path.
+      
+    .. code-block:: sh
+
+        rdagent/scenarios/general_model
+    
+  - Run the following command in your terminal within the same virtual environment:
+  
+    .. code-block:: sh
+
+        rdagent general_model --report_file_path=<path_to_pdf_file>

docs/scens/quant_agent_fin.rst

@@ -0,0 +1,113 @@
+.. _quant_agent_fin:
+
+=====================
+Finance Quant Agent
+=====================
+
+
+**🥇The First Data-Centric Quant Multi-Agent Framework RD-Agent(Q)**
+---------------------------------------------------------------------
+
+R&D-Agent for Quantitative Finance, in short **RD-Agent(Q)**, is the first data-centric, multi-agent framework designed to automate the full-stack research and development of quantitative strategies via coordinated factor-model co-optimization.
+
+You can learn more details about **RD-Agent(Q)** through the `paper <https://arxiv.org/abs/2505.15155>`_.
+
+⚡ Quick Start
+~~~~~~~~~~~~~~~~~
+
+Before you start, please make sure you have installed RD-Agent and configured the environment for RD-Agent correctly. If you want to know how to install and configure the RD-Agent, please refer to the `documentation <../installation_and_configuration.html>`_.
+
+Then, you can run the framework by running the following command:
+
+- 🐍 Create a Conda Environment
+
+  - Create a new conda environment with Python (3.10 and 3.11 are well tested in our CI):
+
+    .. code-block:: sh
+
+          conda create -n rdagent python=3.10
+
+  - Activate the environment:
+
+    .. code-block:: sh
+
+        conda activate rdagent
+
+- 📦 Install the RDAgent
+  
+  - You can install the RDAgent package from PyPI:
+
+    .. code-block:: sh
+
+        pip install rdagent
+
+- 🚀 Run the Application
+    
+  - You can directly run the application by using the following command:
+    
+    .. code-block:: sh
+
+        rdagent fin_quant
+
+
+🛠️ Usage of modules
+~~~~~~~~~~~~~~~~~~~~~
+
+.. _Env Config: 
+
+- **Env Config**
+
+The following environment variables can be set in the `.env` file to customize the application's behavior:
+
+.. autopydantic_settings:: rdagent.app.qlib_rd_loop.conf.QuantBasePropSetting
+    :settings-show-field-summary: False
+    :exclude-members: Config
+
+.. autopydantic_settings:: rdagent.components.coder.factor_coder.config.FactorCoSTEERSettings
+    :settings-show-field-summary: False
+    :members: coder_use_cache, data_folder, data_folder_debug, file_based_execution_timeout, select_method, max_loop, knowledge_base_path, new_knowledge_base_path
+    :exclude-members: Config, fail_task_trial_limit, v1_query_former_trace_limit, v1_query_similar_success_limit, v2_query_component_limit, v2_query_error_limit, v2_query_former_trace_limit, v2_error_summary, v2_knowledge_sampler
+    :no-index:
+
+- **Qlib Configuration**
+    - The `.yaml` files in both the `model_template` and `factor_template` directories contain some configurations for running the corresponding models or factors within the Qlib framework. Below is an overview of their contents and roles:
+        - **General Settings**:
+            - **provider_uri**: Specifies the local Qlib data path, set to `~/.qlib/qlib_data/cn_data`.
+            - **market**: Configured to `csi300`, representing the CSI 300 index constituents.
+            - **benchmark**: Set to `SH000300`, used for backtesting evaluation.
+        
+        - **Data Handling**:
+            - **start_time** and **end_time**: Define the full data range, from `2008-01-01` to `2022-08-01`.
+            - **fit_start_time**: The start date for fitting the model, set to `2008-01-01`.
+            - **fit_end_time**: The end date for fitting the model, set to `2014-12-31`.
+            - **features and labels**: Generated via a nested data loader combining `Alpha158DL` (for engineered features such as `RESI5`, `WVMA5`, `RSQR5`, `KLEN`, etc.) and a `StaticDataLoader` that loads precomputed factor files (`combined_factors_df.parquet`).
+            -  **normalization**: The pipeline includes `RobustZScoreNorm` (with clipping) and `Fillna` for inference, and `DropnaLabel` with `CSZScoreNorm` for training.
+        
+        - **Training Configuration**:
+            - **Model**: Uses `GeneralPTNN`, a PyTorch-based neural network model.
+            - **Dataset Splits**:
+                - **train**: `2008-01-01` to `2014-12-31`
+                - **valid**: `2015-01-01` to `2016-12-31`
+                - **test**: `2017-01-01` to `2020-08-01`
+
+        - **Default Hyperparameters** (can be overridden by command-line arguments):
+            - **n_epochs**: `100`
+            - **lr**: `2e-4`
+            - **early_stop**: `10`
+            - **batch_size**: `256`
+            - **weight_decay**: `0.0`
+            - **metric**: `loss`
+            - **loss**: `mse`
+            - **n_jobs**: `20`
+            - **GPU**: `0` (uses GPU 0 if available)
+            
+        - **Backtesting and Evaluation**:
+            - **strategy**: `TopkDropoutStrategy`, which selects the top 50 stocks and randomly drops 5 to introduce exploration.
+            - **backtest period**: `2017-01-01` to `2020-08-01`
+            - **initial capital**: `100,000,000`
+            - **cost configuration**: Includes open/close costs, minimum transaction costs, and slippage control.
+            
+        - **Recording and Analysis**:
+            - **SignalRecord**: Logs predicted signals.
+            - **SigAnaRecord**: Performs signal analysis without long-short separation.
+            - **PortAnaRecord**: Conducts portfolio analysis using the configured strategy and backtest settings.

docs/ui.rst

@@ -0,0 +1,49 @@
+==============
+User Interface
+==============
+
+
+Introduction
+============
+
+RD-Agent will generate some logs during the R&D process. These logs are very useful for debugging and understanding the R&D process. However, just viewing the terminal log is not intuitive enough. RD-Agent provides a web app as UI to visualize the R&D process. You can easily view the R&D process and understand the R&D process better.
+
+A Quick Demo
+============
+
+Start Web App
+-------------
+
+In `RD-Agent/` folder, run:
+
+.. code-block:: bash
+
+    rdagent ui --port <port> --log_dir <log_dir like "log/"> [--debug]
+
+This will start a web app on `http://localhost:<port>`.
+
+**NOTE**: The log_dir parameter is not required. You can manually enter the log_path in the web app. If you set the log_dir parameter, you can easily select a different log_path in the web app.
+
+--debug is optional, it will show a "Single Step Run" button in sidebar and saved objects info in the web app.
+
+Use Web App
+-----------
+
+1. Open the sidebar.
+
+.. TODO: update these
+
+2. Select the scenario you want to show. There are some pre-defined scenarios:
+    - Qlib Model
+    - Qlib Factor
+    - Data Mining
+    - Model from Paper
+    - Kaggle
+
+3. Click the `Config⚙️` button and input the log path (if you set the log_dir parameter, you can select a log_path in the dropdown list).
+
+4. Click the buttons below Config⚙️ to show the scenario execution process. Buttons are:
+    - All Loops: Show complete scenario execution process.
+    - Next Loop: Show one success **R&D Loop**.
+    - One Evolving: Show one **evolving** step of **development** part.
+    - refresh logs: clear shown logs.

pyproject.toml

@@ -4,7 +4,6 @@ requires = [
   "setuptools",
   "setuptools-scm",
 ]
-root= "rdagent"
 
 [project]
 authors = [
@@ -16,8 +15,6 @@ classifiers = [
   "Operating System :: OS Independent",
   "Programming Language :: Python :: 3.10",
   "Programming Language :: Python :: 3.11",
-  "Programming Language :: Python :: 3.8",
-  "Programming Language :: Python :: 3.9",
 ]
 description = "Research & Development Agent"
 dynamic = [
@@ -27,13 +24,15 @@ dynamic = [
 ]
 keywords = [
   "Autonomous Agents",
-  "Research and Development",
   "Large Language Models",
+  "Research and Development",
 ]
 name = "rdagent"
 readme = "README.md"
-requires-python = ">=3.8"
+requires-python = ">=3.10"
 
+[project.scripts]
+rdagent = "rdagent.app.cli:app"
 
 [project.urls]
 homepage = "https://github.com/microsoft/RD-Agent/"
@@ -52,16 +51,20 @@ color_output = true
 profile = "black"
 
 [tool.mypy]
-explicit_package_bases = true
 check_untyped_defs = true
 disallow_any_unimported = true
 disallow_untyped_defs = true
 enable_error_code = [
   "ignore-without-code",
 ]
+explicit_package_bases = true
 warn_return_any = true
 warn_unused_ignores = true
 
+[[tool.mypy.overrides]]
+ignore_missing_imports = true
+module = "llama"
+
 [tool.pytest.ini_options]
 addopts = "-l -s --durations=0"
 log_cli = true
@@ -78,10 +81,10 @@ src = ["rdagent"]
 [tool.ruff.lint]
 ignore = [
   # https://docs.astral.sh/ruff/rules/#pydocstyle-d
-  "ANN101",
   "ANN401",
   "D",
   "ERA001",
+  "EXE002",
   "FIX",
   "INP001",
   "PGH",
@@ -89,7 +92,7 @@ ignore = [
   "S101",
   "S301",
   "T20",
-  "TCH003",
+  "TC003",
   "TD",
 ]
 select = ["ALL"]
@@ -111,6 +114,8 @@ package = {file = ["requirements/package.txt"]}
 test = {file = ["requirements/test.txt"]}
 
 [tool.setuptools_scm]
+local_scheme = "no-local-version"
+version_scheme = "guess-next-dev"
 
 [tool.tomlsort]
 all = true

rdagent/app/CI/README.md

@@ -0,0 +1,38 @@
+# CI 检查
+
+`.github/workflows/ci.yml`配置了提交时自动运行`Makefile`: 91~103行的命令，可以在这调整执行的命令
+
+在`.env`中设置`USE_CHAT_CACHE=True`可以让第二次修复快一些
+
+# Rules
+
+`pyproject.toml`中配置全局屏蔽的规则
+- ruff: `[tool.ruff.lint].ignore`
+- mypy: `[tool.mypy]`
+
+## ruff rules
+ruff rules 比较好修改, 大多可以自动修复
+
+对于一些规则可以在代码中添加注释来局部屏蔽, 例如添加 `# noqa E234,ANN001`
+遇到的不好修改的规则:
+- 捕获异常时应该处理每一种异常，不应该统一当作`Exception`处理
+- `subprogress()` 调用命令应该先判断命令是否安全
+- ...
+
+规则列表: [ruff rules](https://docs.astral.sh/ruff/rules/)
+
+## mypy rules
+
+Mypy检查Python中类型标注, 常遇到需要修改结构/同时修改其他文件的情况, 自动修复效果不好
+
+局部屏蔽: `# type: ignore`
+
+规则列表: [mypy rules](https://mypy.readthedocs.io/en/stable/error_code_list.html)
+
+# Optimization (Maybe)
+
+- 添加指定文件夹检查的功能
+- 增加一个修改选项: 调用`vim`, 用户直接修改此部分代码
+- 显示时把`Original Code`部分去掉, 直接在输出的表示修改的diff部分用`^^^^^^`在代码行下标注出错误位置，这样能更直观地观察错误修复情况
+- 当前为线性执行完所有修复后交给用户检查, 可修改成 后台多线程 / 进程处理修复的任务, 终端实时展示处理完的修复让用户检查
+- ...

rdagent/app/CI/run.py

@@ -13,17 +13,6 @@ from pathlib import Path
 from typing import Any, Literal
 
 import tree_sitter_python
-from rdagent.core.evolving_framework import (
-    Evaluator,
-    EvoAgent,
-    EvolvableSubjects,
-    EvolvingStrategy,
-    EvoStep,
-    Feedback,
-    Knowledge,
-)
-from rdagent.core.prompts import Prompts
-from rdagent.oai.llm_utils import APIBackend
 from rich import print
 from rich.panel import Panel
 from rich.progress import Progress, SpinnerColumn, TimeElapsedColumn
@@ -34,6 +23,18 @@ from rich.table import Table
 from rich.text import Text
 from tree_sitter import Language, Node, Parser
 
+from rdagent.core.evaluation import Evaluator
+from rdagent.core.evolving_agent import EvoAgent
+from rdagent.core.evolving_framework import (
+    EvolvableSubjects,
+    EvolvingStrategy,
+    EvoStep,
+    Feedback,
+    Knowledge,
+)
+from rdagent.core.prompts import Prompts
+from rdagent.oai.llm_utils import APIBackend
+
 py_parser = Parser(Language(tree_sitter_python.language()))
 CI_prompts = Prompts(file_path=Path(__file__).parent / "prompts.yaml")
 
@@ -303,7 +304,7 @@ class RuffEvaluator(Evaluator):
 
         return RuffRule(**json.loads(out))
 
-    def evaluate(self, evo: Repo, **kwargs: Any) -> CIFeedback:  # noqa: ARG002
+    def evaluate(self, evo: Repo, **kwargs: dict) -> CIFeedback:
         """Simply run ruff to get the feedbacks."""
         try:
             out = subprocess.check_output(
@@ -355,14 +356,13 @@ class RuffEvaluator(Evaluator):
 
 
 class MypyEvaluator(Evaluator):
-
     def __init__(self, command: str | None = None) -> None:
         if command is None:
             self.command = "mypy . --pretty --no-error-summary --show-column-numbers"
         else:
             self.command = command
 
-    def evaluate(self, evo: Repo, **kwargs: Any) -> CIFeedback:  # noqa: ARG002
+    def evaluate(self, evo: Repo, **kwargs: dict) -> CIFeedback:
         try:
             out = subprocess.check_output(
                 shlex.split(self.command),  # noqa: S603
@@ -411,12 +411,10 @@ class MypyEvaluator(Evaluator):
 
 
 class MultiEvaluator(Evaluator):
-
     def __init__(self, *evaluators: Evaluator) -> None:
         self.evaluators = evaluators
 
-    def evaluate(self, evo: Repo, **kwargs: Any) -> CIFeedback:
-
+    def evaluate(self, evo: Repo, **kwargs: dict) -> CIFeedback:
         all_errors = defaultdict(list)
         for evaluator in self.evaluators:
             feedback: CIFeedback = evaluator.evaluate(evo, **kwargs)
@@ -435,10 +433,9 @@ class CIEvoStr(EvolvingStrategy):
         self,
         evo: Repo,
         evolving_trace: list[EvoStep] | None = None,
-        knowledge_l: list[Knowledge] | None = None,  # noqa: ARG002
-        **kwargs: Any,  # noqa: ARG002
+        knowledge_l: list[Knowledge] | None = None,
+        **kwargs: dict,
     ) -> Repo:
-
         @dataclass
         class CodeFixGroup:
             start_line: int
@@ -552,7 +549,7 @@ class CIEvoStr(EvolvingStrategy):
                         style="bright_blue",
                         align="left",
                         characters=".",
-                    )
+                    ),
                 )
 
                 file = evo.files[evo.project_path / Path(file_path)]
@@ -633,12 +630,16 @@ class CIEvoStr(EvolvingStrategy):
                         for i in diff:
                             if i.startswith("+"):
                                 table.add_row(
-                                    "", Text(str(diff_new_lineno), style="green bold"), Text(i, style="green")
+                                    "",
+                                    Text(str(diff_new_lineno), style="green bold"),
+                                    Text(i, style="green"),
                                 )
                                 diff_new_lineno += 1
                             elif i.startswith("-"):
                                 table.add_row(
-                                    Text(str(diff_original_lineno), style="red bold"), "", Text(i, style="red")
+                                    Text(str(diff_original_lineno), style="red bold"),
+                                    "",
+                                    Text(i, style="red"),
                                 )
                                 diff_original_lineno += 1
                             elif i.startswith("?"):
@@ -658,7 +659,7 @@ class CIEvoStr(EvolvingStrategy):
                         operation = Prompt.ask(
                             "Input your operation [ [red]([bold]s[/bold])kip[/red] / "
                             "[green]([bold]a[/bold])pply[/green] / "
-                            "[yellow]manual instruction[/yellow] ]"
+                            "[yellow]manual instruction[/yellow] ]",
                         )
                         print()
                         if operation in ("s", "skip"):
@@ -692,6 +693,22 @@ class CIEvoStr(EvolvingStrategy):
         return evo
 
 
+class CIEvoAgent(EvoAgent):
+    def __init__(self, evolving_strategy: CIEvoStr) -> None:
+        super().__init__(max_loop=1, evolving_strategy=evolving_strategy)
+        self.evolving_trace = []
+
+    def multistep_evolve(self, evo: Repo, eva: Evaluator) -> Repo:
+        evo = self.evolving_strategy.evolve(
+            evo=evo,
+            evolving_trace=self.evolving_trace,
+        )
+
+        self.evolving_trace.append(EvoStep(evo, feedback=eva.evaluate(evo)))
+
+        return evo
+
+
 DIR = None
 while DIR is None or not DIR.exists():
     DIR = Prompt.ask("Please input the [cyan]project directory[/cyan]")
@@ -710,12 +727,11 @@ repo = Repo(DIR, excludes=excludes)
 # evaluator = MultiEvaluator(MypyEvaluator(), RuffEvaluator())
 evaluator = RuffEvaluator()
 estr = CIEvoStr()
-rag = None  # RAG is not enable firstly.
-ea = EvoAgent(estr, rag=rag)
-ea.step_evolving(repo, evaluator)
+ea = CIEvoAgent(estr)
+ea.multistep_evolve(repo, evaluator)
 while True:
     print(Rule(f"Round {len(ea.evolving_trace)} repair", style="blue"))
-    repo: Repo = ea.step_evolving(repo, evaluator)
+    repo: Repo = ea.multistep_evolve(repo, evaluator)
 
     fix_records = repo.fix_records
     filename = f"{DIR.name}_{start_timestamp}_round_{len(ea.evolving_trace)}_fix_records.json"

rdagent/app/benchmark/factor/analysis.py

@@ -0,0 +1,225 @@
+import json
+import pickle
+from pathlib import Path
+
+import fire
+import matplotlib.pyplot as plt
+import numpy as np
+import pandas as pd
+import seaborn as sns
+
+from rdagent.components.benchmark.conf import BenchmarkSettings
+from rdagent.components.benchmark.eval_method import FactorImplementEval
+
+
+class BenchmarkAnalyzer:
+    def __init__(self, settings, only_correct_format=False):
+        self.settings = settings
+        self.index_map = self.load_index_map()
+        self.only_correct_format = only_correct_format
+
+    def load_index_map(self):
+        index_map = {}
+        with open(self.settings.bench_data_path, "r") as file:
+            factor_dict = json.load(file)
+        for factor_name, data in factor_dict.items():
+            index_map[factor_name] = (factor_name, data["Category"], data["Difficulty"])
+        return index_map
+
+    def load_data(self, file_path):
+        file_path = Path(file_path)
+        if not (file_path.is_file() and file_path.suffix == ".pkl"):
+            raise ValueError("Invalid file path")
+
+        with file_path.open("rb") as f:
+            res = pickle.load(f)
+
+        return res
+
+    def process_results(self, results):
+        final_res = {}
+        for experiment, path in results.items():
+            data = self.load_data(path)
+            summarized_data = FactorImplementEval.summarize_res(data)
+            processed_data = self.analyze_data(summarized_data)
+            final_res[experiment] = processed_data.iloc[-1, :]
+        return final_res
+
+    def reformat_index(self, display_df):
+        """
+        reform the results from
+
+        .. code-block:: python
+
+                              success rate
+            High_Beta_Factor           0.2
+
+        to
+
+        .. code-block:: python
+
+                                                    success rate
+            Category Difficulty Factor
+            量价       Hard       High_Beta_Factor           0.2
+
+        """
+        new_idx = []
+        display_df = display_df[display_df.index.isin(self.index_map.keys())]
+        for idx in display_df.index:
+            new_idx.append(self.index_map[idx])
+
+        display_df.index = pd.MultiIndex.from_tuples(
+            new_idx,
+            names=["Factor", "Category", "Difficulty"],
+        )
+        display_df = display_df.swaplevel(0, 2).swaplevel(0, 1).sort_index(axis=0)
+
+        return display_df.sort_index(
+            key=lambda x: [{"Easy": 0, "Medium": 1, "Hard": 2, "New Discovery": 3}.get(i, i) for i in x]
+        )
+
+    def result_all_key_order(self, x):
+        order_v = []
+        for i in x:
+            order_v.append(
+                {
+                    "Avg Run SR": 0,
+                    "Avg Format SR": 1,
+                    "Avg Correlation": 2,
+                    "Max Correlation": 3,
+                    "Max Accuracy": 4,
+                    "Avg Accuracy": 5,
+                }.get(i, i),
+            )
+        return order_v
+
+    def analyze_data(self, sum_df):
+        index = [
+            "FactorSingleColumnEvaluator",
+            "FactorRowCountEvaluator",
+            "FactorIndexEvaluator",
+            "FactorEqualValueRatioEvaluator",
+            "FactorCorrelationEvaluator",
+            "run factor error",
+        ]
+        sum_df = sum_df.reindex(index, axis=0)
+        sum_df_clean = sum_df.T.groupby(level=0).apply(lambda x: x.reset_index(drop=True))
+
+        run_error = sum_df_clean["run factor error"].unstack().T.fillna(False).astype(bool)
+        succ_rate = ~run_error
+        succ_rate = succ_rate.mean(axis=0).to_frame("success rate")
+
+        succ_rate_f = self.reformat_index(succ_rate)
+
+        # if it rasis Error when running the evaluator, we will get NaN
+        # Running failures are reguarded to zero score.
+        format_issue = sum_df_clean[["FactorRowCountEvaluator", "FactorIndexEvaluator"]].apply(
+            lambda x: np.mean(x.fillna(0.0)), axis=1
+        )
+        format_succ_rate = format_issue.unstack().T.mean(axis=0).to_frame("success rate")
+        format_succ_rate_f = self.reformat_index(format_succ_rate)
+
+        corr = sum_df_clean["FactorCorrelationEvaluator"].fillna(0.0)
+        if self.only_correct_format:
+            corr = corr.loc[format_issue == 1.0]
+
+        corr_res = corr.unstack().T.mean(axis=0).to_frame("corr(only success)")
+        corr_res = self.reformat_index(corr_res)
+
+        corr_max = corr.unstack().T.max(axis=0).to_frame("corr(only success)")
+        corr_max_res = self.reformat_index(corr_max)
+
+        value_max = sum_df_clean["FactorEqualValueRatioEvaluator"]
+        value_max = value_max.unstack().T.max(axis=0).to_frame("max_value")
+        value_max_res = self.reformat_index(value_max)
+
+        value_avg = (
+            (sum_df_clean["FactorEqualValueRatioEvaluator"] * format_issue)
+            .unstack()
+            .T.mean(axis=0)
+            .to_frame("avg_value")
+        )
+        value_avg_res = self.reformat_index(value_avg)
+
+        result_all = pd.concat(
+            {
+                "Avg Correlation": corr_res.iloc[:, 0],
+                "Avg Format SR": format_succ_rate_f.iloc[:, 0],
+                "Avg Run SR": succ_rate_f.iloc[:, 0],
+                "Max Correlation": corr_max_res.iloc[:, 0],
+                "Max Accuracy": value_max_res.iloc[:, 0],
+                "Avg Accuracy": value_avg_res.iloc[:, 0],
+            },
+            axis=1,
+        )
+
+        df = result_all.sort_index(axis=1, key=self.result_all_key_order).sort_index(axis=0)
+        print(df)
+
+        print()
+        print(df.groupby("Category").mean())
+
+        print()
+        print(df.mean())
+
+        # Calculate the mean of each column
+        mean_values = df.fillna(0.0).mean()
+        mean_df = pd.DataFrame(mean_values).T
+
+        # Assign the MultiIndex to the DataFrame
+        mean_df.index = pd.MultiIndex.from_tuples([("-", "-", "Average")], names=["Factor", "Category", "Difficulty"])
+
+        # Append the mean values to the end of the dataframe
+        df_w_mean = pd.concat([df, mean_df]).astype("float")
+
+        return df_w_mean
+
+
+class Plotter:
+    @staticmethod
+    def change_fs(font_size):
+        plt.rc("font", size=font_size)
+        plt.rc("axes", titlesize=font_size)
+        plt.rc("axes", labelsize=font_size)
+        plt.rc("xtick", labelsize=font_size)
+        plt.rc("ytick", labelsize=font_size)
+        plt.rc("legend", fontsize=font_size)
+        plt.rc("figure", titlesize=font_size)
+
+    @staticmethod
+    def plot_data(data, file_name, title):
+        plt.figure(figsize=(10, 10))
+        plt.ylabel("Value")
+        colors = ["#3274A1", "#E1812C", "#3A923A", "#C03D3E"]
+        plt.bar(data["a"], data["b"], color=colors, capsize=5)
+        for idx, row in data.iterrows():
+            plt.text(idx, row["b"] + 0.01, f"{row['b']:.2f}", ha="center", va="bottom")
+        plt.suptitle(title, y=0.98)
+        plt.xticks(rotation=45)
+        plt.ylim(0, 1)
+        plt.tight_layout()
+        plt.savefig(file_name)
+
+
+def main(
+    path="git_ignore_folder/eval_results/res_promptV220240724-060037.pkl",
+    round=1,
+    title="Comparison of Different Methods",
+    only_correct_format=False,
+):
+    settings = BenchmarkSettings()
+    benchmark = BenchmarkAnalyzer(settings, only_correct_format=only_correct_format)
+    results = {
+        f"{round} round experiment": path,
+    }
+    final_results = benchmark.process_results(results)
+    final_results_df = pd.DataFrame(final_results)
+
+    Plotter.change_fs(20)
+    plot_data = final_results_df.drop(["Max Accuracy", "Avg Accuracy"], axis=0).T
+    plot_data = plot_data.reset_index().melt("index", var_name="a", value_name="b")
+    Plotter.plot_data(plot_data, "./comparison_plot.png", title)
+
+
+if __name__ == "__main__":
+    fire.Fire(main)

rdagent/app/benchmark/factor/eval.py

@@ -0,0 +1,35 @@
+from rdagent.app.qlib_rd_loop.conf import FACTOR_PROP_SETTING
+from rdagent.components.benchmark.conf import BenchmarkSettings
+from rdagent.components.benchmark.eval_method import FactorImplementEval
+from rdagent.core.scenario import Scenario
+from rdagent.core.utils import import_class
+from rdagent.log import rdagent_logger as logger
+from rdagent.scenarios.qlib.factor_experiment_loader.json_loader import (
+    FactorTestCaseLoaderFromJsonFile,
+)
+
+if __name__ == "__main__":
+    # 1.read the settings
+    bs = BenchmarkSettings()
+
+    # 2.read and prepare the eval_data
+    test_cases = FactorTestCaseLoaderFromJsonFile().load(bs.bench_data_path)
+
+    # 3.declare the method to be tested and pass the arguments.
+
+    scen: Scenario = import_class(FACTOR_PROP_SETTING.scen)()
+    generate_method = import_class(bs.bench_method_cls)(scen=scen, **bs.bench_method_extra_kwargs)
+    # 4.declare the eval method and pass the arguments.
+    eval_method = FactorImplementEval(
+        method=generate_method,
+        test_cases=test_cases,
+        scen=scen,
+        catch_eval_except=True,
+        test_round=bs.bench_test_round,
+    )
+
+    # 5.run the eval
+    res = eval_method.eval(eval_method.develop())
+
+    # 6.save the result
+    logger.log_object(res)

rdagent/app/benchmark/model/README.md

@@ -0,0 +1,30 @@
+# Tasks
+
+## Task Extraction
+From paper to task.
+```bash
+# python rdagent/app/model_implementation/task_extraction.py
+# It may based on rdagent/document_reader/document_reader.py
+python rdagent/components/task_implementation/model_implementation/task_extraction.py ./PaperImpBench/raw_paper/
+```
+
+## Complete workflow
+From paper to implementation
+``` bash
+# Similar to
+# rdagent/app/factor_extraction_and_implementation/factor_extract_and_implement.py
+```
+
+## Paper benchmark
+```bash
+# TODO: it does not work well now.
+python rdagent/app/model_implementation/eval.py
+```
+
+TODO:
+- Create reasonable benchmark
+  - with uniform input
+  - manually create task
+- Create reasonable evaluation metrics
+
+## Evolving

rdagent/app/benchmark/model/eval.py

@@ -0,0 +1,42 @@
+from pathlib import Path
+
+from rdagent.components.coder.model_coder import ModelCoSTEER
+from rdagent.components.loader.task_loader import ModelTaskLoaderJson, ModelWsLoader
+from rdagent.scenarios.qlib.experiment.model_experiment import (
+    QlibModelExperiment,
+    QlibModelScenario,
+)
+
+if __name__ == "__main__":
+    DIRNAME = Path(__file__).absolute().resolve().parent
+
+    from rdagent.components.coder.model_coder.benchmark.eval import ModelImpValEval
+    from rdagent.components.coder.model_coder.one_shot import ModelCodeWriter
+
+    bench_folder = DIRNAME.parent.parent.parent / "components" / "coder" / "model_coder" / "benchmark"
+    mtl = ModelTaskLoaderJson(str(bench_folder / "model_dict.json"))
+
+    task_l = mtl.load()
+
+    task_l = [t for t in task_l if t.name == "A-DGN"]  # FIXME: other models does not work well
+
+    model_experiment = QlibModelExperiment(sub_tasks=task_l)
+    # mtg = ModelCodeWriter(scen=QlibModelScenario())
+    mtg = ModelCoSTEER(scen=QlibModelScenario())
+
+    model_experiment = mtg.develop(model_experiment)
+
+    # TODO: Align it with the benchmark framework after @wenjun's refine the evaluation part.
+    # Currently, we just handcraft a workflow for fast evaluation.
+
+    mil = ModelWsLoader(bench_folder / "gt_code")
+
+    mie = ModelImpValEval()
+    # Evaluation:
+    eval_l = []
+    for impl in model_experiment.sub_workspace_list:
+        print(impl.target_task)
+        gt_impl = mil.load(impl.target_task)
+        eval_l.append(mie.evaluate(gt_impl, impl))
+
+    print(eval_l)

rdagent/app/cli.py

@@ -0,0 +1,68 @@
+"""
+CLI entrance for all rdagent application.
+
+This will
+- make rdagent a nice entry and
+- autoamtically load dotenv
+"""
+
+from dotenv import load_dotenv
+
+load_dotenv(".env")
+# 1) Make sure it is at the beginning of the script so that it will load dotenv before initializing BaseSettings.
+# 2) The ".env" argument is necessary to make sure it loads `.env` from the current directory.
+
+import subprocess
+from importlib.resources import path as rpath
+
+import fire
+
+from rdagent.app.data_science.loop import main as data_science
+from rdagent.app.general_model.general_model import (
+    extract_models_and_implement as general_model,
+)
+from rdagent.app.qlib_rd_loop.factor import main as fin_factor
+from rdagent.app.qlib_rd_loop.factor_from_report import main as fin_factor_report
+from rdagent.app.qlib_rd_loop.model import main as fin_model
+from rdagent.app.qlib_rd_loop.quant import main as fin_quant
+from rdagent.app.utils.health_check import health_check
+from rdagent.app.utils.info import collect_info
+
+
+def ui(port=19899, log_dir="", debug=False):
+    """
+    start web app to show the log traces.
+    """
+    with rpath("rdagent.log.ui", "app.py") as app_path:
+        cmds = ["streamlit", "run", app_path, f"--server.port={port}"]
+        if log_dir or debug:
+            cmds.append("--")
+        if log_dir:
+            cmds.append(f"--log_dir={log_dir}")
+        if debug:
+            cmds.append("--debug")
+        subprocess.run(cmds)
+
+
+def server_ui(port=19899):
+    """
+    start web app to show the log traces in real time
+    """
+    subprocess.run(["python", "rdagent/log/server/app.py", f"--port={port}"])
+
+
+def app():
+    fire.Fire(
+        {
+            "fin_factor": fin_factor,
+            "fin_factor_report": fin_factor_report,
+            "fin_model": fin_model,
+            "fin_quant": fin_quant,
+            "general_model": general_model,
+            "ui": ui,
+            "health_check": health_check,
+            "collect_info": collect_info,
+            "data_science": data_science,
+            "server_ui": server_ui,
+        }
+    )

rdagent/app/data_science/conf.py

@@ -0,0 +1,115 @@
+from typing import Literal
+
+from pydantic_settings import SettingsConfigDict
+
+from rdagent.app.kaggle.conf import KaggleBasePropSetting
+
+
+class DataScienceBasePropSetting(KaggleBasePropSetting):
+    # TODO: Kaggle Setting should be the subclass of DataScience
+    model_config = SettingsConfigDict(env_prefix="DS_", protected_namespaces=())
+
+    # Main components
+    ## Scen
+    scen: str = "rdagent.scenarios.data_science.scen.KaggleScen"
+    """
+    Scenario class for data science tasks.
+    - For Kaggle competitions, use: "rdagent.scenarios.data_science.scen.KaggleScen"
+    - For custom data science scenarios, use: "rdagent.scenarios.data_science.scen.DataScienceScen"
+    """
+
+    hypothesis_gen: str = "rdagent.scenarios.data_science.proposal.exp_gen.proposal.DSProposalV2ExpGen"
+    """Hypothesis generation class"""
+
+    ## Workflow Related
+    consecutive_errors: int = 5
+
+    ## Coding Related
+    coding_fail_reanalyze_threshold: int = 3
+
+    debug_timeout: int = 600
+    """The timeout limit for running on debugging data"""
+    full_timeout: int = 3600
+    """The timeout limit for running on full data"""
+
+    ### specific feature
+
+    #### enable specification
+    spec_enabled: bool = True
+
+    #### proposal related
+    proposal_version: str = "v1"
+    coder_on_whole_pipeline: bool = False
+    max_trace_hist: int = 3
+
+    coder_max_loop: int = 10
+    runner_max_loop: int = 1
+
+    rule_base_eval: bool = False
+    sample_data: bool = True
+    use_raw_description: bool = False
+    show_nan_columns: bool = False
+
+    #### model dump
+    enable_model_dump: bool = False
+    enable_doc_dev: bool = False
+    model_dump_check_level: Literal["medium", "high"] = "medium"
+
+    ### knowledge base
+    enable_knowledge_base: bool = False
+    knowledge_base_version: str = "v1"
+    knowledge_base_path: str | None = None
+    idea_pool_json_path: str | None = None
+
+    ### archive log folder after each loop
+    enable_log_archive: bool = True
+    log_archive_path: str | None = None
+    log_archive_temp_path: str | None = (
+        None  # This is to store the mid tar file since writing the tar file is preferred in local storage then copy to target storage
+    )
+
+    #### Evaluation on Test related
+    eval_sub_dir: str = "eval"  # TODO: fixme, this is not a good name
+    """We'll use f"{DS_RD_SETTING.local_data_path}/{DS_RD_SETTING.eval_sub_dir}/{competition}"
+    to find the scriipt to evaluate the submission on test"""
+
+    """---below are the settings for multi-trace---"""
+
+    ### multi-trace related
+    max_trace_num: int = 3
+    """The maximum number of traces to grow before merging"""
+
+    #### multi-trace:checkpoint selector
+    selector_name: str = "rdagent.scenarios.data_science.proposal.exp_gen.ckp_select.LatestCKPSelector"
+    """The name of the selector to use"""
+    sota_count_window: int = 5
+    """The number of trials to consider for SOTA count"""
+    sota_count_threshold: int = 1
+    """The threshold for SOTA count"""
+
+    #### multi-trace: SOTA experiment selector
+    sota_exp_selector_name: str = "rdagent.scenarios.data_science.proposal.exp_gen.sota_exp_select.GlobalSOTASelector"
+    """The name of the SOTA experiment selector to use"""
+
+    ### multi-trace:inject optimals for multi-trace
+    # inject diverse when start a new sub-trace
+    enable_inject_diverse: bool = False
+
+    # inject knowledge at the root of the trace
+    enable_inject_knowledge_at_root: bool = False
+
+    # enable different version of DSExpGen for multi-trace
+    enable_multi_version_exp_gen: bool = False
+    exp_gen_version_list: str = "v3,v2"
+
+    #### multi-trace: time for final multi-trace merge
+    merge_hours: int = 2
+    """The time for merge"""
+
+    #### multi-trace: max SOTA-retrieved number, used in AutoSOTAexpSelector
+    # constrains the number of SOTA experiments to retrieve, otherwise too many SOTA experiments to retrieve will cause the exceed of the context window of LLM
+    max_sota_retrieved_num: int = 10
+    """The maximum number of SOTA experiments to retrieve in a LLM call"""
+
+
+DS_RD_SETTING = DataScienceBasePropSetting()

rdagent/app/data_science/debug.py

@@ -0,0 +1,6 @@
+import fire
+
+from rdagent.scenarios.data_science.debug.data import create_debug_data
+
+if __name__ == "__main__":
+    fire.Fire(create_debug_data)

rdagent/app/data_science/loop.py

@@ -0,0 +1,74 @@
+import asyncio
+from pathlib import Path
+
+import fire
+
+from rdagent.app.data_science.conf import DS_RD_SETTING
+from rdagent.core.utils import import_class
+from rdagent.log import rdagent_logger as logger
+from rdagent.scenarios.data_science.loop import DataScienceRDLoop
+
+
+def main(
+    path: str | None = None,
+    checkout: bool | str | Path = True,
+    step_n: int | None = None,
+    loop_n: int | None = None,
+    competition="bms-molecular-translation",
+    timeout=None,
+    replace_timer=True,
+    exp_gen_cls: str | None = None,
+):
+    """
+
+    Parameters
+    ----------
+    path :
+        A path like `$LOG_PATH/__session__/1/0_propose`. This indicates that we restore the state after finishing step 0 in loop 1.
+    checkout :
+        Used only when a path is provided.
+        Can be True, False, or a path.
+        Default is True.
+        - If True, the new loop will use the existing folder and clear logs for sessions after the one corresponding to the given path.
+        - If False, the new loop will use the existing folder but keep the logs for sessions after the one corresponding to the given path.
+        - If a path (or a str like Path) is provided, the new loop will be saved to that path, leaving the original path unchanged.
+    step_n :
+        Number of steps to run; if None, the process will run indefinitely until an error or KeyboardInterrupt occurs.
+    loop_n :
+        Number of loops to run; if None, the process will run indefinitely until an error or KeyboardInterrupt occurs.
+        - If the current loop is incomplete, it will be counted as the first loop for completion.
+        - If both step_n and loop_n are provided, the process will stop as soon as either condition is met.
+    competition :
+        Competition name.
+    replace_timer :
+        If a session is loaded, determines whether to replace the timer with session.timer.
+    exp_gen_cls :
+        When there are different stages, the exp_gen can be replaced with the new proposal.
+
+
+    Auto R&D Evolving loop for models in a Kaggle scenario.
+    You can continue running a session by using the command:
+    .. code-block:: bash
+        dotenv run -- python rdagent/app/data_science/loop.py [--competition titanic] $LOG_PATH/__session__/1/0_propose  --step_n 1   # `step_n` is an optional parameter
+        rdagent kaggle --competition playground-series-s4e8  # This command is recommended.
+    """
+    if competition is not None:
+        DS_RD_SETTING.competition = competition
+
+    if not DS_RD_SETTING.competition:
+        logger.error("Please specify competition name.")
+
+    if path is None:
+        kaggle_loop = DataScienceRDLoop(DS_RD_SETTING)
+    else:
+        kaggle_loop: DataScienceRDLoop = DataScienceRDLoop.load(path, checkout=checkout, replace_timer=replace_timer)
+
+    # replace exp_gen if we have new class
+    if exp_gen_cls is not None:
+        kaggle_loop.exp_gen = import_class(exp_gen_cls)(kaggle_loop.exp_gen.scen)
+
+    asyncio.run(kaggle_loop.run(step_n=step_n, loop_n=loop_n, all_duration=timeout))
+
+
+if __name__ == "__main__":
+    fire.Fire(main)

rdagent/app/factor_extraction_and_implementation/factor_extract_and_implement.py

@@ -1,16 +0,0 @@
-# %%
-from dotenv import load_dotenv
-from rdagent.factor_implementation.CoSTEER import CoSTEERFG
-from rdagent.factor_implementation.task_loader.pdf_loader import FactorImplementationTaskLoaderFromPDFfiles
-
-assert load_dotenv()
-
-
-def extract_factors_and_implement(report_file_path: str) -> None:
-    factor_tasks = FactorImplementationTaskLoaderFromPDFfiles().load(report_file_path)
-    implementation_result = CoSTEERFG().generate(factor_tasks)
-    return implementation_result
-
-
-if __name__ == "__main__":
-    extract_factors_and_implement("/home/xuyang1/workspace/report.pdf")

rdagent/app/factor_implementation/eval_implement.py

@@ -1,30 +0,0 @@
-from rdagent.core.conf import BenchmarkSettings
-from rdagent.core.utils import import_class
-from rdagent.benchmark.eval_method import FactorImplementEval
-from rdagent.benchmark.data_process import load_eval_data
-
-# 1.read the settings
-bs = BenchmarkSettings()
-
-# 2.read and prepare the eval_data
-test_cases = load_eval_data(bs.bench_version)
-
-# 3.declare the method to be tested and pass the arguments.
-# TODO: Whether it is necessary to define two Eval method classes for two data type?
-method_cls = import_class(bs.bench_method_cls)
-generate_method = method_cls(bs.bench_method_extra_kwargs)
-
-# 4.declare the eval method and pass the arguments.
-eval_method = FactorImplementEval(
-    method=generate_method,
-    test_cases=test_cases,
-    catch_eval_except=True,
-    test_round=bs.bench_test_round,
-)
-
-# 5.run the eval
-eval_method.eval()
-
-# 6.save the result
-eval_method.save(output_path = bs.bench_result_path)
-

rdagent/app/general_model/general_model.py

@@ -0,0 +1,45 @@
+import fire
+
+from rdagent.components.coder.model_coder.task_loader import (
+    ModelExperimentLoaderFromPDFfiles,
+)
+from rdagent.components.document_reader.document_reader import (
+    extract_first_page_screenshot_from_pdf,
+)
+from rdagent.log import rdagent_logger as logger
+from rdagent.scenarios.general_model.scenario import GeneralModelScenario
+from rdagent.scenarios.qlib.developer.model_coder import QlibModelCoSTEER
+
+
+def extract_models_and_implement(report_file_path: str) -> None:
+    """
+    This is a research copilot to automatically implement models from a report file or paper.
+
+    It extracts models from a given PDF report file and implements the necessary operations.
+
+    Parameters:
+    report_file_path (str): The path to the report file. The file must be a PDF file.
+
+    Example URLs of PDF reports:
+    - https://arxiv.org/pdf/2210.09789
+    - https://arxiv.org/pdf/2305.10498
+    - https://arxiv.org/pdf/2110.14446
+    - https://arxiv.org/pdf/2205.12454
+    - https://arxiv.org/pdf/2210.16518
+
+    Returns:
+    None
+    """
+    scenario = GeneralModelScenario()
+    logger.log_object(scenario, tag="scenario")
+    # Save Relevant Images
+    img = extract_first_page_screenshot_from_pdf(report_file_path)
+    logger.log_object(img, tag="pdf_image")
+    exp = ModelExperimentLoaderFromPDFfiles().load(report_file_path)
+    logger.log_object(exp, tag="load_experiment")
+    exp = QlibModelCoSTEER(scenario).develop(exp)
+    logger.log_object(exp, tag="developed_experiment")
+
+
+if __name__ == "__main__":
+    fire.Fire(extract_models_and_implement)

rdagent/app/kaggle/conf.py

@@ -0,0 +1,79 @@
+from pydantic_settings import SettingsConfigDict
+
+from rdagent.core.conf import ExtendedBaseSettings
+
+
+class KaggleBasePropSetting(ExtendedBaseSettings):
+    model_config = SettingsConfigDict(env_prefix="KG_", protected_namespaces=())
+
+    # 1) overriding the default
+    scen: str = "rdagent.scenarios.kaggle.experiment.scenario.KGScenario"
+    """Scenario class for data mining model"""
+
+    hypothesis_gen: str = "rdagent.scenarios.kaggle.proposal.proposal.KGHypothesisGen"
+    """Hypothesis generation class"""
+
+    hypothesis2experiment: str = "rdagent.scenarios.kaggle.proposal.proposal.KGHypothesis2Experiment"
+    """Hypothesis to experiment class"""
+
+    feature_coder: str = "rdagent.scenarios.kaggle.developer.coder.KGFactorCoSTEER"
+    """Feature Coder class"""
+
+    model_feature_selection_coder: str = "rdagent.scenarios.kaggle.developer.coder.KGModelFeatureSelectionCoder"
+    """Model Feature Selection Coder class"""
+
+    model_coder: str = "rdagent.scenarios.kaggle.developer.coder.KGModelCoSTEER"
+    """Model Coder class"""
+
+    feature_runner: str = "rdagent.scenarios.kaggle.developer.runner.KGFactorRunner"
+    """Feature Runner class"""
+
+    model_runner: str = "rdagent.scenarios.kaggle.developer.runner.KGModelRunner"
+    """Model Runner class"""
+
+    summarizer: str = "rdagent.scenarios.kaggle.developer.feedback.KGExperiment2Feedback"
+    """Summarizer class"""
+
+    evolving_n: int = 10
+    """Number of evolutions"""
+
+    competition: str = ""
+    """Kaggle competition name, e.g., 'sf-crime'"""
+
+    template_path: str = "rdagent/scenarios/kaggle/experiment/templates"
+    """Kaggle competition base templates path"""
+
+    local_data_path: str = ""
+    """Folder storing Kaggle competition data"""
+
+    # Evaluation on Test related
+    if_using_mle_data: bool = False
+    auto_submit: bool = False
+    """Automatically upload and submit each experiment result to Kaggle platform"""
+
+    # Conditionally set the knowledge_base based on the use of graph RAG
+    knowledge_base: str = ""
+    """Knowledge base class, uses 'KGKnowledgeGraph' when advanced graph-based RAG is enabled, otherwise empty."""
+    if_action_choosing_based_on_UCB: bool = False
+    """Enable decision mechanism based on UCB algorithm"""
+
+    domain_knowledge_path: str = "/data/userdata/share/kaggle/domain_knowledge"
+    """Folder storing domain knowledge files in .case format"""
+
+    knowledge_base_path: str = "kg_graph.pkl"
+    """Advanced version of graph-based RAG"""
+
+    rag_path: str = "git_ignore_folder/kaggle_vector_base.pkl"
+    """Base version of vector-based RAG"""
+
+    if_using_vector_rag: bool = False
+    """Enable basic vector-based RAG"""
+
+    if_using_graph_rag: bool = False
+    """Enable advanced graph-based RAG"""
+
+    mini_case: bool = False
+    """Enable mini-case study for experiments"""
+
+
+KAGGLE_IMPLEMENT_SETTING = KaggleBasePropSetting()

rdagent/app/kaggle/loop.py

@@ -0,0 +1,139 @@
+import subprocess
+from typing import Any
+
+import fire
+
+from rdagent.app.kaggle.conf import KAGGLE_IMPLEMENT_SETTING
+from rdagent.components.workflow.conf import BasePropSetting
+from rdagent.components.workflow.rd_loop import RDLoop
+from rdagent.core.developer import Developer
+from rdagent.core.exception import CoderError, FactorEmptyError, ModelEmptyError
+from rdagent.core.proposal import (
+    Experiment2Feedback,
+    Hypothesis2Experiment,
+    HypothesisGen,
+)
+from rdagent.core.scenario import Scenario
+from rdagent.core.utils import import_class
+from rdagent.log import rdagent_logger as logger
+from rdagent.scenarios.kaggle.experiment.scenario import (
+    KG_ACTION_FEATURE_ENGINEERING,
+    KG_ACTION_FEATURE_PROCESSING,
+    KG_ACTION_MODEL_FEATURE_SELECTION,
+)
+from rdagent.scenarios.kaggle.experiment.utils import python_files_to_notebook
+from rdagent.scenarios.kaggle.kaggle_crawler import download_data
+from rdagent.scenarios.kaggle.proposal.proposal import KGTrace
+
+
+class KaggleRDLoop(RDLoop):
+    def __init__(self, PROP_SETTING: BasePropSetting):
+        scen: Scenario = import_class(PROP_SETTING.scen)(PROP_SETTING.competition)
+        logger.log_object(scen, tag="scenario")
+        knowledge_base = (
+            import_class(PROP_SETTING.knowledge_base)(PROP_SETTING.knowledge_base_path, scen)
+            if PROP_SETTING.knowledge_base != ""
+            else None
+        )
+        logger.log_object(knowledge_base, tag="knowledge_base")
+        self.hypothesis_gen: HypothesisGen = import_class(PROP_SETTING.hypothesis_gen)(scen)
+        logger.log_object(self.hypothesis_gen, tag="hypothesis generator")
+        self.hypothesis2experiment: Hypothesis2Experiment = import_class(PROP_SETTING.hypothesis2experiment)()
+        logger.log_object(self.hypothesis2experiment, tag="hypothesis2experiment")
+        self.feature_coder: Developer = import_class(PROP_SETTING.feature_coder)(scen)
+        logger.log_object(self.feature_coder, tag="feature coder")
+        self.model_feature_selection_coder: Developer = import_class(PROP_SETTING.model_feature_selection_coder)(scen)
+        logger.log_object(self.model_feature_selection_coder, tag="model feature selection coder")
+        self.model_coder: Developer = import_class(PROP_SETTING.model_coder)(scen)
+        logger.log_object(self.model_coder, tag="model coder")
+        self.feature_runner: Developer = import_class(PROP_SETTING.feature_runner)(scen)
+        logger.log_object(self.feature_runner, tag="feature runner")
+        self.model_runner: Developer = import_class(PROP_SETTING.model_runner)(scen)
+        logger.log_object(self.model_runner, tag="model runner")
+        self.summarizer: Experiment2Feedback = import_class(PROP_SETTING.summarizer)(scen)
+        logger.log_object(self.summarizer, tag="summarizer")
+        self.trace = KGTrace(scen=scen, knowledge_base=knowledge_base)
+        super(RDLoop, self).__init__()
+
+    def coding(self, prev_out: dict[str, Any]):
+        if prev_out["direct_exp_gen"]["propose"].action in [
+            KG_ACTION_FEATURE_ENGINEERING,
+            KG_ACTION_FEATURE_PROCESSING,
+        ]:
+            exp = self.feature_coder.develop(prev_out["direct_exp_gen"]["exp_gen"])
+        elif prev_out["direct_exp_gen"]["propose"].action == KG_ACTION_MODEL_FEATURE_SELECTION:
+            exp = self.model_feature_selection_coder.develop(prev_out["direct_exp_gen"]["exp_gen"])
+        else:
+            exp = self.model_coder.develop(prev_out["direct_exp_gen"]["exp_gen"])
+        logger.log_object(exp.sub_workspace_list, tag="coder result")
+        return exp
+
+    def running(self, prev_out: dict[str, Any]):
+        if prev_out["direct_exp_gen"]["propose"].action in [
+            KG_ACTION_FEATURE_ENGINEERING,
+            KG_ACTION_FEATURE_PROCESSING,
+        ]:
+            exp = self.feature_runner.develop(prev_out["coding"])
+        else:
+            exp = self.model_runner.develop(prev_out["coding"])
+        logger.log_object(exp, tag="runner result")
+        if KAGGLE_IMPLEMENT_SETTING.competition in [
+            "optiver-realized-volatility-prediction",
+            "covid19-global-forecasting-week-1",
+        ]:
+            try:
+                python_files_to_notebook(KAGGLE_IMPLEMENT_SETTING.competition, exp.experiment_workspace.workspace_path)
+            except Exception as e:
+                logger.error(f"Merge python files to one file failed: {e}")
+        if KAGGLE_IMPLEMENT_SETTING.auto_submit:
+            csv_path = exp.experiment_workspace.workspace_path / "submission.csv"
+            try:
+                subprocess.run(
+                    [
+                        "kaggle",
+                        "competitions",
+                        "submit",
+                        "-f",
+                        str(csv_path.absolute()),
+                        "-m",
+                        str(csv_path.parent.absolute()),
+                        KAGGLE_IMPLEMENT_SETTING.competition,
+                    ],
+                    check=True,
+                )
+            except subprocess.CalledProcessError as e:
+                logger.error(f"Auto submission failed: \n{e}")
+            except Exception as e:
+                logger.error(f"Other exception when use kaggle api:\n{e}")
+
+        return exp
+
+    skip_loop_error = (ModelEmptyError, FactorEmptyError, CoderError)
+
+
+def main(path=None, step_n=None, competition=None):
+    """
+    Auto R&D Evolving loop for models in a kaggle{} scenario.
+    You can continue running session by
+    .. code-block:: bash
+        dotenv run -- python rdagent/app/kaggle/loop.py [--competition titanic] $LOG_PATH/__session__/1/0_propose  --step_n 1   # `step_n` is a optional parameter
+        rdagent kaggle --competition playground-series-s4e8  # You are encouraged to use this one.
+    """
+    if competition:
+        KAGGLE_IMPLEMENT_SETTING.competition = competition
+        download_data(competition=competition, settings=KAGGLE_IMPLEMENT_SETTING)
+        if KAGGLE_IMPLEMENT_SETTING.if_using_graph_rag:
+            KAGGLE_IMPLEMENT_SETTING.knowledge_base = (
+                "rdagent.scenarios.kaggle.knowledge_management.graph.KGKnowledgeGraph"
+            )
+    else:
+        logger.error("Please specify competition name.")
+    if path is None:
+        kaggle_loop = KaggleRDLoop(KAGGLE_IMPLEMENT_SETTING)
+    else:
+        kaggle_loop = KaggleRDLoop.load(path)
+    kaggle_loop.run(step_n=step_n)
+
+
+if __name__ == "__main__":
+    fire.Fire(main)

rdagent/app/model_implementation/README.md

@@ -1,39 +0,0 @@
-
-# Preparation
-
-## Install Pytorch
-CPU CUDA will be enough for verify the implementation
-
-Please install pytorch based on your system.
-Here is an example on my system
-```bash
-pip3 install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu
-pip3 install torch_geometric
-
-```
-
-# Tasks
-
-## Task Extraction
-From paper to task.
-```bash
-python rdagent/app/model_implementation/task_extraction.py
-# It may based on rdagent/document_reader/document_reader.py
-```
-
-## Complete workflow
-From paper to implementation
-``` bash
-# Similar to
-# rdagent/app/factor_extraction_and_implementation/factor_extract_and_implement.py
-```
-
-## Paper benchmark
-```bash
-python rdagent/app/model_implementation/eval.py
-
-TODO:
-- Is evaluation reasonable
-```
-
-## Evolving

rdagent/app/model_implementation/eval.py

@@ -1,29 +0,0 @@
-from pathlib import Path
-
-DIRNAME = Path(__file__).absolute().resolve().parent
-
-from rdagent.model_implementation.benchmark.eval import ModelImpValEval
-from rdagent.model_implementation.one_shot import ModelTaskGen
-from rdagent.model_implementation.task import ModelImpLoader, ModelTaskLoderJson
-
-mtl = ModelTaskLoderJson("TODO: A Path to json")
-
-task_l = mtl.load()
-
-mtg = ModelTaskGen()
-
-impl_l = mtg.generate(task_l)
-
-# TODO: Align it with the benchmark framework after @wenjun's refine the evaluation part.
-# Currently, we just handcraft a workflow for fast evaluation.
-
-mil = ModelImpLoader(DIRNAME.parent.parent / "model_implementation" / "benchmark" /  "gt_code")
-
-mie = ModelImpValEval()
-# Evaluation:
-eval_l = []
-for impl in impl_l:
-    gt_impl = mil.load(impl.target_task)
-    eval_l.append(mie.evaluate(gt_impl, impl))
-
-print(eval_l)

rdagent/app/qlib_rd_loop/conf.py

@@ -0,0 +1,120 @@
+from pydantic_settings import SettingsConfigDict
+
+from rdagent.components.workflow.conf import BasePropSetting
+
+
+class ModelBasePropSetting(BasePropSetting):
+    model_config = SettingsConfigDict(env_prefix="QLIB_MODEL_", protected_namespaces=())
+
+    # 1) override base settings
+    scen: str = "rdagent.scenarios.qlib.experiment.model_experiment.QlibModelScenario"
+    """Scenario class for Qlib Model"""
+
+    hypothesis_gen: str = "rdagent.scenarios.qlib.proposal.model_proposal.QlibModelHypothesisGen"
+    """Hypothesis generation class"""
+
+    hypothesis2experiment: str = "rdagent.scenarios.qlib.proposal.model_proposal.QlibModelHypothesis2Experiment"
+    """Hypothesis to experiment class"""
+
+    coder: str = "rdagent.scenarios.qlib.developer.model_coder.QlibModelCoSTEER"
+    """Coder class"""
+
+    runner: str = "rdagent.scenarios.qlib.developer.model_runner.QlibModelRunner"
+    """Runner class"""
+
+    summarizer: str = "rdagent.scenarios.qlib.developer.feedback.QlibModelExperiment2Feedback"
+    """Summarizer class"""
+
+    evolving_n: int = 10
+    """Number of evolutions"""
+
+
+class FactorBasePropSetting(BasePropSetting):
+    model_config = SettingsConfigDict(env_prefix="QLIB_FACTOR_", protected_namespaces=())
+
+    # 1) override base settings
+    scen: str = "rdagent.scenarios.qlib.experiment.factor_experiment.QlibFactorScenario"
+    """Scenario class for Qlib Factor"""
+
+    hypothesis_gen: str = "rdagent.scenarios.qlib.proposal.factor_proposal.QlibFactorHypothesisGen"
+    """Hypothesis generation class"""
+
+    hypothesis2experiment: str = "rdagent.scenarios.qlib.proposal.factor_proposal.QlibFactorHypothesis2Experiment"
+    """Hypothesis to experiment class"""
+
+    coder: str = "rdagent.scenarios.qlib.developer.factor_coder.QlibFactorCoSTEER"
+    """Coder class"""
+
+    runner: str = "rdagent.scenarios.qlib.developer.factor_runner.QlibFactorRunner"
+    """Runner class"""
+
+    summarizer: str = "rdagent.scenarios.qlib.developer.feedback.QlibFactorExperiment2Feedback"
+    """Summarizer class"""
+
+    evolving_n: int = 10
+    """Number of evolutions"""
+
+
+class FactorFromReportPropSetting(FactorBasePropSetting):
+    # 1) override the scen attribute
+    scen: str = "rdagent.scenarios.qlib.experiment.factor_from_report_experiment.QlibFactorFromReportScenario"
+    """Scenario class for Qlib Factor from Report"""
+
+    # 2) sub task specific:
+    report_result_json_file_path: str = "git_ignore_folder/report_list.json"
+    """Path to the JSON file listing research reports for factor extraction"""
+
+    max_factors_per_exp: int = 10000
+    """Maximum number of factors implemented per experiment"""
+
+    report_limit: int = 10000
+    """Maximum number of reports to process"""
+
+
+class QuantBasePropSetting(BasePropSetting):
+    model_config = SettingsConfigDict(env_prefix="QLIB_QUANT_", protected_namespaces=())
+
+    # 1) override base settings
+    scen: str = "rdagent.scenarios.qlib.experiment.quant_experiment.QlibQuantScenario"
+    """Scenario class for Qlib Model"""
+
+    quant_hypothesis_gen: str = "rdagent.scenarios.qlib.proposal.quant_proposal.QlibQuantHypothesisGen"
+    """Hypothesis generation class"""
+
+    model_hypothesis2experiment: str = "rdagent.scenarios.qlib.proposal.model_proposal.QlibModelHypothesis2Experiment"
+    """Hypothesis to experiment class"""
+
+    model_coder: str = "rdagent.scenarios.qlib.developer.model_coder.QlibModelCoSTEER"
+    """Coder class"""
+
+    model_runner: str = "rdagent.scenarios.qlib.developer.model_runner.QlibModelRunner"
+    """Runner class"""
+
+    model_summarizer: str = "rdagent.scenarios.qlib.developer.feedback.QlibModelExperiment2Feedback"
+    """Summarizer class"""
+
+    factor_hypothesis2experiment: str = (
+        "rdagent.scenarios.qlib.proposal.factor_proposal.QlibFactorHypothesis2Experiment"
+    )
+    """Hypothesis to experiment class"""
+
+    factor_coder: str = "rdagent.scenarios.qlib.developer.factor_coder.QlibFactorCoSTEER"
+    """Coder class"""
+
+    factor_runner: str = "rdagent.scenarios.qlib.developer.factor_runner.QlibFactorRunner"
+    """Runner class"""
+
+    factor_summarizer: str = "rdagent.scenarios.qlib.developer.feedback.QlibFactorExperiment2Feedback"
+    """Summarizer class"""
+
+    evolving_n: int = 10
+    """Number of evolutions"""
+
+    action_selection: str = "bandit"
+    """Action selection strategy: 'bandit' for bandit-based selection, 'llm' for LLM-based selection, 'random' for random selection"""
+
+
+FACTOR_PROP_SETTING = FactorBasePropSetting()
+FACTOR_FROM_REPORT_PROP_SETTING = FactorFromReportPropSetting()
+MODEL_PROP_SETTING = ModelBasePropSetting()
+QUANT_PROP_SETTING = QuantBasePropSetting()

rdagent/app/qlib_rd_loop/factor.py

@@ -0,0 +1,47 @@
+"""
+Factor workflow with session control
+"""
+
+import asyncio
+from typing import Any
+
+import fire
+
+from rdagent.app.qlib_rd_loop.conf import FACTOR_PROP_SETTING
+from rdagent.components.workflow.rd_loop import RDLoop
+from rdagent.core.exception import FactorEmptyError
+from rdagent.log import rdagent_logger as logger
+
+
+class FactorRDLoop(RDLoop):
+    skip_loop_error = (FactorEmptyError,)
+
+    def running(self, prev_out: dict[str, Any]):
+        exp = self.runner.develop(prev_out["coding"])
+        if exp is None:
+            logger.error(f"Factor extraction failed.")
+            raise FactorEmptyError("Factor extraction failed.")
+        logger.log_object(exp, tag="runner result")
+        return exp
+
+
+def main(path=None, step_n=None, loop_n=None, all_duration=None, checkout=True):
+    """
+    Auto R&D Evolving loop for fintech factors.
+
+    You can continue running session by
+
+    .. code-block:: python
+
+        dotenv run -- python rdagent/app/qlib_rd_loop/factor.py $LOG_PATH/__session__/1/0_propose  --step_n 1   # `step_n` is a optional paramter
+
+    """
+    if path is None:
+        model_loop = FactorRDLoop(FACTOR_PROP_SETTING)
+    else:
+        model_loop = FactorRDLoop.load(path, checkout=checkout)
+    asyncio.run(model_loop.run(step_n=step_n, loop_n=loop_n, all_duration=all_duration))
+
+
+if __name__ == "__main__":
+    fire.Fire(main)

rdagent/app/qlib_rd_loop/factor_from_report.py

@@ -0,0 +1,160 @@
+import asyncio
+import json
+from pathlib import Path
+from typing import Any, Dict, Tuple
+
+import fire
+
+from rdagent.app.qlib_rd_loop.conf import FACTOR_FROM_REPORT_PROP_SETTING
+from rdagent.app.qlib_rd_loop.factor import FactorRDLoop
+from rdagent.components.document_reader.document_reader import (
+    extract_first_page_screenshot_from_pdf,
+    load_and_process_pdfs_by_langchain,
+)
+from rdagent.core.conf import RD_AGENT_SETTINGS
+from rdagent.core.proposal import Hypothesis
+from rdagent.log import rdagent_logger as logger
+from rdagent.oai.llm_utils import APIBackend
+from rdagent.scenarios.qlib.experiment.factor_experiment import QlibFactorExperiment
+from rdagent.scenarios.qlib.factor_experiment_loader.pdf_loader import (
+    FactorExperimentLoaderFromPDFfiles,
+)
+from rdagent.utils.agent.tpl import T
+from rdagent.utils.workflow import LoopMeta
+
+
+def generate_hypothesis(factor_result: dict, report_content: str) -> str:
+    """
+    Generate a hypothesis based on factor results and report content.
+
+    Args:
+        factor_result (dict): The results of the factor analysis.
+        report_content (str): The content of the report.
+
+    Returns:
+        str: The generated hypothesis.
+    """
+    system_prompt = T(".prompts:hypothesis_generation.system").r()
+    user_prompt = T(".prompts:hypothesis_generation.user").r(
+        factor_descriptions=json.dumps(factor_result), report_content=report_content
+    )
+
+    response = APIBackend().build_messages_and_create_chat_completion(
+        user_prompt=user_prompt,
+        system_prompt=system_prompt,
+        json_mode=True,
+        json_target_type=Dict[str, str],
+    )
+
+    response_json = json.loads(response)
+
+    return Hypothesis(
+        hypothesis=response_json.get("hypothesis", "No hypothesis provided"),
+        reason=response_json.get("reason", "No reason provided"),
+        concise_reason=response_json.get("concise_reason", "No concise reason provided"),
+        concise_observation=response_json.get("concise_observation", "No concise observation provided"),
+        concise_justification=response_json.get("concise_justification", "No concise justification provided"),
+        concise_knowledge=response_json.get("concise_knowledge", "No concise knowledge provided"),
+    )
+
+
+def extract_hypothesis_and_exp_from_reports(report_file_path: str) -> QlibFactorExperiment | None:
+    """
+    Extract hypothesis and experiment details from report files.
+
+    Args:
+        report_file_path (str): Path to the report file.
+
+    Returns:
+        QlibFactorExperiment: An instance of QlibFactorExperiment containing the extracted details.
+        None: If no valid experiment is found in the report.
+    """
+    exp = FactorExperimentLoaderFromPDFfiles().load(report_file_path)
+    if exp is None or exp.sub_tasks == []:
+        return None
+
+    pdf_screenshot = extract_first_page_screenshot_from_pdf(report_file_path)
+    logger.log_object(pdf_screenshot, tag="load_pdf_screenshot")
+
+    docs_dict = load_and_process_pdfs_by_langchain(report_file_path)
+
+    factor_result = {
+        task.factor_name: {
+            "description": task.factor_description,
+            "formulation": task.factor_formulation,
+            "variables": task.variables,
+            "resources": task.factor_resources,
+        }
+        for task in exp.sub_tasks
+    }
+
+    report_content = "\n".join(docs_dict.values())
+    hypothesis = generate_hypothesis(factor_result, report_content)
+    exp.hypothesis = hypothesis
+    return exp
+
+
+class FactorReportLoop(FactorRDLoop, metaclass=LoopMeta):
+    def __init__(self, report_folder: str = None):
+        super().__init__(PROP_SETTING=FACTOR_FROM_REPORT_PROP_SETTING)
+        if report_folder is None:
+            self.judge_pdf_data_items = json.load(
+                open(FACTOR_FROM_REPORT_PROP_SETTING.report_result_json_file_path, "r")
+            )
+        else:
+            self.judge_pdf_data_items = [i for i in Path(report_folder).rglob("*.pdf")]
+
+        self.loop_n = min(len(self.judge_pdf_data_items), FACTOR_FROM_REPORT_PROP_SETTING.report_limit)
+        self.shift_report = (
+            0  # some reports does not contain viable factor, so we ship some of them to avoid infinite loop
+        )
+
+    async def direct_exp_gen(self, prev_out: dict[str, Any]):
+        while True:
+            if self.get_unfinished_loop_cnt(self.loop_idx) < RD_AGENT_SETTINGS.get_max_parallel():
+                report_file_path = self.judge_pdf_data_items[self.loop_idx + self.shift_report]
+                logger.info(f"Processing number {self.loop_idx} report: {report_file_path}")
+                exp = extract_hypothesis_and_exp_from_reports(str(report_file_path))
+                if exp is None:
+                    self.shift_report += 1
+                    self.loop_n -= 1
+                    if self.loop_n < 0:  # NOTE: on every step, we self.loop_n -= 1 at first.
+                        raise self.LoopTerminationError("Reach stop criterion and stop loop")
+                    continue
+                exp.based_experiments = [QlibFactorExperiment(sub_tasks=[], hypothesis=exp.hypothesis)] + [
+                    t[0] for t in self.trace.hist if t[1]
+                ]
+                exp.sub_workspace_list = exp.sub_workspace_list[: FACTOR_FROM_REPORT_PROP_SETTING.max_factors_per_exp]
+                exp.sub_tasks = exp.sub_tasks[: FACTOR_FROM_REPORT_PROP_SETTING.max_factors_per_exp]
+                logger.log_object(exp.hypothesis, tag="hypothesis generation")
+                logger.log_object(exp.sub_tasks, tag="experiment generation")
+                return exp
+            await asyncio.sleep(1)
+
+    def coding(self, prev_out: dict[str, Any]):
+        exp = self.coder.develop(prev_out["direct_exp_gen"])
+        logger.log_object(exp.sub_workspace_list, tag="coder result")
+        return exp
+
+
+def main(report_folder=None, path=None, all_duration=None, checkout=True):
+    """
+    Auto R&D Evolving loop for fintech factors (the factors are extracted from finance reports).
+
+    Args:
+        report_folder (str, optional): The folder contains the report PDF files. Reports will be loaded from this folder.
+        path (str, optional): The path for loading a session. If provided, the session will be loaded.
+        step_n (int, optional): Step number to continue running a session.
+    """
+    if path is None and report_folder is None:
+        model_loop = FactorReportLoop()
+    elif path is not None:
+        model_loop = FactorReportLoop.load(path, checkout=checkout)
+    else:
+        model_loop = FactorReportLoop(report_folder=report_folder)
+
+    asyncio.run(model_loop.run(all_duration=all_duration))
+
+
+if __name__ == "__main__":
+    fire.Fire(main)

rdagent/app/qlib_rd_loop/model.py

@@ -0,0 +1,37 @@
+"""
+Model workflow with session control
+"""
+
+import asyncio
+
+import fire
+
+from rdagent.app.qlib_rd_loop.conf import MODEL_PROP_SETTING
+from rdagent.components.workflow.rd_loop import RDLoop
+from rdagent.core.exception import ModelEmptyError
+
+
+class ModelRDLoop(RDLoop):
+    skip_loop_error = (ModelEmptyError,)
+
+
+def main(path=None, step_n=None, loop_n=None, all_duration=None, checkout=True):
+    """
+    Auto R&D Evolving loop for fintech models
+
+    You can continue running session by
+
+    .. code-block:: python
+
+        dotenv run -- python rdagent/app/qlib_rd_loop/model.py $LOG_PATH/__session__/1/0_propose  --step_n 1   # `step_n` is a optional paramter
+
+    """
+    if path is None:
+        model_loop = ModelRDLoop(MODEL_PROP_SETTING)
+    else:
+        model_loop = ModelRDLoop.load(path, checkout=checkout)
+    asyncio.run(model_loop.run(step_n=step_n, loop_n=loop_n, all_duration=all_duration))
+
+
+if __name__ == "__main__":
+    fire.Fire(main)

rdagent/app/qlib_rd_loop/prompts.yaml

@@ -0,0 +1,15 @@
+hypothesis_generation:
+  system: |-
+    You are an expert in financial analysis. Your task is to generate a well-reasoned hypothesis based on the provided financial factors and report content.
+    Please ensure your response is in JSON format as shown below:
+    {
+      "hypothesis": "A clear and concise hypothesis based on the provided information.",
+      "reason": "A detailed explanation supporting the generated hypothesis.",
+    }
+
+  user: |-
+    The following are the financial factors and their descriptions:
+    {{ factor_descriptions }}
+
+    The report content is as follows:
+    {{ report_content }}

rdagent/app/qlib_rd_loop/quant.py

@@ -0,0 +1,137 @@
+"""
+Quant (Factor & Model) workflow with session control
+"""
+
+import asyncio
+from typing import Any
+
+import fire
+
+from rdagent.app.qlib_rd_loop.conf import QUANT_PROP_SETTING
+from rdagent.components.workflow.conf import BasePropSetting
+from rdagent.components.workflow.rd_loop import RDLoop
+from rdagent.core.conf import RD_AGENT_SETTINGS
+from rdagent.core.developer import Developer
+from rdagent.core.exception import FactorEmptyError, ModelEmptyError
+from rdagent.core.proposal import (
+    Experiment2Feedback,
+    Hypothesis2Experiment,
+    HypothesisFeedback,
+    HypothesisGen,
+)
+from rdagent.core.scenario import Scenario
+from rdagent.core.utils import import_class
+from rdagent.log import rdagent_logger as logger
+from rdagent.scenarios.qlib.proposal.quant_proposal import QuantTrace
+
+
+class QuantRDLoop(RDLoop):
+    skip_loop_error = (
+        FactorEmptyError,
+        ModelEmptyError,
+    )
+
+    def __init__(self, PROP_SETTING: BasePropSetting):
+        scen: Scenario = import_class(PROP_SETTING.scen)()
+        logger.log_object(scen, tag="scenario")
+
+        self.hypothesis_gen: HypothesisGen = import_class(PROP_SETTING.quant_hypothesis_gen)(scen)
+        logger.log_object(self.hypothesis_gen, tag="quant hypothesis generator")
+
+        self.factor_hypothesis2experiment: Hypothesis2Experiment = import_class(
+            PROP_SETTING.factor_hypothesis2experiment
+        )()
+        logger.log_object(self.factor_hypothesis2experiment, tag="factor hypothesis2experiment")
+        self.model_hypothesis2experiment: Hypothesis2Experiment = import_class(
+            PROP_SETTING.model_hypothesis2experiment
+        )()
+        logger.log_object(self.model_hypothesis2experiment, tag="model hypothesis2experiment")
+
+        self.factor_coder: Developer = import_class(PROP_SETTING.factor_coder)(scen)
+        logger.log_object(self.factor_coder, tag="factor coder")
+        self.model_coder: Developer = import_class(PROP_SETTING.model_coder)(scen)
+        logger.log_object(self.model_coder, tag="model coder")
+
+        self.factor_runner: Developer = import_class(PROP_SETTING.factor_runner)(scen)
+        logger.log_object(self.factor_runner, tag="factor runner")
+        self.model_runner: Developer = import_class(PROP_SETTING.model_runner)(scen)
+        logger.log_object(self.model_runner, tag="model runner")
+
+        self.factor_summarizer: Experiment2Feedback = import_class(PROP_SETTING.factor_summarizer)(scen)
+        logger.log_object(self.factor_summarizer, tag="factor summarizer")
+        self.model_summarizer: Experiment2Feedback = import_class(PROP_SETTING.model_summarizer)(scen)
+        logger.log_object(self.model_summarizer, tag="model summarizer")
+
+        self.trace = QuantTrace(scen=scen)
+        super(RDLoop, self).__init__()
+
+    async def direct_exp_gen(self, prev_out: dict[str, Any]):
+        while True:
+            if self.get_unfinished_loop_cnt(self.loop_idx) < RD_AGENT_SETTINGS.get_max_parallel():
+                hypo = self._propose()
+                assert hypo.action in ["factor", "model"]
+                if hypo.action == "factor":
+                    exp = self.factor_hypothesis2experiment.convert(hypo, self.trace)
+                else:
+                    exp = self.model_hypothesis2experiment.convert(hypo, self.trace)
+                logger.log_object(exp.sub_tasks, tag="experiment generation")
+                return {"propose": hypo, "exp_gen": exp}
+            await asyncio.sleep(1)
+
+    def coding(self, prev_out: dict[str, Any]):
+        if prev_out["direct_exp_gen"]["propose"].action == "factor":
+            exp = self.factor_coder.develop(prev_out["direct_exp_gen"]["exp_gen"])
+        elif prev_out["direct_exp_gen"]["propose"].action == "model":
+            exp = self.model_coder.develop(prev_out["direct_exp_gen"]["exp_gen"])
+        logger.log_object(exp, tag="coder result")
+        return exp
+
+    def running(self, prev_out: dict[str, Any]):
+        if prev_out["direct_exp_gen"]["propose"].action == "factor":
+            exp = self.factor_runner.develop(prev_out["coding"])
+            if exp is None:
+                logger.error(f"Factor extraction failed.")
+                raise FactorEmptyError("Factor extraction failed.")
+        elif prev_out["direct_exp_gen"]["propose"].action == "model":
+            exp = self.model_runner.develop(prev_out["coding"])
+        logger.log_object(exp, tag="runner result")
+        return exp
+
+    def feedback(self, prev_out: dict[str, Any]):
+        e = prev_out.get(self.EXCEPTION_KEY, None)
+        if e is not None:
+            feedback = HypothesisFeedback(
+                observations=str(e),
+                hypothesis_evaluation="",
+                new_hypothesis="",
+                reason="",
+                decision=False,
+            )
+            logger.log_object(feedback, tag="feedback")
+            self.trace.hist.append((prev_out["direct_exp_gen"]["exp_gen"], feedback))
+        else:
+            if prev_out["direct_exp_gen"]["propose"].action == "factor":
+                feedback = self.factor_summarizer.generate_feedback(prev_out["running"], self.trace)
+            elif prev_out["direct_exp_gen"]["propose"].action == "model":
+                feedback = self.model_summarizer.generate_feedback(prev_out["running"], self.trace)
+            logger.log_object(feedback, tag="feedback")
+            self.trace.hist.append((prev_out["running"], feedback))
+
+
+def main(path=None, step_n=None, loop_n=None, all_duration=None, checkout=True):
+    """
+    Auto R&D Evolving loop for fintech factors.
+    You can continue running session by
+    .. code-block:: python
+        dotenv run -- python rdagent/app/qlib_rd_loop/quant.py $LOG_PATH/__session__/1/0_propose  --step_n 1   # `step_n` is a optional paramter
+    """
+    if path is None:
+        quant_loop = QuantRDLoop(QUANT_PROP_SETTING)
+    else:
+        quant_loop = QuantRDLoop.load(path, checkout=checkout)
+
+    asyncio.run(quant_loop.run(step_n=step_n, loop_n=loop_n, all_duration=all_duration))
+
+
+if __name__ == "__main__":
+    fire.Fire(main)

rdagent/app/utils/ape.py

@@ -0,0 +1,49 @@
+"""
+This is the preliminary version of the APE (Automated Prompt Engineering)
+"""
+
+import pickle
+from pathlib import Path
+
+from rdagent.log.conf import LOG_SETTINGS
+
+
+def get_llm_qa(file_path):
+    data_flt = []
+    with open(file_path, "rb") as f:
+        data = pickle.load(f)
+        print(len(data))
+        for item in data:
+            if "debug_llm" in item["tag"]:
+                data_flt.append(item)
+    return data_flt
+
+
+# Example usage
+# use
+file_path = Path(LOG_SETTINGS.trace_path) / "debug_llm.pkl"
+llm_qa = get_llm_qa(file_path)
+print(len(llm_qa))
+
+print(llm_qa[0])
+
+# Initialize APE backend
+from rdagent.oai.llm_utils import APIBackend
+from rdagent.utils.agent.tpl import T
+
+api = APIBackend()
+
+# Analyze test data and generate improved prompts
+for qa in llm_qa:
+    # Generate system prompt for APE
+    system_prompt = T(".prompts:ape.system").r()
+
+    # Generate user prompt with context from LLM QA
+    user_prompt = T(".prompts:ape.user").r(
+        system=qa["obj"].get("system", ""), user=qa["obj"]["user"], answer=qa["obj"]["resp"]
+    )
+    analysis_result = api.build_messages_and_create_chat_completion(
+        system_prompt=system_prompt, user_prompt=user_prompt
+    )
+    print(f"█" * 60)
+    yes = input("Do you want to continue? (y/n)")

rdagent/app/utils/health_check.py

@@ -0,0 +1,52 @@
+import socket
+
+import docker
+
+from rdagent.log import rdagent_logger as logger
+from rdagent.utils.env import cleanup_container
+
+
+def check_docker() -> None:
+    container = None
+    try:
+        client = docker.from_env()
+        client.images.pull("hello-world")
+        container = client.containers.run("hello-world", detach=True)
+        logs = container.logs().decode("utf-8")
+        print(logs)
+        logger.info(f"The docker status is normal")
+    except docker.errors.DockerException as e:
+        logger.error(f"An error occurred: {e}")
+        logger.warning(
+            f"Docker status is exception, please check the docker configuration or reinstall it. Refs: https://docs.docker.com/engine/install/ubuntu/."
+        )
+    finally:
+        cleanup_container(container, "health check")
+
+
+def is_port_in_use(port):
+    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+        return s.connect_ex(("127.0.0.1", port)) == 0
+
+
+def check_and_list_free_ports(start_port=19899, max_ports=10) -> None:
+    is_occupied = is_port_in_use(port=start_port)
+    if is_occupied:
+        free_ports = []
+        for port in range(start_port, start_port + max_ports):
+            if not is_port_in_use(port):
+                free_ports.append(port)
+        logger.warning(
+            f"Port 19899 is occupied, please replace it with an available port when running the `rdagent ui` command. Available ports: {free_ports}"
+        )
+    else:
+        logger.info(f"Port 19899 is not occupied, you can run the `rdagent ui` command")
+
+
+def health_check():
+    """
+    Check that docker is installed correctly,
+    and that the ports used in the sample README are not occupied.
+    """
+    check_docker()
+    check_and_list_free_ports()

rdagent/app/utils/info.py

@@ -0,0 +1,86 @@
+import importlib.metadata
+import platform
+import sys
+from pathlib import Path
+
+import docker
+import requests
+from setuptools_scm import get_version
+
+from rdagent.log import rdagent_logger as logger
+
+
+def sys_info():
+    """collect system related info"""
+    method_list = [
+        ["Name of current operating system: ", "system"],
+        ["Processor architecture: ", "machine"],
+        ["System, version, and hardware information: ", "platform"],
+        ["Version number of the system: ", "version"],
+    ]
+    for method in method_list:
+        logger.info(f"{method[0]}{getattr(platform, method[1])()}")
+    return None
+
+
+def python_info():
+    """collect Python related info"""
+    python_version = sys.version.replace("\n", " ")
+    logger.info(f"Python version: {python_version}")
+    return None
+
+
+def docker_info():
+    client = docker.from_env()
+    containers = client.containers.list(all=True)
+    if containers:
+        containers.sort(key=lambda c: c.attrs["Created"])
+        last_container = containers[-1]
+        logger.info(f"Container ID: {last_container.id}")
+        logger.info(f"Container Name: {last_container.name}")
+        logger.info(f"Container Status: {last_container.status}")
+        logger.info(f"Image ID used by the container: {last_container.image.id}")
+        logger.info(f"Image tag used by the container: {last_container.image.tags}")
+        logger.info(f"Container port mapping: {last_container.ports}")
+        logger.info(f"Container Label: {last_container.labels}")
+        logger.info(f"Startup Commands: {' '.join(client.containers.get(last_container.id).attrs['Config']['Cmd'])}")
+    else:
+        logger.info(f"No run containers.")
+
+
+def rdagent_info():
+    """collect rdagent related info"""
+    current_version = importlib.metadata.version("rdagent")
+    logger.info(f"RD-Agent version: {current_version}")
+    api_url = f"https://api.github.com/repos/microsoft/RD-Agent/contents/requirements.txt?ref=main"
+    response = requests.get(api_url)
+    if response.status_code == 200:
+        files = response.json()
+        file_url = files["download_url"]
+        file_response = requests.get(file_url)
+        if file_response.status_code == 200:
+            all_file_contents = file_response.text.split("\n")
+        else:
+            logger.warning(f"Failed to retrieve {files['name']}, status code: {file_response.status_code}")
+    else:
+        logger.warning(f"Failed to retrieve files in folder, status code: {response.status_code}")
+    package_list = [
+        item.split("#")[0].strip() for item in all_file_contents if item.strip() and not item.startswith("#")
+    ]
+    package_version_list = []
+    for package in package_list:
+        if package == "typer[all]":
+            package = "typer"
+        version = importlib.metadata.version(package)
+        package_version_list.append(f"{package}=={version}")
+    logger.info(f"Package version: {package_version_list}")
+    return None
+
+
+def collect_info():
+    """Prints information about the system and the installed packages."""
+    sys_info()
+    python_info()
+    docker_info()
+    rdagent_info()
+    return None

rdagent/app/utils/prompts.yaml

@@ -0,0 +1,119 @@
+ape:
+  system: |-
+    We'll provide you with a pair of Chat QA about data science.
+    We are creating solutions for a Kaggle Competition based on the answers.
+    Good questions are crucial for getting good answers.
+    Please suggest how to improve the question.
+    You can analyze based on these aspects:
+    - Is the question complete (is all the information needed to answer the question provided?)
+
+    The conversation will be provided in the following format:
+
+    <question>
+      <part1>
+      ...text to describe the question...
+      </part1>
+      <part2>
+      ...text to describe the question...
+      </part2>
+    </question>
+
+    <answer>
+      ...text to describe the answer.
+    </answer>
+
+    You response should be very concorete and concise(less than 20 words) and focuse on the mentioned aspects, like
+    ```
+    Info Missing: the question ask for changing code, but it does not provide the description of current code.
+    ```
+    Please be very conversatiive when you propose improvements. Only propose improvements when it becomes impossible to give the answer.
+
+    Don't propose conerete modifications
+
+  user: |-
+    <question>
+      <part1>
+      {{system}}
+      </part1>
+      <part2>
+      {{user}}
+      </part2>
+    </question>
+
+    <answer>
+      {{answer}}
+    </answer>
+
+  optional: |-
+    If you want to suggest modification on  the question. Please follow the *SEARCH/REPLACE block* Rules!!!! It is optional.
+    Please make it concise and less than 20 lines!!!
+
+    # *SEARCH/REPLACE block* Rules:
+
+    Every *SEARCH/REPLACE block* must use this format:
+    1. The *FULL* file path alone on a line, verbatim. No bold asterisks, no quotes around it, no escaping of characters, etc.
+    2. The opening fence and code language, eg: ```python
+    3. The start of search block: <<<<<<< SEARCH
+    4. A contiguous chunk of lines to search for in the existing source code
+    5. The dividing line: =======
+    6. The lines to replace into the source code
+    7. The end of the replace block: >>>>>>> REPLACE
+    8. The closing fence: ```
+
+    Use the *FULL* file path, as shown to you by the user.
+
+    Every *SEARCH* section must *EXACTLY MATCH* the existing file content, character for character, including all comments, docstrings, etc.
+    If the file contains code or other data wrapped/escaped in json/xml/quotes or other containers, you need to propose edits to the literal contents of the file, including the container markup.
+
+    *SEARCH/REPLACE* blocks will *only* replace the first match occurrence.
+    Including multiple unique *SEARCH/REPLACE* blocks if needed.
+    Include enough lines in each SEARCH section to uniquely match each set of lines that need to change.
+
+    Keep *SEARCH/REPLACE* blocks concise.
+    Break large *SEARCH/REPLACE* blocks into a series of smaller blocks that each change a small portion of the file.
+    Include just the changing lines, and a few surrounding lines if needed for uniqueness.
+    Do not include long runs of unchanging lines in *SEARCH/REPLACE* blocks.
+
+    Only create *SEARCH/REPLACE* blocks for files that the user has added to the chat!
+
+    To move code within a file, use 2 *SEARCH/REPLACE* blocks: 1 to delete it from its current location, 1 to insert it in the new location.
+
+    Pay attention to which filenames the user wants you to edit, especially if they are asking you to create a new file.
+
+    If you want to put code in a new file, use a *SEARCH/REPLACE block* with:
+    - A new file path, including dir name if needed
+    - An empty `SEARCH` section
+    - The new file's contents in the `REPLACE` section
+
+    To rename files which have been added to the chat, use shell commands at the end of your response.
+
+    If the user just says something like "ok" or "go ahead" or "do that" they probably want you to make SEARCH/REPLACE blocks for the code changes you just proposed.
+    The user will say when they've applied your edits. If they haven't explicitly confirmed the edits have been applied, they probably want proper SEARCH/REPLACE blocks.
+
+    You are diligent and tireless!
+    You NEVER leave comments describing code without implementing it!
+    You always COMPLETELY IMPLEMENT the needed code!
+
+
+    ONLY EVER RETURN CODE IN A *SEARCH/REPLACE BLOCK*!
+    Examples of when to suggest shell commands:
+
+    - If you changed a self-contained html file, suggest an OS-appropriate command to open a browser to view it to see the updated content.
+    - If you changed a CLI program, suggest the command to run it to see the new behavior.
+    - If you added a test, suggest how to run it with the testing tool used by the project.
+    - Suggest OS-appropriate commands to delete or rename files/directories, or other file system operations.
+    - If your code changes add new dependencies, suggest the command to install them.
+    - Etc.
+
+    Here is a example of SEARCH/REPLACE BLOCK to change a function implementation to import.
+
+    <<<<<<< SEARCH
+    def hello():
+        "print a greeting"
+
+        print("hello")
+    =======
+    from hello import hello
+
+    >>>>>>> REPLACE
+# - Is there any ambiguity in the question?

rdagent/benchmark/conf.py

@@ -1,27 +0,0 @@
-from dotenv import load_dotenv
-load_dotenv(verbose=True, override=True)
-from dataclasses import field
-from pathlib import Path
-from typing import Literal, Optional, Union
-
-from pydantic_settings import BaseSettings
-
-DIRNAME = Path(__file__).absolute().resolve().parent
-
-BENCHMARK_VERSION = Literal["paper", "amcV01", "amcV02train", "amcV02test"]
-
-class BenchmarkSettings(BaseSettings):
-
-    ground_truth_dir: Path = DIRNAME / "ground_truth"
-
-    bench_version: Union[BENCHMARK_VERSION, str] = "paper"
-
-    bench_test_round: int = 20 
-    bench_test_case_n: Optional[int] = None  # how many test cases to run; If not given, all test cases will be run
-
-    bench_method_cls: str = "scripts.factor_implementation.baselines.naive.one_shot.OneshotFactorGen"
-    bench_method_extra_kwargs: dict = field(
-        default_factory=dict,
-    )  # extra kwargs for the method to be tested except the task list
-
-    bench_result_path: Path = DIRNAME / "result"

rdagent/benchmark/eval_method.py

@@ -1,163 +0,0 @@
-from pathlib import Path
-from typing import List, Tuple, Union
-
-from tqdm import tqdm
-from collections import defaultdict
-from rdagent.core.conf import RD_AGENT_SETTINGS
-from rdagent.core.exception import ImplementRunException
-from rdagent.core.task import (
-    TaskImplementation,
-    TestCase,
-)
-from rdagent.factor_implementation.evolving.evaluators import (
-    FactorImplementationCorrelationEvaluator,
-    FactorImplementationIndexEvaluator,
-    FactorImplementationIndexFormatEvaluator,
-    FactorImplementationMissingValuesEvaluator,
-    FactorImplementationRowCountEvaluator,
-    FactorImplementationSingleColumnEvaluator,
-    FactorImplementationValuesEvaluator,
-    FactorImplementationEvaluator,
-)
-from rdagent.core.implementation import TaskGenerator
-from rdagent.core.utils import multiprocessing_wrapper
-from rdagent.factor_implementation.evolving.factor import FileBasedFactorImplementation
-
-
-class BaseEval:
-    """
-    The benchmark benchmark evaluation.
-    """
-
-    def __init__(
-        self,
-        evaluator_l: List[FactorImplementationEvaluator],
-        test_cases: List[TestCase],
-        generate_method: TaskGenerator,
-        catch_eval_except: bool = True,
-    ):
-        """Parameters
-        ----------
-        test_cases : List[TestCase]
-            cases to be evaluated, ground truth are included in the test cases.
-        evaluator_l : List[FactorImplementationEvaluator]
-            A list of evaluators to evaluate the generated code.
-        catch_eval_except : bool
-            If we want to debug the evaluators, we recommend to set the this parameter to True.
-        """
-        self.evaluator_l = evaluator_l
-        self.test_cases = test_cases
-        self.generate_method = generate_method
-        self.catch_eval_except = catch_eval_except
-
-    def load_cases_to_eval(
-        self,
-        path: Union[Path, str],
-        **kwargs,
-    ) -> List[TaskImplementation]:
-        path = Path(path)
-        fi_l = []
-        for tc in self.test_cases:
-            try:
-                fi = FileBasedFactorImplementation.from_folder(tc.task, path, **kwargs)
-                fi_l.append(fi)
-            except FileNotFoundError:
-                print("Fail to load test case for factor: ", tc.task.factor_name)
-        return fi_l
-
-    def eval_case(
-        self,
-        case_gt: TaskImplementation,
-        case_gen: TaskImplementation,
-    ) -> List[Union[Tuple[FactorImplementationEvaluator, object], Exception]]:
-        """Parameters
-        ----------
-        case_gt : FactorImplementation
-
-        case_gen : FactorImplementation
-
-
-        Returns
-        -------
-        List[Union[Tuple[FactorImplementationEvaluator, object],Exception]]
-            for each item
-                If the evaluation run successfully, return the evaluate results.  Otherwise, return the exception.
-        """
-        eval_res = []
-        for ev in self.evaluator_l:
-            try:
-                eval_res.append((ev, ev.evaluate(case_gt, case_gen)))
-                # if the corr ev is successfully evaluated and achieve the best performance, then break
-            except ImplementRunException as e:
-                return e
-            except Exception as e:
-                # exception when evaluation
-                if self.catch_eval_except:
-                    eval_res.append((ev, e))
-                else:
-                    raise e
-        return eval_res
-
-
-class FactorImplementEval(BaseEval):
-    def __init__(
-        self,
-        test_case: TestCase,
-        method: TaskGenerator,
-        test_round: int = 10,
-        *args,
-        **kwargs,
-    ):
-        # evaluator collection for online evaluation
-        online_evaluator_l = (
-            [
-                FactorImplementationCorrelationEvaluator,
-                FactorImplementationIndexEvaluator,
-                FactorImplementationIndexFormatEvaluator,
-                FactorImplementationMissingValuesEvaluator,
-                FactorImplementationRowCountEvaluator,
-                FactorImplementationSingleColumnEvaluator,
-                FactorImplementationValuesEvaluator,
-            ],
-        )
-        super().__init__(online_evaluator_l, test_case, method, *args, **kwargs)
-        self.test_round = test_round
-
-    def eval(self):
-
-        gen_factor_l_all_rounds = []
-        test_cases_all_rounds = []
-        res = defaultdict(list)
-        for _ in tqdm(range(self.test_round), desc="Rounds of Eval"):
-            print("\n========================================================")
-            print(f"Eval {_}-th times...")
-            print("========================================================\n")
-            try:
-                gen_factor_l = self.generate_method.generate(self.test_cases.target_task)
-            except KeyboardInterrupt:
-                # TODO: Why still need to save result after KeyboardInterrupt?
-                print("Manually interrupted the evaluation. Saving existing results")
-                break
-
-            if len(gen_factor_l) != len(self.test_cases):
-                raise ValueError(
-                    "The number of cases to eval should be equal to the number of test cases.",
-                )
-            gen_factor_l_all_rounds.extend(gen_factor_l)
-            test_cases_all_rounds.extend(self.test_cases)
-
-        eval_res_l = []
-
-        eval_res_list = multiprocessing_wrapper(
-            [
-                (self.eval_case, (gt_case.ground_truth, gen_factor))
-                for gt_case, gen_factor in zip(test_cases_all_rounds, gen_factor_l_all_rounds)
-            ],
-            n=RD_AGENT_SETTINGS.evo_multi_proc_n,
-        )
-
-        for gt_case, eval_res, gen_factor in tqdm(zip(test_cases_all_rounds, eval_res_list, gen_factor_l_all_rounds)):
-            res[gt_case.task.factor_name].append((gen_factor, eval_res))
-            eval_res_l.append(eval_res)
-
-        return res

rdagent/components/benchmark/conf.py

@@ -0,0 +1,33 @@
+from dataclasses import field
+from pathlib import Path
+from typing import Optional
+
+from rdagent.core.conf import ExtendedBaseSettings
+
+DIRNAME = Path("./")
+
+
+class BenchmarkSettings(ExtendedBaseSettings):
+    class Config:
+        env_prefix = "BENCHMARK_"
+        """Use `BENCHMARK_` as prefix for environment variables"""
+
+    bench_data_path: Path = DIRNAME / "example.json"
+    """data for benchmark"""
+
+    bench_test_round: int = 10
+    """how many rounds to run, each round may cost 10 minutes"""
+
+    bench_test_case_n: Optional[int] = None
+    """how many test cases to run; If not given, all test cases will be run"""
+
+    bench_method_cls: str = "rdagent.components.coder.factor_coder.FactorCoSTEER"
+    """method to be used for test cases"""
+
+    bench_method_extra_kwargs: dict = field(
+        default_factory=dict,
+    )
+    """extra kwargs for the method to be tested except the task list"""
+
+    bench_result_path: Path = DIRNAME / "result"
+    """result save path"""

rdagent/components/benchmark/eval_method.py

@@ -0,0 +1,222 @@
+from collections import defaultdict
+from pathlib import Path
+from typing import Dict, List, Tuple, Union
+
+import pandas as pd
+from tqdm import tqdm
+
+from rdagent.components.coder.factor_coder.config import FACTOR_COSTEER_SETTINGS
+from rdagent.components.coder.factor_coder.eva_utils import (
+    FactorCorrelationEvaluator,
+    FactorEqualValueRatioEvaluator,
+    FactorEvaluator,
+    FactorIndexEvaluator,
+    FactorRowCountEvaluator,
+    FactorSingleColumnEvaluator,
+)
+from rdagent.components.coder.factor_coder.factor import FactorFBWorkspace
+from rdagent.core.conf import RD_AGENT_SETTINGS
+from rdagent.core.developer import Developer
+from rdagent.core.exception import CoderError
+from rdagent.core.experiment import Experiment, Task, Workspace
+from rdagent.core.scenario import Scenario
+from rdagent.core.utils import multiprocessing_wrapper
+
+EVAL_RES = Dict[
+    str,
+    List[Tuple[FactorEvaluator, Union[object, CoderError]]],
+]
+
+
+class TestCase:
+    def __init__(
+        self,
+        target_task: Task,
+        ground_truth: Workspace,
+    ):
+        self.target_task = target_task
+        self.ground_truth = ground_truth
+
+
+class TestCases:
+    def __init__(self, test_case_l: list[TestCase] = []):
+        # self.test_case_l = [TestCase(task, gt) for task, gt in zip(target_task, ground_truth)]
+        self.test_case_l = test_case_l
+
+    def __getitem__(self, item):
+        return self.test_case_l[item]
+
+    def __len__(self):
+        return len(self.test_case_l)
+
+    def get_exp(self):
+        return Experiment([case.target_task for case in self.test_case_l])
+
+    @property
+    def target_task(self):
+        return [case.target_task for case in self.test_case_l]
+
+    @property
+    def ground_truth(self):
+        return [case.ground_truth for case in self.test_case_l]
+
+
+class BaseEval:
+    """
+    The benchmark benchmark evaluation.
+    """
+
+    def __init__(
+        self,
+        evaluator_l: List[FactorEvaluator],
+        test_cases: TestCases,
+        generate_method: Developer,
+        catch_eval_except: bool = True,
+    ):
+        """Parameters
+        ----------
+        test_cases : TestCases
+            cases to be evaluated, ground truth are included in the test cases.
+        evaluator_l : List[FactorEvaluator]
+            A list of evaluators to evaluate the generated code.
+        catch_eval_except : bool
+            If we want to debug the evaluators, we recommend to set the this parameter to True.
+        """
+        self.evaluator_l = evaluator_l
+        self.test_cases = test_cases
+        self.generate_method = generate_method
+        self.catch_eval_except = catch_eval_except
+
+    def load_cases_to_eval(
+        self,
+        path: Union[Path, str],
+        **kwargs,
+    ) -> List[Workspace]:
+        path = Path(path)
+        fi_l = []
+        for tc in self.test_cases:
+            try:
+                fi = FactorFBWorkspace.from_folder(tc.task, path, **kwargs)
+                fi_l.append(fi)
+            except FileNotFoundError:
+                print("Fail to load test case for factor: ", tc.task.factor_name)
+        return fi_l
+
+    def eval_case(
+        self,
+        case_gt: Workspace,
+        case_gen: Workspace,
+    ) -> List[Union[Tuple[FactorEvaluator, object], Exception]]:
+        """Parameters
+        ----------
+        case_gt : FactorImplementation
+
+        case_gen : FactorImplementation
+
+
+        Returns
+        -------
+        List[Union[Tuple[FactorEvaluator, object],Exception]]
+            for each item
+                If the evaluation run successfully, return the evaluate results.  Otherwise, return the exception.
+        """
+        eval_res = []
+        for ev in self.evaluator_l:
+            try:
+                case_gen.raise_exception = True
+                eval_res.append((ev, ev.evaluate(implementation=case_gen, gt_implementation=case_gt)))
+                # if the corr ev is successfully evaluated and achieve the best performance, then break
+            except CoderError as e:
+                return e
+            except Exception as e:
+                # exception when evaluation
+                if self.catch_eval_except:
+                    eval_res.append((ev, e))
+                else:
+                    raise e
+        return eval_res
+
+
+class FactorImplementEval(BaseEval):
+    def __init__(
+        self,
+        test_cases: TestCases,
+        method: Developer,
+        *args,
+        scen: Scenario,
+        test_round: int = 10,
+        **kwargs,
+    ):
+        online_evaluator_l = [
+            FactorSingleColumnEvaluator(scen),
+            FactorRowCountEvaluator(scen),
+            FactorIndexEvaluator(scen),
+            FactorEqualValueRatioEvaluator(scen),
+            FactorCorrelationEvaluator(hard_check=False, scen=scen),
+        ]
+        super().__init__(online_evaluator_l, test_cases, method, *args, **kwargs)
+        self.test_round = test_round
+
+    def develop(self):
+        gen_factor_l_all_rounds = []
+        for _ in tqdm(range(self.test_round), desc="Rounds of Eval"):
+            print("\n========================================================")
+            print(f"Eval {_}-th times...")
+            print("========================================================\n")
+            try:
+                gen_factor_l = self.generate_method.develop(self.test_cases.get_exp())
+            except KeyboardInterrupt:
+                # TODO: Why still need to save result after KeyboardInterrupt?
+                print("Manually interrupted the evaluation. Saving existing results")
+                break
+
+            if len(gen_factor_l.sub_workspace_list) != len(self.test_cases.ground_truth):
+                raise ValueError(
+                    "The number of cases to eval should be equal to the number of test cases.",
+                )
+            gen_factor_l_all_rounds.extend(gen_factor_l.sub_workspace_list)
+
+        return gen_factor_l_all_rounds
+
+    def eval(self, gen_factor_l_all_rounds):
+        test_cases_all_rounds = []
+        res = defaultdict(list)
+        for _ in range(self.test_round):
+            test_cases_all_rounds.extend(self.test_cases.ground_truth)
+        eval_res_list = multiprocessing_wrapper(
+            [
+                (self.eval_case, (gt_case, gen_factor))
+                for gt_case, gen_factor in zip(test_cases_all_rounds, gen_factor_l_all_rounds)
+            ],
+            n=RD_AGENT_SETTINGS.multi_proc_n,
+        )
+
+        for gt_case, eval_res, gen_factor in tqdm(zip(test_cases_all_rounds, eval_res_list, gen_factor_l_all_rounds)):
+            res[gt_case.target_task.factor_name].append((gen_factor, eval_res))
+
+        return res
+
+    @staticmethod
+    def summarize_res(res: EVAL_RES) -> pd.DataFrame:
+        # None: indicate that it raises exception and get no results
+        sum_res = {}
+        for factor_name, runs in res.items():
+            for fi, err_or_res_l in runs:
+                # NOTE:  str(fi) may not be unique!!  Because the workspace can be skipped when hitting the cache.
+                uniq_key = f"{str(fi)},{id(fi)}"
+
+                key = (factor_name, uniq_key)
+                val = {}
+                if isinstance(err_or_res_l, Exception):
+                    val["run factor error"] = str(err_or_res_l.__class__)
+                else:
+                    val["run factor error"] = None
+                    for ev_obj, err_or_res in err_or_res_l:
+                        if isinstance(err_or_res, Exception):
+                            val[str(ev_obj)] = None
+                        else:
+                            feedback, metric = err_or_res
+                            val[str(ev_obj)] = metric
+                sum_res[key] = val
+
+        return pd.DataFrame(sum_res)

rdagent/components/benchmark/example.json

@@ -0,0 +1,36 @@
+{
+    "Turnover_Rate_Factor": {
+        "description": "A traditional factor based on 20-day average turnover rate, adjusted for market capitalization, which is further improved by applying the information distribution theory.",
+        "formulation": "\\text{Adjusted Turnover Rate} = \\frac{\\text{mean}(20\\text{-day turnover rate})}{\\text{Market Capitalization}}",
+        "variables": {
+            "20-day turnover rate": "Average turnover rate over the past 20 days.",
+            "Market Capitalization": "Total market value of a company's outstanding shares."
+        },
+        "Category": "Fundamentals",
+        "Difficulty": "Easy",
+        "gt_code": "import pandas as pd\n\ndata_f = pd.read_hdf('daily_f.h5')\n\ndata = data_f.reset_index()\nwindow_size = 20\n\nnominator=data.groupby('instrument')[['TurnoverRate_30D']].rolling(window=window_size).mean().reset_index(0, drop=True)\n# transfer to series\nnew=nominator['TurnoverRate_30D']\ndata['Turnover_Rate_Factor']=new/data['TradableACapital']\n\n# set the datetime and instrument as index and drop the original index\nresult=pd.DataFrame(data['Turnover_Rate_Factor']).set_index(data_f.index)\n\n# transfer the result to series\nresult=result['Turnover_Rate_Factor']\nresult.to_hdf(\"result.h5\", key=\"data\")" 
+    },
+    "PctTurn20": {
+        "description": "A factor representing the percentage change in turnover rate over the past 20 trading days, market-value neutralized.",
+        "formulation": "\\text{PctTurn20} = \\frac{1}{N} \\sum_{i=1}^{N} \\left( \\frac{\\text{Turnover}_{i, t} - \\text{Turnover}_{i, t-20}}{\\text{Turnover}_{i, t-20}} \\right)",
+        "variables": {
+            "N": "Number of stocks in the market.",
+            "Turnover_{i, t}": "Turnover of stock i at day t.",
+            "Turnover_{i, t-20}": "Turnover of stock i at day t-20."
+        },
+        "Category": "Volume&Price",
+        "Difficulty": "Medium",
+        "gt_code": "import pandas as pd\nfrom statsmodels import api as sm\n\ndef fill_mean(s: pd.Series) -> pd.Series:\n    return s.fillna(s.mean()).fillna(0.0)\n\ndef market_value_neutralize(s: pd.Series, mv: pd.Series) -> pd.Series:\n    s = s.groupby(\"datetime\", group_keys=False).apply(fill_mean)\n    mv = mv.groupby(\"datetime\", group_keys=False).apply(fill_mean)\n\n    df_f = mv.to_frame(\"MarketValue\")\n    df_f[\"const\"] = 1\n    X = df_f[[\"MarketValue\", \"const\"]]\n\n    # Perform the Ordinary Least Squares (OLS) regression\n    model = sm.OLS(s, X)\n    results = model.fit()\n\n    # Calculate the residuals\n    df_f[\"residual\"] = results.resid\n    df_f[\"norm_resi\"] = df_f.groupby(level=\"datetime\", group_keys=False)[\"residual\"].apply(\n        lambda x: (x - x.mean()) / x.std(),\n    )\n    return df_f[\"norm_resi\"]\n\n\n# get_turnover\ndf_pv = pd.read_hdf(\"daily_pv.h5\", key=\"data\")\ndf_f = pd.read_hdf(\"daily_f.h5\", key=\"data\")\nturnover = df_pv[\"$money\"] / df_f[\"TradableMarketValue\"]\n\nf = turnover.groupby(\"instrument\").pct_change(periods=20)\n\nf_neutralized = market_value_neutralize(f, df_f[\"TradableMarketValue\"])\n\nf_neutralized.to_hdf(\"result.h5\", key=\"data\")"
+    },
+    "PB_ROE": {
+        "description": "Constructed using the ranking difference between PB and ROE, with PB and ROE replacing original PB and ROE to obtain reconstructed factor values.",
+        "formulation": "\\text{rank}(PB\\_t) - rank(ROE_t)",
+        "variables": {
+            "\\text{rank}(PB_t)": "Ranking PB on cross-section at time t.",
+            "\\text{rank}(ROE_t)": "Ranking single-quarter ROE on cross-section at time t."
+        },
+        "Category": "High-Frequency",
+        "Difficulty": "Hard",
+        "gt_code": "#!/usr/bin/env python\n\nimport pandas as pd\n\ndata_f = pd.read_hdf('daily_f.h5')\n\ndata = data_f.reset_index()\n\n# Calculate the rank of PB and ROE\ndata['PB_rank'] = data.groupby('datetime')['B/P'].rank()\ndata['ROE_rank'] = data.groupby('datetime')['ROE'].rank()\n\n# Calculate the difference between the ranks\ndata['PB_ROE'] = data['PB_rank'] - data['ROE_rank']\n\n# set the datetime and instrument as index and drop the original index\nresult=pd.DataFrame(data['PB_ROE']).set_index(data_f.index)\n\n# transfer the result to series\nresult=result['PB_ROE']\nresult.to_hdf(\"result.h5\", key=\"data\")"
+    }
+}

rdagent/components/coder/CoSTEER/__init__.py

@@ -0,0 +1,144 @@
+import pickle
+from datetime import datetime
+from pathlib import Path
+
+from rdagent.components.coder.CoSTEER.config import CoSTEERSettings
+from rdagent.components.coder.CoSTEER.evaluators import CoSTEERMultiFeedback
+from rdagent.components.coder.CoSTEER.evolvable_subjects import EvolvingItem
+from rdagent.components.coder.CoSTEER.knowledge_management import (
+    CoSTEERKnowledgeBaseV1,
+    CoSTEERKnowledgeBaseV2,
+    CoSTEERRAGStrategyV1,
+    CoSTEERRAGStrategyV2,
+)
+from rdagent.core.developer import Developer
+from rdagent.core.evaluation import Evaluator, Feedback
+from rdagent.core.evolving_agent import EvolvingStrategy, RAGEvoAgent
+from rdagent.core.exception import CoderError
+from rdagent.core.experiment import Experiment
+from rdagent.log import rdagent_logger as logger
+
+
+class CoSTEER(Developer[Experiment]):
+    def __init__(
+        self,
+        settings: CoSTEERSettings,
+        eva: Evaluator,
+        es: EvolvingStrategy,
+        evolving_version: int,
+        *args,
+        with_knowledge: bool = True,
+        with_feedback: bool = True,
+        knowledge_self_gen: bool = True,
+        filter_final_evo: bool = True,
+        max_loop: int | None = None,
+        **kwargs,
+    ) -> None:
+        super().__init__(*args, **kwargs)
+        self.max_loop = settings.max_loop if max_loop is None else max_loop
+        self.max_seconds = settings.max_seconds
+        self.knowledge_base_path = (
+            Path(settings.knowledge_base_path) if settings.knowledge_base_path is not None else None
+        )
+        self.new_knowledge_base_path = (
+            Path(settings.new_knowledge_base_path) if settings.new_knowledge_base_path is not None else None
+        )
+
+        self.with_knowledge = with_knowledge
+        self.with_feedback = with_feedback
+        self.knowledge_self_gen = knowledge_self_gen
+        self.filter_final_evo = filter_final_evo
+        self.evolving_strategy = es
+        self.evaluator = eva
+        self.evolving_version = evolving_version
+
+        # init knowledge base
+        self.knowledge_base = self.load_or_init_knowledge_base(
+            former_knowledge_base_path=self.knowledge_base_path,
+            component_init_list=[],
+        )
+        # init rag method
+        self.rag = (
+            CoSTEERRAGStrategyV2(self.knowledge_base, settings=settings)
+            if self.evolving_version == 2
+            else CoSTEERRAGStrategyV1(self.knowledge_base, settings=settings)
+        )
+
+    def load_or_init_knowledge_base(self, former_knowledge_base_path: Path = None, component_init_list: list = []):
+        if former_knowledge_base_path is not None and former_knowledge_base_path.exists():
+            knowledge_base = pickle.load(open(former_knowledge_base_path, "rb"))
+            if self.evolving_version == 1 and not isinstance(knowledge_base, CoSTEERKnowledgeBaseV1):
+                raise ValueError("The former knowledge base is not compatible with the current version")
+            elif self.evolving_version == 2 and not isinstance(
+                knowledge_base,
+                CoSTEERKnowledgeBaseV2,
+            ):
+                raise ValueError("The former knowledge base is not compatible with the current version")
+        else:
+            knowledge_base = (
+                CoSTEERKnowledgeBaseV2(
+                    init_component_list=component_init_list,
+                )
+                if self.evolving_version == 2
+                else CoSTEERKnowledgeBaseV1()
+            )
+        return knowledge_base
+
+    def develop(self, exp: Experiment) -> Experiment:
+
+        # init intermediate items
+        evo_exp = EvolvingItem.from_experiment(exp)
+
+        self.evolve_agent = RAGEvoAgent(
+            max_loop=self.max_loop,
+            evolving_strategy=self.evolving_strategy,
+            rag=self.rag,
+            with_knowledge=self.with_knowledge,
+            with_feedback=self.with_feedback,
+            knowledge_self_gen=self.knowledge_self_gen,
+        )
+
+        start_datetime = datetime.now()
+        for evo_exp in self.evolve_agent.multistep_evolve(evo_exp, self.evaluator):
+            assert isinstance(evo_exp, Experiment)  # multiple inheritance
+            logger.log_object(evo_exp.sub_workspace_list, tag="evolving code")
+            for sw in evo_exp.sub_workspace_list:
+                logger.info(f"evolving workspace: {sw}")
+            if (datetime.now() - start_datetime).seconds > self.max_seconds:
+                logger.info(f"Reached max time limit {self.max_seconds} seconds, stop evolving")
+                break
+
+        if self.with_feedback and self.filter_final_evo:
+            evo_exp = self._exp_postprocess_by_feedback(evo_exp, self.evolve_agent.evolving_trace[-1].feedback)
+
+        # save new knowledge base
+        if self.new_knowledge_base_path is not None:
+            with self.new_knowledge_base_path.open("wb") as f:
+                pickle.dump(self.knowledge_base, f)
+            logger.info(f"New knowledge base saved to {self.new_knowledge_base_path}")
+        exp.sub_workspace_list = evo_exp.sub_workspace_list
+        exp.experiment_workspace = evo_exp.experiment_workspace
+        return exp
+
+    def _exp_postprocess_by_feedback(self, evo: Experiment, feedback: CoSTEERMultiFeedback) -> Experiment:
+        """
+        Responsibility:
+        - Raise Error if it failed to handle the develop task
+        -
+        """
+        assert isinstance(evo, Experiment)
+        assert isinstance(feedback, CoSTEERMultiFeedback)
+        assert len(evo.sub_workspace_list) == len(feedback)
+
+        # FIXME: when whould the feedback be None?
+        failed_feedbacks = [
+            f"- feedback{index + 1:02d}:\n  - execution: {f.execution}\n  - return_checking: {f.return_checking}\n  - code: {f.code}"
+            for index, f in enumerate(feedback)
+            if f is not None and not f.final_decision
+        ]
+
+        if len(failed_feedbacks) == len(feedback):
+            feedback_summary = "\n".join(failed_feedbacks)
+            raise CoderError(f"All tasks are failed:\n{feedback_summary}")
+
+        return evo

rdagent/components/coder/CoSTEER/config.py

@@ -0,0 +1,39 @@
+from typing import Union
+
+from rdagent.core.conf import ExtendedBaseSettings
+
+
+class CoSTEERSettings(ExtendedBaseSettings):
+    """CoSTEER settings, this setting is supposed not to be used directly!!!"""
+
+    class Config:
+        env_prefix = "CoSTEER_"
+
+    coder_use_cache: bool = False
+    """Indicates whether to use cache for the coder"""
+
+    max_loop: int = 10
+    """Maximum number of task implementation loops"""
+
+    fail_task_trial_limit: int = 20
+
+    v1_query_former_trace_limit: int = 5
+    v1_query_similar_success_limit: int = 5
+
+    v2_query_component_limit: int = 1
+    v2_query_error_limit: int = 1
+    v2_query_former_trace_limit: int = 1
+    v2_add_fail_attempt_to_latest_successful_execution: bool = False
+    v2_error_summary: bool = False
+    v2_knowledge_sampler: float = 1.0
+
+    knowledge_base_path: Union[str, None] = None
+    """Path to the knowledge base"""
+
+    new_knowledge_base_path: Union[str, None] = None
+    """Path to the new knowledge base"""
+
+    max_seconds: int = 10**6
+
+
+CoSTEER_SETTINGS = CoSTEERSettings()

rdagent/components/coder/CoSTEER/evaluators.py

@@ -0,0 +1,277 @@
+from abc import abstractmethod
+from copy import deepcopy
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, List
+
+from rdagent.components.coder.CoSTEER.evolvable_subjects import EvolvingItem
+from rdagent.core.conf import RD_AGENT_SETTINGS
+from rdagent.core.evaluation import Evaluator, Feedback
+from rdagent.core.evolving_framework import QueriedKnowledge
+from rdagent.core.experiment import Task, Workspace
+from rdagent.core.utils import multiprocessing_wrapper
+from rdagent.log import rdagent_logger as logger
+
+if TYPE_CHECKING:
+    from rdagent.core.scenario import Scenario
+
+# TODO:
+# 1. It seems logically sound, but we currently lack a scenario to apply it.
+# 2. If it proves to be useful, relocate it to a more general location.
+#
+# class FBWorkspaceExeFeedback(Feedback):
+#     """
+#     It pairs with FBWorkspace in the abstract level.
+#     """
+#     # ws: FBWorkspace   # potential
+#     stdout: str
+
+
+@dataclass
+class CoSTEERSingleFeedback(Feedback):
+    # TODO: (xiao)
+    # it should be more general class for FBWorkspaceExeFeedback
+    # A better name of it may be NormalFeedback
+    # TODO: It should be a general feeddback for CoSTEERR
+    """
+    The feedback for the data loader evaluation.
+    It is design align the phases of the implemented code
+    - Execution -> Return Value -> Code -> Final Decision
+    """
+    execution: str
+    # execution_feedback
+    return_checking: str | None  # including every check in the testing (constraints about the generated value)
+    # value_feedback, shape_feedback, value_generated_flag
+    code: str
+    final_decision: bool
+
+    @staticmethod
+    def val_and_update_init_dict(data: dict) -> dict:
+        # TODO: (bowen) use a more general method to validate and update the data dictionary before init, like pydantic
+        """
+        Validates and converts the 'final_decision' field in the given data dictionary.
+
+        Args:
+            data (dict): The data dictionary containing the 'final_decision' field.
+
+        Returns:
+            dict: The updated data dictionary with 'final_decision' as a boolean.
+
+        Raises:
+            ValueError: If 'final_decision' is not present or not a boolean.
+        """
+        if "final_decision" not in data:
+            raise ValueError("'final_decision' is required")
+
+        if isinstance(data["final_decision"], str):
+            if data["final_decision"] == "false" or data["final_decision"] == "False":
+                data["final_decision"] = False
+            elif data["final_decision"] == "true" or data["final_decision"] == "True":
+                data["final_decision"] = True
+
+        if not isinstance(data["final_decision"], bool):
+            raise ValueError(f"'final_decision' must be a boolean, not {type(data['final_decision'])}")
+        return data
+
+    def __str__(self) -> str:
+        return f"""------------------Execution------------------
+{self.execution}
+------------------Return Checking------------------
+{self.return_checking if self.return_checking is not None else 'No return checking'}
+------------------Code------------------
+{self.code}
+------------------Final Decision------------------
+This implementation is {'SUCCESS' if self.final_decision else 'FAIL'}.
+"""
+
+    def __bool__(self):
+        return self.final_decision
+
+
+class CoSTEERSingleFeedbackDeprecated(CoSTEERSingleFeedback):
+    """This class is a base class for all code generator feedback to single implementation"""
+
+    def __init__(
+        self,
+        execution_feedback: str = None,
+        shape_feedback: str = None,
+        code_feedback: str = None,
+        value_feedback: str = None,
+        final_decision: bool = None,
+        final_feedback: str = None,
+        value_generated_flag: bool = None,
+        final_decision_based_on_gt: bool = None,
+    ) -> None:
+        self.execution_feedback = execution_feedback
+        self.code_feedback = code_feedback
+        self.value_feedback = value_feedback
+        self.final_decision = final_decision
+        self.final_feedback = final_feedback
+        self.value_generated_flag = value_generated_flag
+        self.final_decision_based_on_gt = final_decision_based_on_gt
+
+        # TODO:
+        # Not general enough. So we should not put them in the general costeer feedback
+        # Instead, we should create subclass for it.
+        self.shape_feedback = shape_feedback  # Not general enough. So
+
+    @property
+    def execution(self):
+        return self.execution_feedback
+
+    @execution.setter
+    def execution(self, value):
+        self.execution_feedback = value
+
+    @property
+    def return_checking(self):
+        if self.value_generated_flag:
+            return f"value feedback: {self.value_feedback}\n\nshape feedback: {self.shape_feedback}"
+        return None
+
+    @return_checking.setter
+    def return_checking(self, value):
+        # Since return_checking is derived from value_feedback and shape_feedback,
+        # we don't need to do anything here
+        self.value_feedback = value
+        self.shape_feedback = value
+
+    @property
+    def code(self):
+        return self.code_feedback
+
+    @code.setter
+    def code(self, value):
+        self.code_feedback = value
+
+    def __str__(self) -> str:
+        return f"""------------------Execution Feedback------------------
+{self.execution_feedback if self.execution_feedback is not None else 'No execution feedback'}
+------------------Shape Feedback------------------
+{self.shape_feedback if self.shape_feedback is not None else 'No shape feedback'}
+------------------Code Feedback------------------
+{self.code_feedback if self.code_feedback is not None else 'No code feedback'}
+------------------Value Feedback------------------
+{self.value_feedback if self.value_feedback is not None else 'No value feedback'}
+------------------Final Feedback------------------
+{self.final_feedback if self.final_feedback is not None else 'No final feedback'}
+------------------Final Decision------------------
+This implementation is {'SUCCESS' if self.final_decision else 'FAIL'}.
+"""
+
+
+class CoSTEERMultiFeedback(Feedback):
+    """Feedback contains a list, each element is the corresponding feedback for each factor implementation."""
+
+    def __init__(self, feedback_list: List[CoSTEERSingleFeedback]) -> None:
+        self.feedback_list = feedback_list
+
+    def __getitem__(self, index: int) -> CoSTEERSingleFeedback:
+        return self.feedback_list[index]
+
+    def __len__(self) -> int:
+        return len(self.feedback_list)
+
+    def append(self, feedback: CoSTEERSingleFeedback) -> None:
+        self.feedback_list.append(feedback)
+
+    def __iter__(self):
+        return iter(self.feedback_list)
+
+    def finished(self) -> bool:
+        """
+        In some implementations, tasks may fail multiple times, leading agents to skip the implementation.
+        This results in None feedback. However, we want to accept the correct parts and ignore None feedback.
+        """
+        return all(feedback.final_decision for feedback in self.feedback_list if feedback is not None)
+
+    def __bool__(self) -> bool:
+        return all(feedback.final_decision for feedback in self.feedback_list)
+
+
+class CoSTEEREvaluator(Evaluator):
+    def __init__(
+        self,
+        scen: "Scenario",
+    ) -> None:
+        self.scen = scen
+
+    # TODO:
+    # I think we should have unified interface for all evaluates, for examples.
+    # So we should adjust the interface of other factors
+    @abstractmethod
+    def evaluate(
+        self,
+        target_task: Task,
+        implementation: Workspace,
+        gt_implementation: Workspace,
+        **kwargs,
+    ) -> CoSTEERSingleFeedback:
+        raise NotImplementedError("Please implement the `evaluator` method")
+
+
+class CoSTEERMultiEvaluator(CoSTEEREvaluator):
+    """This is for evaluation of experiment. Due to we have multiple tasks, so we will return a list of evaluation feebacks"""
+
+    def __init__(self, single_evaluator: CoSTEEREvaluator | list[CoSTEEREvaluator], *args, **kwargs) -> None:
+        super().__init__(*args, **kwargs)
+        self.single_evaluator = single_evaluator
+
+    def evaluate(
+        self,
+        evo: EvolvingItem,
+        queried_knowledge: QueriedKnowledge = None,
+        **kwargs,
+    ) -> CoSTEERMultiFeedback:
+        eval_l = self.single_evaluator if isinstance(self.single_evaluator, list) else [self.single_evaluator]
+        task_li_feedback_li = []
+        for ev in eval_l:
+            multi_implementation_feedback = multiprocessing_wrapper(
+                [
+                    (
+                        ev.evaluate,
+                        (
+                            evo.sub_tasks[index],
+                            evo.sub_workspace_list[index],
+                            evo.sub_gt_implementations[index] if evo.sub_gt_implementations is not None else None,
+                            queried_knowledge,
+                        ),
+                    )
+                    for index in range(len(evo.sub_tasks))
+                ],
+                n=RD_AGENT_SETTINGS.multi_proc_n,
+            )
+            task_li_feedback_li.append(multi_implementation_feedback)
+        # merge the feedbacks
+        merged_task_feedback = []
+        for task_id, fb in enumerate(task_li_feedback_li[0]):
+            fb = deepcopy(fb)  # deep copy to make it more robust
+
+            fb.final_decision = all(
+                task_li_feedback[task_id].final_decision for task_li_feedback in task_li_feedback_li
+            )
+            for attr in "execution", "return_checking", "code":
+                setattr(
+                    fb,
+                    attr,
+                    "\n\n".join(
+                        [
+                            getattr(task_li_feedback[task_id], attr)
+                            for task_li_feedback in task_li_feedback_li
+                            if getattr(task_li_feedback[task_id], attr) is not None
+                        ]
+                    ),
+                )
+            merged_task_feedback.append(fb)
+
+        final_decision = [
+            None if single_feedback is None else single_feedback.final_decision
+            for single_feedback in merged_task_feedback
+        ]
+        logger.info(f"Final decisions: {final_decision} True count: {final_decision.count(True)}")
+
+        # TODO: this is to be compatible with factor_implementation;
+        for index in range(len(evo.sub_tasks)):
+            if final_decision[index]:
+                evo.sub_tasks[index].factor_implementation = True
+
+        return CoSTEERMultiFeedback(merged_task_feedback)

rdagent/components/coder/CoSTEER/evolvable_subjects.py

@@ -0,0 +1,32 @@
+from rdagent.core.evolving_framework import EvolvableSubjects
+from rdagent.core.experiment import Experiment, FBWorkspace, Task
+from rdagent.log import rdagent_logger as logger
+
+
+class EvolvingItem(Experiment, EvolvableSubjects):
+    """
+    Intermediate item of factor implementation.
+    """
+
+    def __init__(
+        self,
+        sub_tasks: list[Task],
+        sub_gt_implementations: list[FBWorkspace] = None,
+    ):
+        Experiment.__init__(self, sub_tasks=sub_tasks)
+        if sub_gt_implementations is not None and len(
+            sub_gt_implementations,
+        ) != len(self.sub_tasks):
+            self.sub_gt_implementations = None
+            logger.warning(
+                "The length of sub_gt_implementations is not equal to the length of sub_tasks, set sub_gt_implementations to None",
+            )
+        else:
+            self.sub_gt_implementations = sub_gt_implementations
+
+    @classmethod
+    def from_experiment(cls, exp: Experiment) -> Experiment:
+        ei = cls(sub_tasks=exp.sub_tasks)
+        ei.based_experiments = exp.based_experiments
+        ei.experiment_workspace = exp.experiment_workspace
+        return ei

rdagent/components/coder/CoSTEER/evolving_strategy.py

@@ -0,0 +1,120 @@
+from __future__ import annotations
+
+from abc import abstractmethod
+
+from rdagent.components.coder.CoSTEER.config import CoSTEERSettings
+from rdagent.components.coder.CoSTEER.evaluators import (
+    CoSTEERMultiFeedback,
+    CoSTEERSingleFeedback,
+)
+from rdagent.components.coder.CoSTEER.evolvable_subjects import EvolvingItem
+from rdagent.components.coder.CoSTEER.knowledge_management import (
+    CoSTEERQueriedKnowledge,
+)
+from rdagent.core.conf import RD_AGENT_SETTINGS
+from rdagent.core.evolving_framework import EvolvingStrategy, EvoStep, QueriedKnowledge
+from rdagent.core.experiment import FBWorkspace, Task
+from rdagent.core.scenario import Scenario
+from rdagent.core.utils import multiprocessing_wrapper
+
+
+class MultiProcessEvolvingStrategy(EvolvingStrategy):
+    def __init__(self, scen: Scenario, settings: CoSTEERSettings):
+        super().__init__(scen)
+        self.settings = settings
+
+    @abstractmethod
+    def implement_one_task(
+        self,
+        target_task: Task,
+        queried_knowledge: QueriedKnowledge | None = None,
+        workspace: FBWorkspace | None = None,
+        prev_task_feedback: CoSTEERSingleFeedback | None = None,
+    ) -> dict[str, str]:  # FIXME: fix interface of previous implement
+        """
+        This method will input the task & current workspace,
+        and output the modification to applied to the workspace.
+        (i.e. replace the content <filename> with <content>)
+
+        Parameters
+        ----------
+        target_task : Task
+
+        queried_knowledge : QueriedKnowledge | None
+
+        workspace : FBWorkspace | None
+
+        prev_task_feedback : CoSTEERSingleFeedback | None
+            task feedback for previous evolving step
+            None indicate it is the first loop.
+
+        Return
+        ------
+        The new files {<filename>: <content>} to update the workspace.
+        """
+        raise NotImplementedError
+
+    @abstractmethod
+    def assign_code_list_to_evo(self, code_list: list[dict], evo: EvolvingItem) -> None:
+        """
+        Assign the code list to the evolving item.
+
+        Due to the implement_one_task take `workspace` as input and output the `modification`.
+        We should apply implementation to evo
+
+        The code list is aligned with the evolving item's sub-tasks.
+        If a task is not implemented, put a None in the list.
+        """
+        raise NotImplementedError
+
+    def evolve(
+        self,
+        *,
+        evo: EvolvingItem,
+        queried_knowledge: CoSTEERQueriedKnowledge | None = None,
+        evolving_trace: list[EvoStep] = [],
+        **kwargs,
+    ) -> EvolvingItem:
+        # 1.找出需要evolve的task
+        to_be_finished_task_index: list[int] = []
+        for index, target_task in enumerate(evo.sub_tasks):
+            target_task_desc = target_task.get_task_information()
+            if target_task_desc in queried_knowledge.success_task_to_knowledge_dict:
+                # NOTE: very weird logic:
+                # it depends on the knowledge to set the already finished task
+                evo.sub_workspace_list[index] = queried_knowledge.success_task_to_knowledge_dict[
+                    target_task_desc
+                ].implementation
+            elif (
+                target_task_desc not in queried_knowledge.success_task_to_knowledge_dict
+                and target_task_desc not in queried_knowledge.failed_task_info_set
+            ):
+                to_be_finished_task_index.append(index)
+
+        last_feedback = None
+        if len(evolving_trace) > 0:
+            last_feedback = evolving_trace[-1].feedback
+            assert isinstance(last_feedback, CoSTEERMultiFeedback)
+
+        result = multiprocessing_wrapper(
+            [
+                (
+                    self.implement_one_task,
+                    (
+                        evo.sub_tasks[target_index],
+                        queried_knowledge,
+                        evo.experiment_workspace,
+                        None if last_feedback is None else last_feedback[target_index],
+                    ),
+                )
+                for target_index in to_be_finished_task_index
+            ],
+            n=RD_AGENT_SETTINGS.multi_proc_n,
+        )
+        code_list = [None for _ in range(len(evo.sub_tasks))]
+        for index, target_index in enumerate(to_be_finished_task_index):
+            code_list[target_index] = result[index]
+
+        evo = self.assign_code_list_to_evo(code_list, evo)
+
+        return evo

rdagent/components/coder/CoSTEER/knowledge_management.py

@@ -6,93 +6,90 @@ import random
 import re
 from itertools import combinations
 from pathlib import Path
-from jinja2 import Template
-from typing import Union
+from typing import List, Union
 
-from jinja2 import Template
+from rdagent.components.coder.CoSTEER.config import CoSTEERSettings
+from rdagent.components.coder.CoSTEER.evaluators import CoSTEERSingleFeedback
+from rdagent.components.knowledge_management.graph import (
+    UndirectedGraph,
+    UndirectedNode,
+)
+from rdagent.core.evolving_agent import Feedback
 from rdagent.core.evolving_framework import (
     EvolvableSubjects,
+    EvolvingKnowledgeBase,
     EvoStep,
     Knowledge,
-    KnowledgeBase,
     QueriedKnowledge,
     RAGStrategy,
 )
-from rdagent.core.log import RDAgentLog
-from rdagent.core.prompts import Prompts
-from rdagent.factor_implementation.evolving.evaluators import FactorImplementationSingleFeedback
-from rdagent.core.task import (
-    TaskImplementation,
-)
-from rdagent.factor_implementation.evolving.evolving_strategy import FactorImplementTask
-from rdagent.core.prompts import Prompts
-from rdagent.knowledge_management.graph import UndirectedGraph, UndirectedNode
-from rdagent.oai.llm_utils import APIBackend, calculate_embedding_distance_between_str_list
-
-from rdagent.factor_implementation.share_modules.factor_implementation_config import (
-    FACTOR_IMPLEMENT_SETTINGS,
+from rdagent.core.experiment import FBWorkspace, Task
+from rdagent.log import rdagent_logger as logger
+from rdagent.oai.llm_utils import (
+    APIBackend,
+    calculate_embedding_distance_between_str_list,
 )
+from rdagent.utils.agent.tpl import T
 
 
-class FactorImplementationKnowledge(Knowledge):
+class CoSTEERKnowledge(Knowledge):
     def __init__(
         self,
-        target_task: FactorImplementTask,
-        implementation: TaskImplementation,
-        feedback: FactorImplementationSingleFeedback,
+        target_task: Task,
+        implementation: FBWorkspace,
+        feedback: Feedback,
     ) -> None:
-        """
-        Initialize a FactorKnowledge object. The FactorKnowledge object is used to store a factor implementation without the ground truth code and value.
-
-        Args:
-            factor (Factor): The factor object associated with the KnowledgeManagement.
-
-        Returns:
-            None
-        """
         self.target_task = target_task
-        self.implementation = implementation
+        self.implementation = implementation.copy()
         self.feedback = feedback
 
     def get_implementation_and_feedback_str(self) -> str:
-        return f"""------------------Factor implementation code:------------------
-{self.implementation.code}
-------------------Factor implementation feedback:------------------
+        return f"""------------------implementation code:------------------
+{self.implementation.all_codes}
+------------------implementation feedback:------------------
 {self.feedback!s}
 """
 
 
-class FactorImplementationQueriedKnowledge(QueriedKnowledge):
+class CoSTEERQueriedKnowledge(QueriedKnowledge):
     def __init__(self, success_task_to_knowledge_dict: dict = {}, failed_task_info_set: set = set()) -> None:
         self.success_task_to_knowledge_dict = success_task_to_knowledge_dict
         self.failed_task_info_set = failed_task_info_set
 
 
-class FactorImplementationKnowledgeBaseV1(KnowledgeBase):
-    def __init__(self) -> None:
-        self.implementation_trace: dict[str, FactorImplementationKnowledge] = dict()
+class CoSTEERKnowledgeBaseV1(EvolvingKnowledgeBase):
+    def __init__(self, path: str | Path = None) -> None:
+        self.implementation_trace: dict[str, CoSTEERKnowledge] = dict()
         self.success_task_info_set: set[str] = set()
 
         self.task_to_embedding = dict()
+        super().__init__(path)
 
-    def query(self) -> QueriedKnowledge | None:
+    def query(self) -> CoSTEERQueriedKnowledge | None:
         """
         Query the knowledge base to get the queried knowledge. So far is handled in RAG strategy.
         """
         raise NotImplementedError
 
 
-class FactorImplementationQueriedKnowledgeV1(FactorImplementationQueriedKnowledge):
-    def __init__(self) -> None:
-        self.working_task_to_former_failed_knowledge_dict = dict()
-        self.working_task_to_similar_successful_knowledge_dict = dict()
-        super().__init__()
+class CoSTEERQueriedKnowledgeV1(CoSTEERQueriedKnowledge):
+    def __init__(
+        self,
+        *args,
+        task_to_former_failed_traces: dict = {},
+        task_to_similar_task_successful_knowledge: dict = {},
+        **kwargs,
+    ) -> None:
+        self.task_to_former_failed_traces = task_to_former_failed_traces
+        self.task_to_similar_task_successful_knowledge = task_to_similar_task_successful_knowledge
+        super().__init__(*args, **kwargs)
 
 
-class FactorImplementationRAGStrategyV1(RAGStrategy):
-    def __init__(self, knowledgebase: FactorImplementationKnowledgeBaseV1) -> None:
+class CoSTEERRAGStrategyV1(RAGStrategy):
+    def __init__(self, knowledgebase: CoSTEERKnowledgeBaseV1, settings: CoSTEERSettings) -> None:
         super().__init__(knowledgebase)
         self.current_generated_trace_count = 0
+        self.settings = settings
 
     def generate_knowledge(
         self,
@@ -100,6 +97,9 @@ class FactorImplementationRAGStrategyV1(RAGStrategy):
         *,
         return_knowledge: bool = False,
     ) -> Knowledge | None:
+        raise NotImplementedError(
+            "This method should be considered as an un-implemented method because we encourage everyone to use v2."
+        )
         if len(evolving_trace) == self.current_generated_trace_count:
             return
         else:
@@ -110,14 +110,14 @@ class FactorImplementationRAGStrategyV1(RAGStrategy):
                 evo_step = evolving_trace[trace_index]
                 implementations = evo_step.evolvable_subjects
                 feedback = evo_step.feedback
-                for task_index in range(len(implementations.target_factor_tasks)):
-                    target_task = implementations.target_factor_tasks[task_index]
-                    target_task_information = target_task.get_factor_information()
-                    implementation = implementations.corresponding_implementations[task_index]
+                for task_index in range(len(implementations.sub_tasks)):
+                    target_task = implementations.sub_tasks[task_index]
+                    target_task_information = target_task.get_task_information()
+                    implementation = implementations.sub_workspace_list[task_index]
                     single_feedback = feedback[task_index]
                     if single_feedback is None:
                         continue
-                    single_knowledge = FactorImplementationKnowledge(
+                    single_knowledge = CoSTEERKnowledge(
                         target_task=target_task,
                         implementation=implementation,
                         feedback=single_feedback,
@@ -138,32 +138,35 @@ class FactorImplementationRAGStrategyV1(RAGStrategy):
         self,
         evo: EvolvableSubjects,
         evolving_trace: list[EvoStep],
-    ) -> QueriedKnowledge | None:
-        v1_query_former_trace_limit = FACTOR_IMPLEMENT_SETTINGS.v1_query_former_trace_limit
-        v1_query_similar_success_limit = FACTOR_IMPLEMENT_SETTINGS.v1_query_similar_success_limit
-        fail_task_trial_limit = FACTOR_IMPLEMENT_SETTINGS.fail_task_trial_limit
+    ) -> CoSTEERQueriedKnowledge | None:
+        raise NotImplementedError(
+            "This method should be considered as an un-implemented method because we encourage everyone to use v2."
+        )
+        v1_query_former_trace_limit = self.settings.v1_query_former_trace_limit
+        v1_query_similar_success_limit = self.settings.v1_query_similar_success_limit
+        fail_task_trial_limit = self.settings.fail_task_trial_limit
 
-        queried_knowledge = FactorImplementationQueriedKnowledgeV1()
-        for target_factor_task in evo.target_factor_tasks:
-            target_factor_task_information = target_factor_task.get_factor_information()
-            if target_factor_task_information in self.knowledgebase.success_task_info_set:
-                queried_knowledge.success_task_to_knowledge_dict[target_factor_task_information] = (
-                    self.knowledgebase.implementation_trace[target_factor_task_information][-1]
+        queried_knowledge = CoSTEERQueriedKnowledgeV1()
+        for target_task in evo.sub_tasks:
+            target_task_information = target_task.get_task_information()
+            if target_task_information in self.knowledgebase.success_task_info_set:
+                queried_knowledge.success_task_to_knowledge_dict[target_task_information] = (
+                    self.knowledgebase.implementation_trace[target_task_information][-1]
                 )
             elif (
                 len(
                     self.knowledgebase.implementation_trace.setdefault(
-                        target_factor_task_information,
+                        target_task_information,
                         [],
                     ),
                 )
                 >= fail_task_trial_limit
             ):
-                queried_knowledge.failed_task_info_set.add(target_factor_task_information)
+                queried_knowledge.failed_task_info_set.add(target_task_information)
             else:
-                queried_knowledge.working_task_to_former_failed_knowledge_dict[target_factor_task_information] = (
+                queried_knowledge.task_to_former_failed_traces[target_task_information] = (
                     self.knowledgebase.implementation_trace.setdefault(
-                        target_factor_task_information,
+                        target_task_information,
                         [],
                     )[-v1_query_former_trace_limit:]
                 )
@@ -172,7 +175,7 @@ class FactorImplementationRAGStrategyV1(RAGStrategy):
                     self.knowledgebase.success_task_info_set,
                 )
                 similarity = calculate_embedding_distance_between_str_list(
-                    [target_factor_task_information],
+                    [target_task_information],
                     knowledge_base_success_task_list,
                 )[0]
                 similar_indexes = sorted(
@@ -187,32 +190,34 @@ class FactorImplementationRAGStrategyV1(RAGStrategy):
                     )[-1]
                     for index in similar_indexes
                 ]
-                queried_knowledge.working_task_to_similar_successful_knowledge_dict[target_factor_task_information] = (
+                queried_knowledge.task_to_similar_task_successful_knowledge[target_task_information] = (
                     similar_successful_knowledge
                 )
         return queried_knowledge
 
 
-class FactorImplementationQueriedGraphKnowledge(FactorImplementationQueriedKnowledge):
+class CoSTEERQueriedKnowledgeV2(CoSTEERQueriedKnowledgeV1):
     # Aggregation of knowledge
     def __init__(
         self,
-        former_traces: dict = {},
-        component_with_success_task: dict = {},
-        error_with_success_task: dict = {},
+        task_to_former_failed_traces: dict = {},
+        task_to_similar_task_successful_knowledge: dict = {},
+        task_to_similar_error_successful_knowledge: dict = {},
         **kwargs,
     ) -> None:
-        self.former_traces = former_traces
-        self.component_with_success_task = component_with_success_task
-        self.error_with_success_task = error_with_success_task
-        super().__init__(**kwargs)
+        self.task_to_similar_error_successful_knowledge = task_to_similar_error_successful_knowledge
+        super().__init__(
+            task_to_former_failed_traces=task_to_former_failed_traces,
+            task_to_similar_task_successful_knowledge=task_to_similar_task_successful_knowledge,
+            **kwargs,
+        )
 
 
-class FactorImplementationGraphRAGStrategy(RAGStrategy):
-    def __init__(self, knowledgebase: FactorImplementationGraphKnowledgeBase) -> None:
+class CoSTEERRAGStrategyV2(RAGStrategy):
+    def __init__(self, knowledgebase: CoSTEERKnowledgeBaseV2, settings: CoSTEERSettings) -> None:
         super().__init__(knowledgebase)
         self.current_generated_trace_count = 0
-        self.prompt = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")
+        self.settings = settings
 
     def generate_knowledge(
         self,
@@ -228,15 +233,14 @@ class FactorImplementationGraphRAGStrategy(RAGStrategy):
                 evo_step = evolving_trace[trace_index]
                 implementations = evo_step.evolvable_subjects
                 feedback = evo_step.feedback
-                for task_index in range(len(implementations.target_factor_tasks)):
-                    single_feedback = feedback[task_index]
-                    target_task = implementations.target_factor_tasks[task_index]
-                    target_task_information = target_task.get_factor_information()
-                    implementation = implementations.corresponding_implementations[task_index]
-                    single_feedback = feedback[task_index]
-                    if single_feedback is None:
+                for task_index in range(len(implementations.sub_tasks)):
+                    target_task = implementations.sub_tasks[task_index]
+                    target_task_information = target_task.get_task_information()
+                    implementation = implementations.sub_workspace_list[task_index]
+                    single_feedback: CoSTEERSingleFeedback = feedback[task_index]
+                    if implementation is None or single_feedback is None:
                         continue
-                    single_knowledge = FactorImplementationKnowledge(
+                    single_knowledge = CoSTEERKnowledge(
                         target_task=target_task,
                         implementation=implementation,
                         feedback=single_feedback,
@@ -260,15 +264,15 @@ class FactorImplementationGraphRAGStrategy(RAGStrategy):
                         else:
                             # generate error node and store into knowledge base
                             error_analysis_result = []
-                            if not single_feedback.value_generated_flag:
+                            if single_feedback.return_checking:
                                 error_analysis_result = self.analyze_error(
-                                    single_feedback.execution_feedback,
-                                    feedback_type="execution",
+                                    single_feedback.return_checking,
+                                    feedback_type="value",
                                 )
                             else:
                                 error_analysis_result = self.analyze_error(
-                                    single_feedback.factor_value_feedback,
-                                    feedback_type="value",
+                                    single_feedback.execution,
+                                    feedback_type="execution",
                                 )
                             self.knowledgebase.working_trace_error_analysis.setdefault(
                                 target_task_information,
@@ -280,55 +284,59 @@ class FactorImplementationGraphRAGStrategy(RAGStrategy):
             self.current_generated_trace_count = len(evolving_trace)
             return None
 
-    def query(self, evo: EvolvableSubjects, evolving_trace: list[EvoStep]) -> QueriedKnowledge | None:
-        conf_knowledge_sampler = FACTOR_IMPLEMENT_SETTINGS.v2_knowledge_sampler
-        factor_implementation_queried_graph_knowledge = FactorImplementationQueriedGraphKnowledge(
+    def query(self, evo: EvolvableSubjects, evolving_trace: list[EvoStep]) -> CoSTEERQueriedKnowledge | None:
+        conf_knowledge_sampler = self.settings.v2_knowledge_sampler
+        queried_knowledge_v2 = CoSTEERQueriedKnowledgeV2(
             success_task_to_knowledge_dict=self.knowledgebase.success_task_to_knowledge_dict,
         )
 
-        factor_implementation_queried_graph_knowledge = self.former_trace_query(
+        queried_knowledge_v2 = self.former_trace_query(
             evo,
-            factor_implementation_queried_graph_knowledge,
-            FACTOR_IMPLEMENT_SETTINGS.v2_query_former_trace_limit,
+            queried_knowledge_v2,
+            self.settings.v2_query_former_trace_limit,
+            self.settings.v2_add_fail_attempt_to_latest_successful_execution,
         )
-        factor_implementation_queried_graph_knowledge = self.component_query(
+        queried_knowledge_v2 = self.component_query(
             evo,
-            factor_implementation_queried_graph_knowledge,
-            FACTOR_IMPLEMENT_SETTINGS.v2_query_component_limit,
+            queried_knowledge_v2,
+            self.settings.v2_query_component_limit,
             knowledge_sampler=conf_knowledge_sampler,
         )
-        factor_implementation_queried_graph_knowledge = self.error_query(
+        queried_knowledge_v2 = self.error_query(
             evo,
-            factor_implementation_queried_graph_knowledge,
-            FACTOR_IMPLEMENT_SETTINGS.v2_query_error_limit,
+            queried_knowledge_v2,
+            self.settings.v2_query_error_limit,
             knowledge_sampler=conf_knowledge_sampler,
         )
-        return factor_implementation_queried_graph_knowledge
+        return queried_knowledge_v2
 
     def analyze_component(
         self,
-        target_factor_task_information,
+        target_task_information,
     ) -> list[UndirectedNode]:  # Hardcode: certain component nodes
         all_component_nodes = self.knowledgebase.graph.get_all_nodes_by_label_list(["component"])
+        if not len(all_component_nodes):
+            return []
         all_component_content = ""
         for _, component_node in enumerate(all_component_nodes):
             all_component_content += f"{component_node.content}, \n"
-        analyze_component_system_prompt = Template(self.prompt["analyze_component_prompt_v1_system"]).render(
+        analyze_component_system_prompt = T(".prompts:analyze_component_prompt_v1_system").r(
             all_component_content=all_component_content,
         )
 
-        analyze_component_user_prompt = target_factor_task_information
+        analyze_component_user_prompt = target_task_information
         try:
             component_no_list = json.loads(
                 APIBackend().build_messages_and_create_chat_completion(
                     system_prompt=analyze_component_system_prompt,
                     user_prompt=analyze_component_user_prompt,
                     json_mode=True,
+                    json_target_type=List[int],
                 ),
             )["component_no_list"]
             return [all_component_nodes[index - 1] for index in sorted(list(set(component_no_list)))]
         except:
-            RDAgentLog().warning("Error when analyzing components.")
+            logger.warning("Error when analyzing components.")
             analyze_component_user_prompt = "Your response is not a valid component index list."
 
         return []
@@ -378,77 +386,87 @@ class FactorImplementationGraphRAGStrategy(RAGStrategy):
     def former_trace_query(
         self,
         evo: EvolvableSubjects,
-        factor_implementation_queried_graph_knowledge: FactorImplementationQueriedGraphKnowledge,
+        queried_knowledge_v2: CoSTEERQueriedKnowledgeV2,
         v2_query_former_trace_limit: int = 5,
-    ) -> Union[QueriedKnowledge, set]:
+        v2_add_fail_attempt_to_latest_successful_execution: bool = False,
+    ) -> Union[CoSTEERQueriedKnowledge, set]:
         """
         Query the former trace knowledge of the working trace, and find all the failed task information which tried more than fail_task_trial_limit times
         """
-        fail_task_trial_limit = FACTOR_IMPLEMENT_SETTINGS.fail_task_trial_limit
+        fail_task_trial_limit = self.settings.fail_task_trial_limit
 
-        for target_factor_task in evo.target_factor_tasks:
-            target_factor_task_information = target_factor_task.get_factor_information()
+        for target_task in evo.sub_tasks:
+            target_task_information = target_task.get_task_information()
             if (
-                target_factor_task_information not in self.knowledgebase.success_task_to_knowledge_dict
-                and target_factor_task_information in self.knowledgebase.working_trace_knowledge
-                and len(self.knowledgebase.working_trace_knowledge[target_factor_task_information])
-                >= fail_task_trial_limit
+                target_task_information not in self.knowledgebase.success_task_to_knowledge_dict
+                and target_task_information in self.knowledgebase.working_trace_knowledge
+                and len(self.knowledgebase.working_trace_knowledge[target_task_information]) >= fail_task_trial_limit
             ):
-                factor_implementation_queried_graph_knowledge.failed_task_info_set.add(target_factor_task_information)
+                queried_knowledge_v2.failed_task_info_set.add(target_task_information)
 
             if (
-                target_factor_task_information not in self.knowledgebase.success_task_to_knowledge_dict
-                and target_factor_task_information
-                not in factor_implementation_queried_graph_knowledge.failed_task_info_set
-                and target_factor_task_information in self.knowledgebase.working_trace_knowledge
+                target_task_information not in self.knowledgebase.success_task_to_knowledge_dict
+                and target_task_information not in queried_knowledge_v2.failed_task_info_set
+                and target_task_information in self.knowledgebase.working_trace_knowledge
             ):
                 former_trace_knowledge = copy.copy(
-                    self.knowledgebase.working_trace_knowledge[target_factor_task_information],
+                    self.knowledgebase.working_trace_knowledge[target_task_information],
                 )
                 # in former trace query we will delete the right trace in the following order:[..., value_generated_flag is True, value_generated_flag is False, ...]
                 # because we think this order means a deterioration of the trial (like a wrong gradient descent)
                 current_index = 1
                 while current_index < len(former_trace_knowledge):
                     if (
-                        not former_trace_knowledge[current_index].feedback.value_generated_flag
-                        and former_trace_knowledge[current_index - 1].feedback.value_generated_flag
+                        not former_trace_knowledge[current_index].feedback.return_checking
+                        and former_trace_knowledge[current_index - 1].feedback.return_checking
                     ):
                         former_trace_knowledge.pop(current_index)
                     else:
                         current_index += 1
 
-                factor_implementation_queried_graph_knowledge.former_traces[target_factor_task_information] = (
-                    former_trace_knowledge[-v2_query_former_trace_limit:]
+                latest_attempt = None
+                if v2_add_fail_attempt_to_latest_successful_execution:
+                    # When the last successful execution is not the last one in the working trace, it means we have tried to correct it. We should tell the agent this fail trial to avoid endless loop in the future.
+                    if (
+                        len(former_trace_knowledge) > 0
+                        and len(self.knowledgebase.working_trace_knowledge[target_task_information]) > 1
+                        and self.knowledgebase.working_trace_knowledge[target_task_information].index(
+                            former_trace_knowledge[-1]
+                        )
+                        < len(self.knowledgebase.working_trace_knowledge[target_task_information]) - 1
+                    ):
+                        latest_attempt = self.knowledgebase.working_trace_knowledge[target_task_information][-1]
+
+                queried_knowledge_v2.task_to_former_failed_traces[target_task_information] = (
+                    former_trace_knowledge[-v2_query_former_trace_limit:],
+                    latest_attempt,
                 )
             else:
-                factor_implementation_queried_graph_knowledge.former_traces[target_factor_task_information] = []
+                queried_knowledge_v2.task_to_former_failed_traces[target_task_information] = ([], None)
 
-        return factor_implementation_queried_graph_knowledge
+        return queried_knowledge_v2
 
     def component_query(
         self,
         evo: EvolvableSubjects,
-        factor_implementation_queried_graph_knowledge: FactorImplementationQueriedGraphKnowledge,
+        queried_knowledge_v2: CoSTEERQueriedKnowledgeV2,
         v2_query_component_limit: int = 5,
         knowledge_sampler: float = 1.0,
-    ) -> QueriedKnowledge | None:
-        # queried_component_knowledge = FactorImplementationQueriedGraphComponentKnowledge()
-        for target_factor_task in evo.target_factor_tasks:
-            target_factor_task_information = target_factor_task.get_factor_information()
+    ) -> CoSTEERQueriedKnowledge | None:
+        for target_task in evo.sub_tasks:
+            target_task_information = target_task.get_task_information()
             if (
-                target_factor_task_information in self.knowledgebase.success_task_to_knowledge_dict
-                or target_factor_task_information in factor_implementation_queried_graph_knowledge.failed_task_info_set
+                target_task_information in self.knowledgebase.success_task_to_knowledge_dict
+                or target_task_information in queried_knowledge_v2.failed_task_info_set
             ):
-                factor_implementation_queried_graph_knowledge.component_with_success_task[
-                    target_factor_task_information
-                ] = []
+                queried_knowledge_v2.task_to_similar_task_successful_knowledge[target_task_information] = []
             else:
-                if target_factor_task_information not in self.knowledgebase.task_to_component_nodes:
-                    self.knowledgebase.task_to_component_nodes[target_factor_task_information] = self.analyze_component(
-                        target_factor_task_information,
+                if target_task_information not in self.knowledgebase.task_to_component_nodes:
+                    self.knowledgebase.task_to_component_nodes[target_task_information] = self.analyze_component(
+                        target_task_information,
                     )
 
-                component_analysis_result = self.knowledgebase.task_to_component_nodes[target_factor_task_information]
+                component_analysis_result = self.knowledgebase.task_to_component_nodes[target_task_information]
 
                 if len(component_analysis_result) > 1:
                     task_des_node_list = self.knowledgebase.graph_query_by_intersection(
@@ -459,9 +477,7 @@ class FactorImplementationGraphRAGStrategy(RAGStrategy):
                 else:
                     task_des_node_list = []
                     single_component_constraint = v2_query_component_limit
-                factor_implementation_queried_graph_knowledge.component_with_success_task[
-                    target_factor_task_information
-                ] = []
+                queried_knowledge_v2.task_to_similar_task_successful_knowledge[target_task_information] = []
                 for component_node in component_analysis_result:
                     # Reverse iterate, a trade-off with intersection search
                     count = 0
@@ -492,19 +508,19 @@ class FactorImplementationGraphRAGStrategy(RAGStrategy):
                             ]
                         if (
                             target_knowledge
-                            not in factor_implementation_queried_graph_knowledge.component_with_success_task[
-                                target_factor_task_information
+                            not in queried_knowledge_v2.task_to_similar_task_successful_knowledge[
+                                target_task_information
                             ]
                         ):
-                            factor_implementation_queried_graph_knowledge.component_with_success_task[
-                                target_factor_task_information
+                            queried_knowledge_v2.task_to_similar_task_successful_knowledge[
+                                target_task_information
                             ].append(target_knowledge)
 
                 # finally add embedding related knowledge
                 knowledge_base_success_task_list = list(self.knowledgebase.success_task_to_knowledge_dict)
 
                 similarity = calculate_embedding_distance_between_str_list(
-                    [target_factor_task_information],
+                    [target_task_information],
                     knowledge_base_success_task_list,
                 )[0]
                 similar_indexes = sorted(
@@ -519,28 +535,24 @@ class FactorImplementationGraphRAGStrategy(RAGStrategy):
                 for knowledge in embedding_similar_successful_knowledge:
                     if (
                         knowledge
-                        not in factor_implementation_queried_graph_knowledge.component_with_success_task[
-                            target_factor_task_information
-                        ]
+                        not in queried_knowledge_v2.task_to_similar_task_successful_knowledge[target_task_information]
                     ):
-                        factor_implementation_queried_graph_knowledge.component_with_success_task[
-                            target_factor_task_information
-                        ].append(knowledge)
+                        queried_knowledge_v2.task_to_similar_task_successful_knowledge[target_task_information].append(
+                            knowledge
+                        )
 
                 if knowledge_sampler > 0:
-                    factor_implementation_queried_graph_knowledge.component_with_success_task[
-                        target_factor_task_information
-                    ] = [
+                    queried_knowledge_v2.task_to_similar_task_successful_knowledge[target_task_information] = [
                         knowledge
-                        for knowledge in factor_implementation_queried_graph_knowledge.component_with_success_task[
-                            target_factor_task_information
+                        for knowledge in queried_knowledge_v2.task_to_similar_task_successful_knowledge[
+                            target_task_information
                         ]
                         if random.uniform(0, 1) <= knowledge_sampler
                     ]
 
                 # Make sure no less than half of the knowledge are from GT
-                queried_knowledge_list = factor_implementation_queried_graph_knowledge.component_with_success_task[
-                    target_factor_task_information
+                queried_knowledge_list = queried_knowledge_v2.task_to_similar_task_successful_knowledge[
+                    target_task_information
                 ]
                 queried_from_gt_knowledge_list = [
                     knowledge
@@ -553,54 +565,46 @@ class FactorImplementationGraphRAGStrategy(RAGStrategy):
                     if knowledge.feedback is not None and knowledge.feedback.final_decision_based_on_gt == False
                 ]
                 queried_from_gt_knowledge_count = max(
-                    min(v2_query_component_limit // 2, len(queried_from_gt_knowledge_list)),
+                    min((v2_query_component_limit // 2 + 1), len(queried_from_gt_knowledge_list)),
                     v2_query_component_limit - len(queried_without_gt_knowledge_list),
                 )
-                factor_implementation_queried_graph_knowledge.component_with_success_task[
-                    target_factor_task_information
-                ] = (
+                queried_knowledge_v2.task_to_similar_task_successful_knowledge[target_task_information] = (
                     queried_from_gt_knowledge_list[:queried_from_gt_knowledge_count]
                     + queried_without_gt_knowledge_list[: v2_query_component_limit - queried_from_gt_knowledge_count]
                 )
 
-        return factor_implementation_queried_graph_knowledge
+        return queried_knowledge_v2
 
     def error_query(
         self,
         evo: EvolvableSubjects,
-        factor_implementation_queried_graph_knowledge: FactorImplementationQueriedGraphKnowledge,
+        queried_knowledge_v2: CoSTEERQueriedKnowledgeV2,
         v2_query_error_limit: int = 5,
         knowledge_sampler: float = 1.0,
-    ) -> QueriedKnowledge | None:
-        # queried_error_knowledge = FactorImplementationQueriedGraphErrorKnowledge()
-        for task_index, target_factor_task in enumerate(evo.target_factor_tasks):
-            target_factor_task_information = target_factor_task.get_factor_information()
-            factor_implementation_queried_graph_knowledge.error_with_success_task[target_factor_task_information] = {}
+    ) -> CoSTEERQueriedKnowledge | None:
+        for task_index, target_task in enumerate(evo.sub_tasks):
+            target_task_information = target_task.get_task_information()
+            queried_knowledge_v2.task_to_similar_error_successful_knowledge[target_task_information] = []
             if (
-                target_factor_task_information in self.knowledgebase.success_task_to_knowledge_dict
-                or target_factor_task_information in factor_implementation_queried_graph_knowledge.failed_task_info_set
+                target_task_information in self.knowledgebase.success_task_to_knowledge_dict
+                or target_task_information in queried_knowledge_v2.failed_task_info_set
             ):
-                factor_implementation_queried_graph_knowledge.error_with_success_task[
-                    target_factor_task_information
-                ] = []
+                queried_knowledge_v2.task_to_similar_error_successful_knowledge[target_task_information] = []
             else:
-                factor_implementation_queried_graph_knowledge.error_with_success_task[
-                    target_factor_task_information
-                ] = []
+                queried_knowledge_v2.task_to_similar_error_successful_knowledge[target_task_information] = []
                 if (
-                    target_factor_task_information in self.knowledgebase.working_trace_error_analysis
-                    and len(self.knowledgebase.working_trace_error_analysis[target_factor_task_information]) > 0
-                    and len(factor_implementation_queried_graph_knowledge.former_traces[target_factor_task_information])
-                    > 0
+                    target_task_information in self.knowledgebase.working_trace_error_analysis
+                    and len(self.knowledgebase.working_trace_error_analysis[target_task_information]) > 0
+                    and len(queried_knowledge_v2.task_to_former_failed_traces[target_task_information]) > 0
                 ):
-                    queried_last_trace = factor_implementation_queried_graph_knowledge.former_traces[
-                        target_factor_task_information
-                    ][-1]
-                    target_index = self.knowledgebase.working_trace_knowledge[target_factor_task_information].index(
+                    queried_last_trace = queried_knowledge_v2.task_to_former_failed_traces[target_task_information][0][
+                        -1
+                    ]
+                    target_index = self.knowledgebase.working_trace_knowledge[target_task_information].index(
                         queried_last_trace,
                     )
                     last_knowledge_error_analysis_result = self.knowledgebase.working_trace_error_analysis[
-                        target_factor_task_information
+                        target_task_information
                     ][target_index]
                 else:
                     last_knowledge_error_analysis_result = []
@@ -693,20 +697,20 @@ class FactorImplementationGraphRAGStrategy(RAGStrategy):
                     ]
 
                 same_error_success_knowledge_pair_list = same_error_success_knowledge_pair_list[:v2_query_error_limit]
-                factor_implementation_queried_graph_knowledge.error_with_success_task[
-                    target_factor_task_information
-                ] = same_error_success_knowledge_pair_list
+                queried_knowledge_v2.task_to_similar_error_successful_knowledge[target_task_information] = (
+                    same_error_success_knowledge_pair_list
+                )
 
-        return factor_implementation_queried_graph_knowledge
+        return queried_knowledge_v2
 
 
-class FactorImplementationGraphKnowledgeBase(KnowledgeBase):
-    def __init__(self, init_component_list=None) -> None:
+class CoSTEERKnowledgeBaseV2(EvolvingKnowledgeBase):
+    def __init__(self, init_component_list=None, path: str | Path = None) -> None:
         """
         Load knowledge, offer brief information of knowledge and common handle interfaces
         """
-        self.graph: UndirectedGraph = UndirectedGraph.load(Path.cwd() / "graph.pkl")
-        RDAgentLog().info(f"Knowledge Graph loaded, size={self.graph.size()}")
+        self.graph: UndirectedGraph = UndirectedGraph(Path.cwd() / "graph.pkl")
+        logger.info(f"CoSTEER Knowledge Graph loaded, size={self.graph.size()}")
 
         if init_component_list:
             for component in init_component_list:
@@ -723,7 +727,7 @@ class FactorImplementationGraphKnowledgeBase(KnowledgeBase):
         # Add already success task
         self.success_task_to_knowledge_dict = {}
 
-        # key:node_id(for task trace and success implement), value:knowledge instance(aka 'FactorImplementationKnowledge')
+        # key:node_id(for task trace and success implement), value:knowledge instance(aka 'CoSTEERKnowledge')
         self.node_to_implementation_knowledge_dict = {}
 
         # store the task description to component nodes

rdagent/components/coder/CoSTEER/prompts.yaml

@@ -0,0 +1,10 @@
+
+analyze_component_prompt_v1_system: |-
+  User is getting a new task that might consist of the components below (given in component_index: component_description):
+  {{all_component_content}}
+
+  You should find out what components does the new task have, and put their indices in a list.
+  Please response the critic in the json format. Here is an example structure for the JSON output, please strictly follow the format:
+  {
+      "component_no_list": the list containing indices of components.
+  }

rdagent/components/coder/CoSTEER/task.py

@@ -0,0 +1,9 @@
+from rdagent.core.experiment import Task
+
+
+class CoSTEERTask(Task):
+    def __init__(self, base_code: str = None, *args, **kwargs) -> None:
+        super().__init__(*args, **kwargs)
+        # TODO: we may upgrade the base_code into a workspace-like thing to know previous.
+        # NOTE: (xiao) think we don't need the base_code anymore. The information should be retrieved from the workspace.
+        self.base_code = base_code

rdagent/components/coder/data_science/conf.py

@@ -0,0 +1,71 @@
+from typing import Literal
+
+from rdagent.app.data_science.conf import DS_RD_SETTING
+from rdagent.components.coder.CoSTEER.config import CoSTEERSettings
+from rdagent.utils.env import (
+    CondaConf,
+    DockerEnv,
+    DSDockerConf,
+    Env,
+    LocalEnv,
+    MLEBDockerConf,
+    MLECondaConf,
+)
+
+
+class DSCoderCoSTEERSettings(CoSTEERSettings):
+    """Data Science CoSTEER settings"""
+
+    class Config:
+        env_prefix = "DS_Coder_CoSTEER_"
+
+    max_seconds: int = 2400
+    env_type: str = "docker"
+    # TODO: extract a function for env and conf.
+
+
+def get_ds_env(
+    conf_type: Literal["kaggle", "mlebench"] = "kaggle",
+    extra_volumes: dict = {},
+    running_timeout_period: int = (
+        DS_RD_SETTING.debug_timeout if DS_RD_SETTING.sample_data else DS_RD_SETTING.full_timeout
+    ),
+) -> Env:
+    """
+    Retrieve the appropriate environment configuration based on the env_type setting.
+
+    Returns:
+        Env: An instance of the environment configured either as DockerEnv or LocalEnv.
+
+    Raises:
+        ValueError: If the env_type is not recognized.
+    """
+    conf = DSCoderCoSTEERSettings()
+    assert conf_type in ["kaggle", "mlebench"], f"Unknown conf_type: {conf_type}"
+
+    if conf.env_type == "docker":
+        env_conf = DSDockerConf() if conf_type == "kaggle" else MLEBDockerConf()
+        env = DockerEnv(conf=env_conf)
+    elif conf.env_type == "conda":
+        env = LocalEnv(
+            conf=(
+                CondaConf(conda_env_name=conf_type) if conf_type == "kaggle" else MLECondaConf(conda_env_name=conf_type)
+            )
+        )
+    else:
+        raise ValueError(f"Unknown env type: {conf.env_type}")
+    env.conf.extra_volumes = extra_volumes
+    env.conf.running_timeout_period = running_timeout_period
+    return env
+
+
+def get_clear_ws_cmd(stage: Literal["before_training", "before_inference"] = "before_training") -> str:
+    """
+    Clean the files in workspace to a specific stage
+    """
+    assert stage in ["before_training", "before_inference"], f"Unknown stage: {stage}"
+    if DS_RD_SETTING.enable_model_dump and stage == "before_training":
+        cmd = "rm -r submission.csv scores.csv models"
+    else:
+        cmd = "rm submission.csv scores.csv"
+    return cmd

rdagent/components/coder/data_science/ensemble/__init__.py

@@ -0,0 +1,164 @@
+"""
+File structure
+- ___init__.py: the entrance/agent of coder
+- evaluator.py
+- conf.py
+- exp.py: everything under the experiment, e.g.
+    - Task
+    - Experiment
+    - Workspace
+- test.py
+    - Each coder could be tested.
+"""
+
+from pathlib import Path
+
+from jinja2 import Environment, StrictUndefined
+
+from rdagent.app.data_science.conf import DS_RD_SETTING
+from rdagent.components.coder.CoSTEER import CoSTEER
+from rdagent.components.coder.CoSTEER.evaluators import (
+    CoSTEERMultiEvaluator,
+    CoSTEERSingleFeedback,
+)
+from rdagent.components.coder.CoSTEER.evolving_strategy import (
+    MultiProcessEvolvingStrategy,
+)
+from rdagent.components.coder.CoSTEER.knowledge_management import (
+    CoSTEERQueriedKnowledge,
+)
+from rdagent.components.coder.data_science.conf import DSCoderCoSTEERSettings
+from rdagent.components.coder.data_science.ensemble.eval import EnsembleCoSTEEREvaluator
+from rdagent.components.coder.data_science.ensemble.exp import EnsembleTask
+from rdagent.core.exception import CoderError
+from rdagent.core.experiment import FBWorkspace
+from rdagent.core.scenario import Scenario
+from rdagent.oai.llm_utils import APIBackend
+from rdagent.utils.agent.ret import PythonAgentOut
+from rdagent.utils.agent.tpl import T
+
+DIRNAME = Path(__file__).absolute().resolve().parent
+
+
+class EnsembleMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
+    def implement_one_task(
+        self,
+        target_task: EnsembleTask,
+        queried_knowledge: CoSTEERQueriedKnowledge | None = None,
+        workspace: FBWorkspace | None = None,
+        prev_task_feedback: CoSTEERSingleFeedback | None = None,
+    ) -> dict[str, str]:
+        # Get task information for knowledge querying
+        ensemble_information_str = target_task.get_task_information()
+
+        # Query knowledge
+        queried_similar_successful_knowledge = (
+            queried_knowledge.task_to_similar_task_successful_knowledge[ensemble_information_str]
+            if queried_knowledge is not None
+            else []
+        )
+        queried_former_failed_knowledge = (
+            queried_knowledge.task_to_former_failed_traces[ensemble_information_str]
+            if queried_knowledge is not None
+            else []
+        )
+        queried_former_failed_knowledge = (
+            [
+                knowledge
+                for knowledge in queried_former_failed_knowledge[0]
+                if knowledge.implementation.file_dict.get("ensemble.py") != workspace.file_dict.get("ensemble.py")
+            ],
+            queried_former_failed_knowledge[1],
+        )
+
+        # Generate code with knowledge integration
+        competition_info = self.scen.get_scenario_all_desc(eda_output=workspace.file_dict.get("EDA.md", None))
+        system_prompt = T(".prompts:ensemble_coder.system").r(
+            task_desc=ensemble_information_str,
+            competition_info=competition_info,
+            queried_similar_successful_knowledge=queried_similar_successful_knowledge,
+            queried_former_failed_knowledge=(
+                queried_former_failed_knowledge[0] if queried_former_failed_knowledge else None
+            ),
+            all_code=workspace.all_codes,
+            out_spec=PythonAgentOut.get_spec(),
+        )
+
+        if DS_RD_SETTING.spec_enabled:
+            code_spec = workspace.file_dict["spec/ensemble.md"]
+        else:
+            test_code = (
+                Environment(undefined=StrictUndefined)
+                .from_string((DIRNAME / "eval_tests" / "ensemble_test.txt").read_text())
+                .render(
+                    model_names=[
+                        fn[:-3] for fn in workspace.file_dict.keys() if fn.startswith("model_") and "test" not in fn
+                    ],
+                    metric_name=self.scen.metric_name,
+                )
+            )
+            code_spec = T("scenarios.data_science.share:component_spec.general").r(
+                spec=T("scenarios.data_science.share:component_spec.Ensemble").r(), test_code=test_code
+            )
+        user_prompt = T(".prompts:ensemble_coder.user").r(
+            code_spec=code_spec,
+            latest_code=workspace.file_dict.get("ensemble.py"),
+            latest_code_feedback=prev_task_feedback,
+        )
+
+        for _ in range(5):
+            ensemble_code = PythonAgentOut.extract_output(
+                APIBackend().build_messages_and_create_chat_completion(
+                    user_prompt=user_prompt,
+                    system_prompt=system_prompt,
+                )
+            )
+            if ensemble_code != workspace.file_dict.get("ensemble.py"):
+                break
+            else:
+                user_prompt = user_prompt + "\nPlease avoid generating same code to former code!"
+        else:
+            raise CoderError("Failed to generate a new ensemble code.")
+
+        return {
+            "ensemble.py": ensemble_code,
+        }
+
+    def assign_code_list_to_evo(self, code_list: list[dict[str, str]], evo):
+        """
+        Assign the code list to the evolving item.
+
+        The code list is aligned with the evolving item's sub-tasks.
+        If a task is not implemented, put a None in the list.
+        """
+        for index in range(len(evo.sub_tasks)):
+            if code_list[index] is None:
+                continue
+            if evo.sub_workspace_list[index] is None:
+                # evo.sub_workspace_list[index] = FBWorkspace(target_task=evo.sub_tasks[index])
+                evo.sub_workspace_list[index] = evo.experiment_workspace
+            evo.sub_workspace_list[index].inject_files(**code_list[index])
+        return evo
+
+
+class EnsembleCoSTEER(CoSTEER):
+    def __init__(
+        self,
+        scen: Scenario,
+        *args,
+        **kwargs,
+    ) -> None:
+        settings = DSCoderCoSTEERSettings()
+        eva = CoSTEERMultiEvaluator(EnsembleCoSTEEREvaluator(scen=scen), scen=scen)
+        es = EnsembleMultiProcessEvolvingStrategy(scen=scen, settings=settings)
+
+        super().__init__(
+            *args,
+            settings=settings,
+            eva=eva,
+            es=es,
+            evolving_version=2,
+            scen=scen,
+            max_loop=DS_RD_SETTING.coder_max_loop,
+            **kwargs,
+        )