chore(main): release 0.3.0 (#258)

Release-As: 0.3.0

.bumpversion.cfg

@@ -0,0 +1,6 @@
+[bumpversion]
+current_version = 0.0.0
+commit = True
+tag = True
+
+[bumpversion:file:pyproject.toml]

.commitlintrc.js

@@ -0,0 +1,21 @@
+module.exports = {
+    extends: ["@commitlint/config-conventional"],
+    rules: {
+        // Configuration Format: [level, applicability, value]
+        // level: Error level, usually expressed as a number:
+        //     0 - disable rule
+        //     1 - Warning (does not prevent commits)
+        //     2 - Error (will block the commit)
+        // applicability: the conditions under which the rule applies, commonly used values:
+        //     “always” - always apply the rule
+        //     “never” - never apply the rule
+        // value: the specific value of the rule, e.g. a maximum length of 100.
+        // Refs: https://commitlint.js.org/reference/rules-configuration.html
+      "header-max-length": [2, "always", 100],
+      "type-enum": [
+        2,
+        "always",
+        ["build", "chore", "ci", "docs", "feat", "fix", "perf", "refactor", "revert", "style", "test", "Release-As"]
+      ]
+    }
+  };

.env.example

@@ -1,27 +1,30 @@
+"""
+This file is a template for the .env file.
+
+Please copy this file to .env and fill in the values.
+
+For more information about configuration options, please refer to the documentation
+
+"""
+
 # Global configs:
-USE_AZURE=True
+USE_AZURE=False
+USE_AZURE_TOKEN_PROVIDER=False
 MAX_RETRY=10
 RETRY_WAIT_SECONDS=20
-DUMP_CHAT_CACHE=True
-USE_CHAT_CACHE=True
-DUMP_EMBEDDING_CACHE=True
-USE_EMBEDDING_CACHE=True
-LOG_LLM_CHAT_CONTENT=False
-CHAT_FREQUENCY_PENALTY=0.0
-CHAT_PRESENCE_PENALTY=0.0
 
-# embedding model configs:
-EMBEDDING_OPENAI_API_KEY=your_api_key
-EMBEDDING_AZURE_API_BASE=your_api_base
-EMBEDDING_AZURE_API_VERSION=your_api_version
-EMBEDDING_MODEL=text-embedding-3-small
-
-
-# chat model configs:
-CHAT_OPENAI_API_KEY=your_api_key # 5c
-CHAT_AZURE_API_BASE=your_api_base
-CHAT_AZURE_API_VERSION=your_api_version
-CHAT_MODEL=your_model_version
+# LLM API Setting:
+OPENAI_API_KEY=<your_api_key>
+CHAT_MODEL=gpt-4-turbo
 CHAT_MAX_TOKENS=3000
 CHAT_TEMPERATURE=0.7
-CHAT_STREAM=True
+# CHAT_AZURE_API_BASE=<for_Azure_user>
+# CHAT_AZURE_API_VERSION=<for_Azure_user>
+
+EMBEDDING_MODEL=text-embedding-3-small
+# EMBEDDING_AZURE_API_BASE=<for_Azure_user>
+# EMBEDDING_AZURE_API_VERSION=<for_Azure_user>
+
+# Cache Setting (Optional):
+
+# Senario Configs:

.github/ISSUE_TEMPLATE/bug-report.md

@@ -0,0 +1,51 @@
+---
+name: "\U0001F41B Bug Report"
+about: Submit a bug report to help us improve RD-Agent
+labels: bug
+
+---
+
+## 🐛 Bug Description
+
+<!-- A clear and concise description of what the bug is. -->
+
+## To Reproduce
+
+Steps to reproduce the behavior:
+
+1.
+2.
+3.
+
+
+## Expected Behavior
+
+<!-- A clear and concise description of what you expected to happen. -->
+
+## Screenshot
+
+<!-- A screenshot of the error message or anything shouldn't appear-->
+
+## Environment
+
+**Note**: Users can run `rdagent collect_info` to get system information and paste it directly here.
+
+ - Name of current operating system:
+ - Processor architecture:
+ - System, version, and hardware information:
+ - Version number of the system:
+ - Python version:
+ - Container ID:
+ - Container Name:
+ - Container Status:
+ - Image ID used by the container:
+ - Image tag used by the container:
+ - Container port mapping:
+ - Container Label:
+ - Startup Commands:
+ - RD-Agent version:
+ - Package version:
+
+## Additional Notes
+
+<!-- Add any other information about the problem here. -->

.github/ISSUE_TEMPLATE/documentation.md

@@ -0,0 +1,9 @@
+---
+name: "\U0001F4D6 Documentation"
+about: Report an issue related to documentation
+
+---
+
+## 📖 Documentation
+
+<!-- Please specify whether it's tutorial part or API reference part, and describe it.-->

.github/ISSUE_TEMPLATE/feature-request.md

@@ -0,0 +1,25 @@
+---
+name: "\U0001F31FFeature Request"
+about: Request for a new RD-Agent feature
+labels: enhancement
+
+---
+
+## 🌟 Feature Description
+<!-- A clear and concise description of the feature proposal -->
+
+## Motivation
+
+1. Application scenario
+2. Related works (Papers, Github repos etc.):
+3. Any other relevant and important information:
+
+<!-- Please describe why the feature is important. -->
+
+## Alternatives
+
+<!-- A short description of any alternative solutions or features you've considered. -->
+
+## Additional Notes
+
+<!-- Add any other context or screenshots about the feature request here. -->

.github/ISSUE_TEMPLATE/question.md

@@ -0,0 +1,10 @@
+---
+name: "❓Questions & Help"
+about: Have some questions? We can offer help.
+labels: question
+
+---
+
+## ❓ Questions and Help
+
+We sincerely suggest you to carefully read the [documentation](http://rdagent.readthedocs.io/). After that, if you still feel puzzled, please describe the question clearly under this issue.

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,36 @@
+<!--- Thank you for submitting a Pull Request! In order to make our work smoother. -->
+<!--- please make sure your Pull Request meets the following requirements: -->
+<!---   1. Provide a general summary of your changes in the Title above; -->
+<!---   2. Add appropriate prefixes to titles, such as `build:`, `chore:`, `ci:`, `docs:`, `feat:`, `fix:`, `perf:`, `refactor:`, `revert:`, `style:`, `test:`(Ref: https://www.conventionalcommits.org/). -->
+<!--- Category: -->
+<!--- Patch Updates: `fix:` -->
+<!---   Example: fix(auth): correct login validation issue -->
+<!--- minor update (introduces new functionality): `feat` -->
+<!---   Example: feature(parser): add ability to parse arrays -->
+<!--- major update(destructive update): Include BREAKING CHANGE in the commit message footer, or add `! ` in the commit footer to indicate that there is a destructive update. -->
+<!---   Example: feat(auth)! : remove support for old authentication method -->
+<!--- Other updates: `build:`, `chore:`, `ci:`, `docs:`, `perf:`, `refactor:`, `revert:`, `style:`, `test:`. -->
+
+## Description
+<!--- Describe your changes in detail -->
+
+## Motivation and Context
+<!--- Are there any related issues? If so, please put the link here. -->
+<!--- Why is this change required? What problem does it solve? -->
+
+## How Has This Been Tested?
+<!---  Put an `x` in all the boxes that apply: --->
+- [ ] Pass the test by running: `pytest qlib/tests/test_all_pipeline.py` under upper directory of `qlib`.
+- [ ] If you are adding a new feature, test on your own test scripts.
+
+<!--- **ATTENTION**: If you are adding a new feature, please make sure your codes are **correctly tested**. If our test scripts do not cover your cases, please provide your own test scripts under the `tests` folder and test them. More information about test scripts can be found [here](https://docs.python.org/3/library/unittest.html#basic-example), or you could refer to those we provide under the `tests` folder. -->
+
+## Screenshots of Test Results (if appropriate):
+1. Pipeline test:
+2. Your own tests:
+
+## Types of changes
+<!--- What types of changes does your code introduce? Put an `x` in all the boxes that apply: -->
+- [ ] Fix bugs
+- [ ] Add new feature
+- [ ] Update documentation

.github/workflows/ci.yml

@@ -19,22 +19,11 @@ jobs:
           python-version: ${{ matrix.python-version }}
       - run: env | sort
       - run: make dev
-      - env:
-          CHAT_AZURE_API_BASE: ${{ secrets.CHAT_AZURE_API_BASE }}
-          CHAT_AZURE_API_VERSION: ${{ secrets.CHAT_AZURE_API_VERSION }}
-          CHAT_MAX_TOKENS: ${{ secrets.CHAT_MAX_TOKENS }}
-          CHAT_MODEL: ${{ secrets.CHAT_MODEL }}
-          CHAT_TEMPERATURE: ${{ secrets.CHAT_TEMPERATURE }}
-          EMBEDDING_AZURE_API_BASE: ${{ secrets.CHAT_AZURE_API_BASE }}
-          EMBEDDING_AZURE_API_VERSION: ${{ secrets.CHAT_AZURE_API_VERSION }}
-          EMBEDDING_MODEL: ${{ secrets.EMBEDDING_MODEL }}
-        name: lint test docs and build
-        run: make lint test docs build
+      - name: lint test docs and build
+        run: make lint docs-gen test-offline # test docs build
     strategy:
       matrix:
         python-version:
-          - '3.8'
-          - '3.9'
           - '3.10'
           - '3.11'
   dependabot:
@@ -56,16 +45,12 @@ jobs:
         with:
           cache: pip
           python-version: |
-            3.8
-            3.9
             3.10
             3.11
       - name: Install pipenv using pipx
         run: pipx install pipenv
       - name: Generate constraints for all supported Python versions
         run: |
-          CI= PYTHON_VERSION=3.8 make constraints
-          CI= PYTHON_VERSION=3.9 make constraints
           CI= PYTHON_VERSION=3.10 make constraints
           CI= PYTHON_VERSION=3.11 make constraints
       - name: Push changes if applicable

.github/workflows/pr.yml

@@ -0,0 +1,35 @@
+name: Lint pull request title
+
+on:
+  pull_request:
+    types:
+      - opened
+      - synchronize
+      - reopened
+      - edited
+
+concurrency:
+  cancel-in-progress: true
+  group: ${{ github.workflow }}-${{ github.ref }}
+
+jobs:
+  lint-title:
+    runs-on: ubuntu-latest
+    steps:
+      # This step is necessary because the lint title uses the .commitlintrc.js file in the project root directory.
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v3
+        with:
+          node-version: '16'
+
+      - name: Install commitlint
+        run: npm install --save-dev @commitlint/{config-conventional,cli}
+
+      - name: Validate PR Title with commitlint
+        env:
+          BODY: ${{ github.event.pull_request.title }}
+        run: |
+          echo "$BODY" | npx commitlint --config .commitlintrc.js

.github/workflows/release.yml

@@ -1,91 +1,50 @@
-jobs:
-  package:
-    needs: release
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          cache: pip
-          python-version: '3.8'
-      - run: env | sort
-      - run: make dev-package
-      - run: make build
-      - env:
-          TWINE_NON_INTERACTIVE: true
-          TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }}
-          TWINE_USERNAME: ${{ vars.TWINE_USERNAME != '' && vars.TWINE_USERNAME || '__token__' }}
-        run: make upload
-  pages-build:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          cache: pip
-          python-version: '3.8'
-      - run: env | sort
-      - run: make dev-docs
-      - run: make docs
-      - name: Upload changelog
-        uses: actions/upload-artifact@v4
-        with:
-          name: changelog
-          path: docs/changelog.md
-      - name: Upload pages artifact
-        uses: actions/upload-pages-artifact@v3
-        with:
-          path: public
-  pages-deploy:
-    needs: release
-    permissions:
-      id-token: write
-      pages: write
-    runs-on: ubuntu-latest
-    steps:
-      - id: deployment
-        name: Deploy to GitHub Pages
-        uses: actions/deploy-pages@v4
-  release:
-    needs: pages-build
-    permissions:
-      contents: write
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - name: Install git-changelog using pipx
-        run: pipx install git-changelog
-      - name: Remove changelog to avoid file already exists error
-        run: rm -v docs/changelog.md
-      - name: Download changelog
-        uses: actions/download-artifact@v4
-        with:
-          name: changelog
-          path: docs/
-      - name: Prepare release notes
-        run: make release-notes > release-notes.md
-      - id: prerelease
-        name: Determine prerelease
-        run: |
-          if [[ "${{ github.ref }}" =~ (a|b|rc)(0|[1-9][0-9]*)?$ ]]; then
-            echo "is_prerelease=true" > $GITHUB_OUTPUT
-          else
-            echo "is_prerelease=false" > $GITHUB_OUTPUT
-          fi
-      - name: Create GitHub release
-        uses: softprops/action-gh-release@v2
-        with:
-          body_path: release-notes.md
-          prerelease: ${{ steps.prerelease.outputs.is_prerelease }}
 name: Release
 on:
   push:
-    tags:
-      - v?[0-9]+.[0-9]+.[0-9]+
-      - v?[0-9]+.[0-9]+.[0-9]+-?a[0-9]*
-      - v?[0-9]+.[0-9]+.[0-9]+-?b[0-9]*
-      - v?[0-9]+.[0-9]+.[0-9]+-?rc[0-9]*
+    branches:
+      - main
+permissions:
+  contents: read
+jobs:
+  release_and_publish:
+    permissions:
+      contents: write
+      pull-requests: read
+    runs-on: ubuntu-latest
+    steps:
+      - name: Release please
+        id: release_please
+        uses: googleapis/release-please-action@v4
+        with:
+          # The current PAT (personal access token) was created on 2024-08-05,
+          # since the maximum validity of PAT is 1 year, you need to change the PAT before 2025-08-05.
+          token: ${{ secrets.PAT }}
+          release-type: simple
+      - uses: actions/checkout@v4
+        if: ${{ steps.release_please.outputs.release_created }}
+        with:
+          fetch-depth: 0
+      - name: Set up Python
+        if: ${{ steps.release_please.outputs.release_created }}
+        uses: actions/setup-python@v5
+        with:
+          cache: pip
+          python-version: '3.10'
+      - name: Install dependencies
+        if: ${{ steps.release_please.outputs.release_created }}
+        run: |
+          python -m pip install --upgrade pip
+          pip install setuptools wheel twine  #  better-exceptions(optional for debug)
+      - run: env | sort
+        if: ${{ steps.release_please.outputs.release_created }}
+      - run: make dev
+        if: ${{ steps.release_please.outputs.release_created }}
+      - run: make build
+        if: ${{ steps.release_please.outputs.release_created }}
+      - name: upload
+        if: ${{ steps.release_please.outputs.release_created }}
+        env:
+          TWINE_USERNAME: __token__
+          TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
+        run: |
+          make upload

.gitignore

@@ -64,6 +64,7 @@ coverage.xml
 
 # Django stuff:
 *.log
+/log/
 local_settings.py
 db.sqlite3
 db.sqlite3-journal
@@ -111,7 +112,7 @@ celerybeat.pid
 # Environments
 .env
 .venv
-env/
+^env/
 venv/
 ENV/
 env.bak/
@@ -138,6 +139,9 @@ dmypy.json
 # all pkl files
 *.pkl
 
+# all h5 files
+*.h5
+
 # all vs-code files
 .vscode/
 
@@ -153,3 +157,16 @@ git_ignore_folder/
 
 # DB files
 *.db
+
+# Docker
+factor_template/mlruns/
+env_tpl
+mlruns/
+
+# possible output from coder or runner
+*.pth
+*qlib_res.csv
+
+# shell script
+*.out
+*.sh

.readthedocs.yaml

@@ -0,0 +1,26 @@
+# .readthedocs.yml
+# Read the Docs configuration file
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+# Required
+version: 2
+
+# Set the version of Python and other tools you might need
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.10"
+
+# Build documentation in the docs/ directory with Sphinx
+sphinx:
+  configuration: docs/conf.py
+
+# Build all formats
+formats: all
+
+# Optionally set the version of Python and requirements required to build your docs
+python:
+  install:
+    - requirements: requirements/docs.txt
+    - method: pip
+      path: .

.streamlit/config.toml

@@ -0,0 +1,2 @@
+[client]
+showSidebarNavigation = false

CHANGELOG.md

@@ -0,0 +1,185 @@
+# Changelog
+
+## [0.3.0](https://github.com/microsoft/RD-Agent/compare/v0.2.1...v0.3.0) (2024-10-21)
+
+
+### Features
+
+* add a new template for kaggle ([#289](https://github.com/microsoft/RD-Agent/issues/289)) ([eee3ab5](https://github.com/microsoft/RD-Agent/commit/eee3ab5b25198224826cb7a8a17eab28bd5d1f7d))
+* add download submission.csv button for kaggle scenario ([#317](https://github.com/microsoft/RD-Agent/issues/317)) ([dcdcbe4](https://github.com/microsoft/RD-Agent/commit/dcdcbe46b4858bfb133ae3cca056e7f602d5cf63))
+* add kaggle command ([#271](https://github.com/microsoft/RD-Agent/issues/271)) ([0938394](https://github.com/microsoft/RD-Agent/commit/0938394b7084ffbf3294d8c23d2d34bf7322ca0b))
+* add kaggle tpl: feedback-prize ([#331](https://github.com/microsoft/RD-Agent/issues/331)) ([a288e39](https://github.com/microsoft/RD-Agent/commit/a288e399e6b0beec62729bd7d46b98a55de5ab79))
+* add more templates for kaggle ([#291](https://github.com/microsoft/RD-Agent/issues/291)) ([da752ec](https://github.com/microsoft/RD-Agent/commit/da752ec806e6f5f5679bc27ac1c072ed9a319251))
+* add normal rag into framework ([#360](https://github.com/microsoft/RD-Agent/issues/360)) ([91b0b1f](https://github.com/microsoft/RD-Agent/commit/91b0b1f66c3c1bf757cb64c4cfbdcaafe59eab74))
+* add qlib_factor_strategy ([#307](https://github.com/microsoft/RD-Agent/issues/307)) ([f8f59ff](https://github.com/microsoft/RD-Agent/commit/f8f59ff0a1be4428a68c8c27f220aabad0b6c9f0))
+* Add ranking in kaggle scenario ([#401](https://github.com/microsoft/RD-Agent/issues/401)) ([b16b4be](https://github.com/microsoft/RD-Agent/commit/b16b4beb402e0c27dfb39ee9d2a120f1b56d447c))
+* Add runtime measurement for each step and loop in RDLoop. ([#281](https://github.com/microsoft/RD-Agent/issues/281)) ([83058c8](https://github.com/microsoft/RD-Agent/commit/83058c864ceeec413dd29bf501030d5a7bd34679))
+* add s3e11 kaggle template ([#324](https://github.com/microsoft/RD-Agent/issues/324)) ([8c57524](https://github.com/microsoft/RD-Agent/commit/8c57524bead1c8f655a08763d608eb7a6dd5975e))
+* Added RepoAnalyzer to empower auto-summary of a workspace ([#264](https://github.com/microsoft/RD-Agent/issues/264)) ([0bd349a](https://github.com/microsoft/RD-Agent/commit/0bd349af50b9b881ba1774bdeb4d723529ef2aa9))
+* Added support for loading and storing RAG in Kaggle scenarios. ([#269](https://github.com/microsoft/RD-Agent/issues/269)) ([c4895de](https://github.com/microsoft/RD-Agent/commit/c4895de83f1ed000e563d42b3468a6bd9e5a4965))
+* announce Discord and WeChat ([#367](https://github.com/microsoft/RD-Agent/issues/367)) ([acac507](https://github.com/microsoft/RD-Agent/commit/acac5078a103b71afa6bd6c053b0766a6a7e609d))
+* auto submit result after one kaggle RDLoop ([#345](https://github.com/microsoft/RD-Agent/issues/345)) ([ab55d70](https://github.com/microsoft/RD-Agent/commit/ab55d7052b53a928b84dc5d5d0d2999d90ca9056))
+* better feedback & evaluation ([#346](https://github.com/microsoft/RD-Agent/issues/346)) ([cc9a8c1](https://github.com/microsoft/RD-Agent/commit/cc9a8c1eab3ca89f8c1e5de4a2bb4e7fcc0cc615))
+* Dynamic scenario based on task ([#392](https://github.com/microsoft/RD-Agent/issues/392)) ([665a037](https://github.com/microsoft/RD-Agent/commit/665a037e4fd7326c450e3fa0d0605eea26fd9ef3))
+* Factor Implement Search Enhancement ([#294](https://github.com/microsoft/RD-Agent/issues/294)) ([4ecf25f](https://github.com/microsoft/RD-Agent/commit/4ecf25f0acf2389a172b14d3dab20895daf2ab89))
+* Feature selection v3 to support all actions  ([#280](https://github.com/microsoft/RD-Agent/issues/280)) ([0047641](https://github.com/microsoft/RD-Agent/commit/00476413fbf00e36e71ab3ccb48d4e766b6ccf4d))
+* fix some bugs and add original features' description ([#259](https://github.com/microsoft/RD-Agent/issues/259)) ([1a5f45a](https://github.com/microsoft/RD-Agent/commit/1a5f45a40d821c017bdba14af8c93710707c5ea5))
+* get kaggle notebooks & disscussion text for RAG ([#371](https://github.com/microsoft/RD-Agent/issues/371)) ([cead345](https://github.com/microsoft/RD-Agent/commit/cead3450a14bf4b142ac988c27fa098c7656a95c))
+* Iceberge competition ([#372](https://github.com/microsoft/RD-Agent/issues/372)) ([c10ea4f](https://github.com/microsoft/RD-Agent/commit/c10ea4f5d4cc56a75b47cf23c7084ee189ba1a25))
+* implement isolated model feature selection loop ([#370](https://github.com/microsoft/RD-Agent/issues/370)) ([cf1292d](https://github.com/microsoft/RD-Agent/commit/cf1292de1a0153ca14ea64971e73a1c93f7d89e3))
+* Initial version if Graph RAG in KAGGLE scenario ([#301](https://github.com/microsoft/RD-Agent/issues/301)) ([fd3c0fd](https://github.com/microsoft/RD-Agent/commit/fd3c0fd26eff7d3be72fa4f2a234e33b9f796627))
+* Integrate RAG into the Kaggle scenarios. ([#262](https://github.com/microsoft/RD-Agent/issues/262)) ([be0e48a](https://github.com/microsoft/RD-Agent/commit/be0e48a7dfbee2b5d2947d09115db5db2e5266f1))
+* Kaggle loop update (Feature & Model) ([#241](https://github.com/microsoft/RD-Agent/issues/241)) ([4cf22a6](https://github.com/microsoft/RD-Agent/commit/4cf22a65c964123b4267569ee02c0c7094c54ca4))
+* kaggle templates related ([#287](https://github.com/microsoft/RD-Agent/issues/287)) ([785fdc1](https://github.com/microsoft/RD-Agent/commit/785fdc144d16fa8454b7c9d2e53e78fe7f22a29a))
+* Model context for tuning and selection ([#284](https://github.com/microsoft/RD-Agent/issues/284)) ([f2831e7](https://github.com/microsoft/RD-Agent/commit/f2831e7442510668b0ca75953b3359894803ef3c))
+* Modify FactorRowCountEvaluator and FactorIndexEvaluator to return the ratio ([#328](https://github.com/microsoft/RD-Agent/issues/328)) ([8f43f8e](https://github.com/microsoft/RD-Agent/commit/8f43f8e87a92e05b541e925910608606ec8f6c4b))
+* New competition - Optiver ([#356](https://github.com/microsoft/RD-Agent/issues/356)) ([3705efe](https://github.com/microsoft/RD-Agent/commit/3705efe3b923748655a57d76b7a236e54d361831))
+* random forest for s3e11 ([#347](https://github.com/microsoft/RD-Agent/issues/347)) ([b57846d](https://github.com/microsoft/RD-Agent/commit/b57846d29314e9a5967945d1b4895f0f48c0f5ce))
+* refine the code in model description and fix some bugs in feedback.py ([#288](https://github.com/microsoft/RD-Agent/issues/288)) ([5b124d7](https://github.com/microsoft/RD-Agent/commit/5b124d7372137e4c613eb2749ddcc773922cc7b6))
+* refine the template in several Kaggle competitions ([#343](https://github.com/microsoft/RD-Agent/issues/343)) ([034f238](https://github.com/microsoft/RD-Agent/commit/034f238ed5ec351486b21250eabc75114961936c))
+* Revise to support better hypothesis proposal  ([#390](https://github.com/microsoft/RD-Agent/issues/390)) ([c55ec0a](https://github.com/microsoft/RD-Agent/commit/c55ec0a0f577bbf7fc6228f7b87d2089ded83b31))
+* show workspace in demo ([#348](https://github.com/microsoft/RD-Agent/issues/348)) ([ddf567c](https://github.com/microsoft/RD-Agent/commit/ddf567c551b553788be022e9312c209ef6137d64))
+* support Multi output ([#330](https://github.com/microsoft/RD-Agent/issues/330)) ([3d36c45](https://github.com/microsoft/RD-Agent/commit/3d36c452ff0983800e5343834cc69f24a508ea70))
+* Supporting COVID-19 competition ([#374](https://github.com/microsoft/RD-Agent/issues/374)) ([a1b63db](https://github.com/microsoft/RD-Agent/commit/a1b63db79600edc9a74ba713c9d0be290214a592))
+* supporting Mnist competition ([#375](https://github.com/microsoft/RD-Agent/issues/375)) ([e958a34](https://github.com/microsoft/RD-Agent/commit/e958a34f5632a46ac43bff8e0d07d6ed020fdfc2))
+* Supporting Model Specifications ([#319](https://github.com/microsoft/RD-Agent/issues/319)) ([e126471](https://github.com/microsoft/RD-Agent/commit/e1264719e10b76158a91cd0ef331848e7c2de7c7))
+* supporting various Kaggle competitions & scenarios for RD-Agent ([#409](https://github.com/microsoft/RD-Agent/issues/409)) ([75eea22](https://github.com/microsoft/RD-Agent/commit/75eea22cc3d4e6f5a94c88cce915e27c507f8c50))
+* template for kaggle ([#308](https://github.com/microsoft/RD-Agent/issues/308)) ([ff97cf0](https://github.com/microsoft/RD-Agent/commit/ff97cf0155ab6941e4b5cf7d103575f934b70dc9))
+* use auto gen seed when using LLM cache ([#441](https://github.com/microsoft/RD-Agent/issues/441)) ([ca15365](https://github.com/microsoft/RD-Agent/commit/ca15365d23eeb094f42cf3dc8f5269b2f1c42bd3))
+* use unified pickle cacher & move llm config into a isolated config ([#424](https://github.com/microsoft/RD-Agent/issues/424)) ([2879ecf](https://github.com/microsoft/RD-Agent/commit/2879ecff816d97688b60909a79c7e568d42608a1))
+* xgboost gpu accelerate ([#359](https://github.com/microsoft/RD-Agent/issues/359)) ([56a5b8f](https://github.com/microsoft/RD-Agent/commit/56a5b8f9b2c6726cc64ec5b04b4ce7935d59b572))
+
+
+### Bug Fixes
+
+* a bug of developer& edit s4e8 template ([#338](https://github.com/microsoft/RD-Agent/issues/338)) ([f12ce72](https://github.com/microsoft/RD-Agent/commit/f12ce726e7de96d478a232a3c27f92439820f8b4))
+* actively raised errors aer also considered as negative feedback. ([#268](https://github.com/microsoft/RD-Agent/issues/268)) ([46ec908](https://github.com/microsoft/RD-Agent/commit/46ec908e3594ac5e4cdc4057268e2f8800f5ed1f))
+* bug of saving preprocess cache files ([#310](https://github.com/microsoft/RD-Agent/issues/310)) ([5fb0608](https://github.com/microsoft/RD-Agent/commit/5fb0608f39f113cc9807fb1f381284a0bd4da318))
+* cache ([#383](https://github.com/microsoft/RD-Agent/issues/383)) ([f2a6e75](https://github.com/microsoft/RD-Agent/commit/f2a6e75b36ca96f7733b9c2a7154ac67bd2d7c6f))
+* change css tag of kaggle competition info crawler ([#306](https://github.com/microsoft/RD-Agent/issues/306)) ([1e3d38b](https://github.com/microsoft/RD-Agent/commit/1e3d38bf1ca3654f3a90ff392ecba1dbb4e80224))
+* debug dsagent ([#387](https://github.com/microsoft/RD-Agent/issues/387)) ([8fe9511](https://github.com/microsoft/RD-Agent/commit/8fe9511e606ba148c66f384add6ab94857079541))
+* eval_method cannot catch run factor error ([#260](https://github.com/microsoft/RD-Agent/issues/260)) ([2aaab31](https://github.com/microsoft/RD-Agent/commit/2aaab317ccb7a0121063bcd85fc36c21c7b8a391))
+* fix a bug in competition metric evaluation ([#407](https://github.com/microsoft/RD-Agent/issues/407)) ([94c47d6](https://github.com/microsoft/RD-Agent/commit/94c47d6fd5c3e38fc786a83e6d0d05e8d04498f3))
+* fix a bug in mini case ([#389](https://github.com/microsoft/RD-Agent/issues/389)) ([e75bb57](https://github.com/microsoft/RD-Agent/commit/e75bb5746f63933b750406bbd34ee63c5ba76b9f))
+* fix a bug in model tuning feedback ([#316](https://github.com/microsoft/RD-Agent/issues/316)) ([8aa088d](https://github.com/microsoft/RD-Agent/commit/8aa088da2dc7525a3970c01d01987246f47d6238))
+* fix a bug in scenario.py ([#388](https://github.com/microsoft/RD-Agent/issues/388)) ([999a1eb](https://github.com/microsoft/RD-Agent/commit/999a1eb0eff9088e1b02419db741db4acf8d9ff7))
+* fix a bug in the format of the model input ([#327](https://github.com/microsoft/RD-Agent/issues/327)) ([8f0574e](https://github.com/microsoft/RD-Agent/commit/8f0574eaaadb245b8c38e09ad4821306996d926f))
+* fix a small bug in cache using module name and function name as unique folder name ([#429](https://github.com/microsoft/RD-Agent/issues/429)) ([4f8134a](https://github.com/microsoft/RD-Agent/commit/4f8134a697d952f7ac824d7ebeec64bbc4545ab3))
+* fix a typo ([#362](https://github.com/microsoft/RD-Agent/issues/362)) ([9fafabd](https://github.com/microsoft/RD-Agent/commit/9fafabdf321b818bdd2211a2324d50cd0ebe1c1f))
+* fix cache result logic ([#430](https://github.com/microsoft/RD-Agent/issues/430)) ([5e34263](https://github.com/microsoft/RD-Agent/commit/5e342637dcc862679fd0642c6ba9ef048c984845))
+* fix command injection ([#421](https://github.com/microsoft/RD-Agent/issues/421)) ([52f30a6](https://github.com/microsoft/RD-Agent/commit/52f30a6184af1295be15e855a80b84bc424fc75d))
+* fix json load error ([#386](https://github.com/microsoft/RD-Agent/issues/386)) ([bba55fb](https://github.com/microsoft/RD-Agent/commit/bba55fb48fe105f4847c1b9c476eedc80835f523))
+* fix some bugs in feedback.py and refine the prompt ([#292](https://github.com/microsoft/RD-Agent/issues/292)) ([d834052](https://github.com/microsoft/RD-Agent/commit/d8340527f133dcc649d599d90d6402eddd37859e))
+* fix some bugs in knowledge base ([#378](https://github.com/microsoft/RD-Agent/issues/378)) ([fa6ff8e](https://github.com/microsoft/RD-Agent/commit/fa6ff8e591cf1847df77d73116649c5623161573))
+* fix some bugs in rag ([#399](https://github.com/microsoft/RD-Agent/issues/399)) ([194215c](https://github.com/microsoft/RD-Agent/commit/194215c4559aee5b6ece18d65c95fb30968e2db6))
+* fix some bugs in the entire loop ([#274](https://github.com/microsoft/RD-Agent/issues/274)) ([8a564ec](https://github.com/microsoft/RD-Agent/commit/8a564ece1d87b27ee98b76db317935e802468965))
+* fix some errors in scenario.py, proposal.py and runner.py and several complex competition scenarios([#365](https://github.com/microsoft/RD-Agent/issues/365)) ([2e383b1](https://github.com/microsoft/RD-Agent/commit/2e383b175d8448a67cb470f4e3ae8977d8ec6b5b))
+* improve_execution_time_in_kaggle_loop ([#279](https://github.com/microsoft/RD-Agent/issues/279)) ([4c8f998](https://github.com/microsoft/RD-Agent/commit/4c8f998c76f1e983a5687d2c65d3251750f2a9a0))
+* kaggle data mount problem ([#297](https://github.com/microsoft/RD-Agent/issues/297)) ([795df31](https://github.com/microsoft/RD-Agent/commit/795df311e3f93cd2f3fb51ba5698adaf10f6bd62))
+* Optiver fixes ([#357](https://github.com/microsoft/RD-Agent/issues/357)) ([b054017](https://github.com/microsoft/RD-Agent/commit/b054017463af0d1784407030f2477d212118f341))
+* partial bug in bench ([#368](https://github.com/microsoft/RD-Agent/issues/368)) ([af9808f](https://github.com/microsoft/RD-Agent/commit/af9808f98736a2df07e121c2f6d7bfeb7b7d3581))
+* preprocess output format & some mistake in spelling ([#358](https://github.com/microsoft/RD-Agent/issues/358)) ([b8b2cd6](https://github.com/microsoft/RD-Agent/commit/b8b2cd6ccd3b27aa73de847e50899a8a53b71b8f))
+* rag save file ([#385](https://github.com/microsoft/RD-Agent/issues/385)) ([1cb01dd](https://github.com/microsoft/RD-Agent/commit/1cb01dd6fe595f2f5fb86487601326611dd1a57a))
+* raise error in demo when no Metric in a Loop ([#313](https://github.com/microsoft/RD-Agent/issues/313)) ([e46a78e](https://github.com/microsoft/RD-Agent/commit/e46a78eb69271cb19978aab2f3b976c2870ca082))
+* refactor Bench ([#302](https://github.com/microsoft/RD-Agent/issues/302)) ([78a87f6](https://github.com/microsoft/RD-Agent/commit/78a87f624780ff67c0fa995ae4692678a120f99c))
+* refine some codes ([#353](https://github.com/microsoft/RD-Agent/issues/353)) ([866c2e6](https://github.com/microsoft/RD-Agent/commit/866c2e63ffa3876a3d16ad37f96da41d0558b714))
+* refine the prompt ([#286](https://github.com/microsoft/RD-Agent/issues/286)) ([77966c4](https://github.com/microsoft/RD-Agent/commit/77966c4f5e9f492c437c5b4b78d89c0f875ef0d8))
+* refine the ucb algorithm ([#406](https://github.com/microsoft/RD-Agent/issues/406)) ([14f7d97](https://github.com/microsoft/RD-Agent/commit/14f7d976e03c92d6e727524e0cdad8a03b585016))
+* revert model and make SOTA model available to COSTEER ([#351](https://github.com/microsoft/RD-Agent/issues/351)) ([3b7437b](https://github.com/microsoft/RD-Agent/commit/3b7437b87e685188259779cd85a78a0b592de9de))
+* stop using markup in docker env print ([#336](https://github.com/microsoft/RD-Agent/issues/336)) ([3009889](https://github.com/microsoft/RD-Agent/commit/3009889b5e2605b5427c76f3084e0e58026bb5ae))
+* support seed and fix absolute path ([#278](https://github.com/microsoft/RD-Agent/issues/278)) ([26352e1](https://github.com/microsoft/RD-Agent/commit/26352e13121cad5be95c0de78bb9f5dda4330614))
+* template for kaggle foreset & s4e9 ([#334](https://github.com/microsoft/RD-Agent/issues/334)) ([2393a41](https://github.com/microsoft/RD-Agent/commit/2393a41e7237615ced2c3fdd5c49308236b9f276))
+* test kaggle method ([#296](https://github.com/microsoft/RD-Agent/issues/296)) ([91a6196](https://github.com/microsoft/RD-Agent/commit/91a619618be1d7db660ea2b413a78dfaba9417a1))
+* update code to fix a small bug in model cache md5 hash ([#303](https://github.com/microsoft/RD-Agent/issues/303)) ([b00e4dc](https://github.com/microsoft/RD-Agent/commit/b00e4dc2eff5b16029a2a12a6589eadac5cfd148))
+* update new feature engineering code format ([#272](https://github.com/microsoft/RD-Agent/issues/272)) ([7850b80](https://github.com/microsoft/RD-Agent/commit/7850b8006a7c89d22629b345b4f361b0f35bc60d))
+* Update prompts.yaml to constrain only one model type   ([#341](https://github.com/microsoft/RD-Agent/issues/341)) ([5b5dfee](https://github.com/microsoft/RD-Agent/commit/5b5dfeefbc7eb9dcbd9923544005c5d281262c03))
+* Update runner.py to fix a small bug ([#282](https://github.com/microsoft/RD-Agent/issues/282)) ([8aef3ab](https://github.com/microsoft/RD-Agent/commit/8aef3abcecd6002bd4bfeedcbe2c786d8bbfe2be))
+* Use fixed file name in model costeer & fixing cache ([#311](https://github.com/microsoft/RD-Agent/issues/311)) ([1f910a5](https://github.com/microsoft/RD-Agent/commit/1f910a5248bc576895ed66c2f7b2c3e046a2bc28))
+
+
+### Performance Improvements
+
+* some small upgrade to factor costeer to improve the performance ([#420](https://github.com/microsoft/RD-Agent/issues/420)) ([9eb931f](https://github.com/microsoft/RD-Agent/commit/9eb931ffd971f252380dbd33ad1db259a4f229fd))
+
+
+### Reverts
+
+* Revert feat: Factor Implement Search Enhancement ([#294](https://github.com/microsoft/RD-Agent/issues/294)) ([#305](https://github.com/microsoft/RD-Agent/issues/305)) ([f663cf4](https://github.com/microsoft/RD-Agent/commit/f663cf42a2f75cd52aef1c6b18be7c27f0641fed))
+
+## [0.2.1](https://github.com/microsoft/RD-Agent/compare/v0.2.0...v0.2.1) (2024-09-10)
+
+
+### Bug Fixes
+
+* default model value in config ([#256](https://github.com/microsoft/RD-Agent/issues/256)) ([c097585](https://github.com/microsoft/RD-Agent/commit/c097585f631f401c2c0966f6ad4c17286924f011))
+* fix_dotenv_error ([#257](https://github.com/microsoft/RD-Agent/issues/257)) ([923063c](https://github.com/microsoft/RD-Agent/commit/923063c1fd957c4ed42e97272c72b5e9545451dc))
+* readme ([#248](https://github.com/microsoft/RD-Agent/issues/248)) ([8cede22](https://github.com/microsoft/RD-Agent/commit/8cede2209922876490148459e1134da828e1fda0))
+
+## [0.2.0](https://github.com/microsoft/RD-Agent/compare/v0.1.0...v0.2.0) (2024-09-07)
+
+
+### Features
+
+* add collect info ([#233](https://github.com/microsoft/RD-Agent/issues/233)) ([89f4af9](https://github.com/microsoft/RD-Agent/commit/89f4af90fb4d95a0689bf9efc8ffd9326469c0aa))
+* add cross validation for kaggle scenario ([#236](https://github.com/microsoft/RD-Agent/issues/236)) ([e0b03ba](https://github.com/microsoft/RD-Agent/commit/e0b03ba6b5c3d9aa552b99d470e106d4e348e64d))
+* add progress status for docker env ([#215](https://github.com/microsoft/RD-Agent/issues/215)) ([538d4ef](https://github.com/microsoft/RD-Agent/commit/538d4ef2e52de795b90d3f75b2e1e877ab85c18d))
+* Added loop code for Kaggle scene. ([#211](https://github.com/microsoft/RD-Agent/issues/211)) ([975c327](https://github.com/microsoft/RD-Agent/commit/975c32715e51aec6b49537401f5fc59115e04a01))
+* Demo display effect and usage ([#162](https://github.com/microsoft/RD-Agent/issues/162)) ([8cf122a](https://github.com/microsoft/RD-Agent/commit/8cf122a0155f434fa4477ae7a6d616b5caecd3e0))
+* piloting of the framework ([#227](https://github.com/microsoft/RD-Agent/issues/227)) ([e9b103e](https://github.com/microsoft/RD-Agent/commit/e9b103e684fdd2b98cd1a89971a3fce2d6e884a1))
+* support more models for kaggle scenario ([#223](https://github.com/microsoft/RD-Agent/issues/223)) ([e3a9659](https://github.com/microsoft/RD-Agent/commit/e3a96598c0720fe092ec86d7ca8c195c7d6bcc72))
+* update model_experiment.py to support basic EDA ([#220](https://github.com/microsoft/RD-Agent/issues/220)) ([bf2684c](https://github.com/microsoft/RD-Agent/commit/bf2684c4d55ab8e1048ac0291695475ad53b0cd6))
+
+
+### Bug Fixes
+
+* fix some bugs in llm calling ([#217](https://github.com/microsoft/RD-Agent/issues/217)) ([7b010f8](https://github.com/microsoft/RD-Agent/commit/7b010f8b5940aba65a58f1d78192aa80bcd0e654))
+* package dependency. ([#234](https://github.com/microsoft/RD-Agent/issues/234)) ([46be295](https://github.com/microsoft/RD-Agent/commit/46be2952952af534fd8d98a656c704c688d7cbdd))
+* remove useless line ([#177](https://github.com/microsoft/RD-Agent/issues/177)) ([64e9a8e](https://github.com/microsoft/RD-Agent/commit/64e9a8e39a2072a962111db18f5b9565df5b0176))
+
+## [0.1.0](https://github.com/microsoft/RD-Agent/compare/v0.0.1...v0.1.0) (2024-08-09)
+
+
+### Features
+
+* add entry for rdagent. ([#187](https://github.com/microsoft/RD-Agent/issues/187)) ([121b6d9](https://github.com/microsoft/RD-Agent/commit/121b6d98de38cd03be30cbee47b40baf39a2b60b))
+* change ui entry ([#197](https://github.com/microsoft/RD-Agent/issues/197)) ([fa5d335](https://github.com/microsoft/RD-Agent/commit/fa5d3354d22240888f4fc4007d9834f7424632aa))
+* remove pdfs and enable online pdf readings ([#183](https://github.com/microsoft/RD-Agent/issues/183)) ([18c0501](https://github.com/microsoft/RD-Agent/commit/18c05016a23d694c7b12759cf1322562dcffc56a))
+
+
+### Bug Fixes
+
+* Fix a fail href in readme ([#189](https://github.com/microsoft/RD-Agent/issues/189)) ([1b89218](https://github.com/microsoft/RD-Agent/commit/1b89218f6bc697494f4a1b8a42ad18963002714f))
+* fix quick start problem ([#191](https://github.com/microsoft/RD-Agent/issues/191)) ([44f61bf](https://github.com/microsoft/RD-Agent/commit/44f61bfa1058a8efb59ca48b7f1417765aeea33e))
+* update command line in readme.md ([#192](https://github.com/microsoft/RD-Agent/issues/192)) ([9c45d24](https://github.com/microsoft/RD-Agent/commit/9c45d24a192da02f7d9765cb001097da1bc36c61))
+
+## 0.0.1 (2024-08-08)
+
+
+### Features
+
+* Add description for scenario experiments. ([#174](https://github.com/microsoft/RD-Agent/issues/174)) ([fbd8c6d](https://github.com/microsoft/RD-Agent/commit/fbd8c6d87e1424c08997103b8e8fbf264858c4ed))
+* Added QlibFactorFromReportScenario and improved the report-factor loop. ([#161](https://github.com/microsoft/RD-Agent/issues/161)) ([882c79b](https://github.com/microsoft/RD-Agent/commit/882c79bf11583980e646b130f71cfa20201ffc7b))
+* filter feature which is high correlation to former implemented features ([#145](https://github.com/microsoft/RD-Agent/issues/145)) ([e818326](https://github.com/microsoft/RD-Agent/commit/e818326422740e04a4863f7c3c18744dde2ad98f))
+* Remove redundant 'key steps' section in frontend scene display. ([#169](https://github.com/microsoft/RD-Agent/issues/169)) ([e767005](https://github.com/microsoft/RD-Agent/commit/e76700513bee29232c93b97414419df330d9be8d))
+* streamlit webapp demo for different scenarios ([#135](https://github.com/microsoft/RD-Agent/issues/135)) ([d8da7db](https://github.com/microsoft/RD-Agent/commit/d8da7db865e6653fc4740efee9a843b69bd79699))
+* Uploaded Documentation, Updated Prompts & Some Code for model demo ([#144](https://github.com/microsoft/RD-Agent/issues/144)) ([529f935](https://github.com/microsoft/RD-Agent/commit/529f935aa98623f0dc1dda29eecee3ef738dd446))
+
+
+### Bug Fixes
+
+* Add framework handling for task coding failure. ([#176](https://github.com/microsoft/RD-Agent/issues/176)) ([5e14fa5](https://github.com/microsoft/RD-Agent/commit/5e14fa54a9dd30a94aebe2643b8c9a3b85517a11))
+* Comprehensive update to factor extraction. ([#143](https://github.com/microsoft/RD-Agent/issues/143)) ([b5ea040](https://github.com/microsoft/RD-Agent/commit/b5ea04019fd5fa15c0f8b9a7e4f18f490f7057d4))
+* first round app folder cleaning ([#166](https://github.com/microsoft/RD-Agent/issues/166)) ([6a5a750](https://github.com/microsoft/RD-Agent/commit/6a5a75021912927deb5e8e4c7ad3ec4b51bfc788))
+* fix pickle problem ([#140](https://github.com/microsoft/RD-Agent/issues/140)) ([7ee4258](https://github.com/microsoft/RD-Agent/commit/7ee42587b60d94417f34332cee395cf210dc8a0e))
+* fix release CI ([#165](https://github.com/microsoft/RD-Agent/issues/165)) ([85d6a5e](https://github.com/microsoft/RD-Agent/commit/85d6a5ed91113fda34ae079b23c89aa24acd2cb2))
+* fix release CI error ([#160](https://github.com/microsoft/RD-Agent/issues/160)) ([1c9f8ef](https://github.com/microsoft/RD-Agent/commit/1c9f8ef287961731944acc9008496b4dddeddca7))
+* fix several bugs in data mining scenario ([#147](https://github.com/microsoft/RD-Agent/issues/147)) ([b233380](https://github.com/microsoft/RD-Agent/commit/b233380e2c66fb030db39424f0f040c86e37f5c4))
+* fix some small bugs in report-factor loop ([#152](https://github.com/microsoft/RD-Agent/issues/152)) ([a79f9f9](https://github.com/microsoft/RD-Agent/commit/a79f9f93406aff6305a76e6a6abd3852642e4c62))
+* fix_release_ci_error ([#150](https://github.com/microsoft/RD-Agent/issues/150)) ([4f82e99](https://github.com/microsoft/RD-Agent/commit/4f82e9960a2638af9d831581185ddd3bac5711fc))
+* Fixed some bugs introduced during refactoring. ([#167](https://github.com/microsoft/RD-Agent/issues/167)) ([f8f1445](https://github.com/microsoft/RD-Agent/commit/f8f1445283fb89aefeb2918243c35a219a51a56c))
+* optimize some prompts in factor loop. ([#158](https://github.com/microsoft/RD-Agent/issues/158)) ([c2c1330](https://github.com/microsoft/RD-Agent/commit/c2c13300b9ad315a663ec2d0eada414e56c6f54f))
+
+
+### Miscellaneous Chores
+
+* release 0.0.1 ([1feacd3](https://github.com/microsoft/RD-Agent/commit/1feacd39b21193de11e9bbecf880ddf96d7c261c))

Makefile

@@ -81,31 +81,56 @@ constraints: deepclean
 
 # Check lint with black.
 black:
-	$(PIPRUN) python -m black --check . --extend-exclude test/scripts --extend-exclude git_ignore_folder -l 120
+	$(PIPRUN) python -m black --check --diff . --extend-exclude test/scripts --extend-exclude git_ignore_folder -l 120
 
 # Check lint with isort.
 isort:
 	$(PIPRUN) python -m isort --check . -s git_ignore_folder -s test/scripts
 
 # Check lint with mypy.
+# First deal with the core folder, and then gradually increase the scope of detection,
+# and eventually realize the detection of the complete project.
 mypy:
-	$(PIPRUN) python -m mypy . --exclude rdagent/scripts --exclude git_ignore_folder
+	$(PIPRUN) python -m mypy rdagent/core   #  --exclude rdagent/scripts,git_ignore_folder
 
 # Check lint with ruff.
+# First deal with the core folder, and then gradually increase the scope of detection,
+# and eventually realize the detection of the complete project.
 ruff:
-	$(PIPRUN) ruff check .  --exclude rdagent/scripts,git_ignore_folder
+	$(PIPRUN) ruff check rdagent/core --ignore FBT001,FBT002,I001   # --exclude rdagent/scripts,git_ignore_folder
 
 # Check lint with toml-sort.
 toml-sort:
 	$(PIPRUN) toml-sort --check pyproject.toml
 
 # Check lint with all linters.
-lint: mypy ruff toml-sort
+# Prioritize fixing isort, then black, otherwise you'll get weird and unfixable black errors.
+# lint: mypy ruff
+lint: mypy ruff isort black toml-sort
 
 # Run pre-commit with autofix against all files.
 pre-commit:
 	pre-commit run --all-files
 
+########################################################################################
+# Auto Lint
+########################################################################################
+
+# Auto lint with black.
+auto-black:
+	$(PIPRUN) python -m black . --extend-exclude test/scripts --extend-exclude git_ignore_folder -l 120
+
+# Auto lint with isort.
+auto-isort:
+	$(PIPRUN) python -m isort . -s git_ignore_folder -s test/scripts
+
+# Auto lint with toml-sort.
+auto-toml-sort:
+	$(PIPRUN) toml-sort pyproject.toml
+
+# Auto lint with all linters.
+auto-lint: auto-isort auto-black auto-toml-sort
+
 ########################################################################################
 # Test
 ########################################################################################
@@ -116,10 +141,21 @@ test-run:
 	$(PIPRUN) python -m coverage run --concurrency=multiprocessing -m pytest --ignore test/scripts
 	$(PIPRUN) python -m coverage combine
 
+test-run-offline:
+	# some test that does not require api calling
+	$(PIPRUN) python -m coverage erase
+	$(PIPRUN) python -m coverage run --concurrency=multiprocessing -m pytest -m "offline" --ignore test/scripts
+	$(PIPRUN) python -m coverage combine
+
 # Generate coverage report for terminal and xml.
+# TODO: we may have higher coverage rate if we have more test
 test: test-run
-	$(PIPRUN) python -m coverage report --fail-under 80
-	$(PIPRUN) python -m coverage xml --fail-under 80
+	$(PIPRUN) python -m coverage report --fail-under 20  # 80
+	$(PIPRUN) python -m coverage xml --fail-under 20  # 80
+
+test-offline: test-run-offline
+	$(PIPRUN) python -m coverage report --fail-under 20  # 80
+	$(PIPRUN) python -m coverage xml --fail-under 20  # 80
 
 ########################################################################################
 # Package
@@ -144,13 +180,19 @@ docs-autobuild:
 		--watch rdagent
 
 # Generate changelog from git commits.
+# The -c and -s arguments should match
+# If -c uses Basic (default, inherits from base class), -s optional argument: # If -c uses conventional (inherits from base class), -s optional parameter: add,fix,change,remove,merge,doc
+# If -c uses conventional (inherits from base class), -s is optional: build,chore,ci,deps,doc,docs,feat,fix,perf,ref,refactor,revert,style,test,tests
+# If -c uses angular (inherits from conventional), -s optional argument: build,chore,ci,deps,doc,docs,feat,fix,perf,ref,refactor,revert,style,test,tests
 # NOTE(xuan.hu): Need to be run before document generation to take effect.
+# $(PIPRUN) git-changelog -ETrio $(CHANGELOG_PATH) -c conventional -s build,chore,ci,docs,feat,fix,perf,refactor,revert,style,test
 changelog:
 	@if wget -q --spider $(CHANGELOG_URL); then \
 		echo "Existing Changelog found at '$(CHANGELOG_URL)', download for incremental generation."; \
 		wget -q -O $(CHANGELOG_PATH) $(CHANGELOG_URL); \
 	fi
-	$(PIPRUN) git-changelog -ETrio $(CHANGELOG_PATH) -c conventional -s build,chore,ci,docs,feat,fix,perf,refactor,revert,style,test
+	$(PIPRUN) LATEST_TAG=$$(git tag --sort=-creatordate | head -n 1); \
+	git-changelog --bump $$LATEST_TAG -Tio docs/changelog.md -c conventional -s build,chore,ci,deps,doc,docs,feat,fix,perf,ref,refactor,revert,style,test,tests
 
 # Generate release notes from changelog.
 release-notes:
@@ -158,7 +200,7 @@ release-notes:
 
 # Build documentation only from rdagent.
 docs-gen:
-	$(PIPRUN) python -m sphinx.cmd.build docs $(PUBLIC_DIR)
+	$(PIPRUN) python -m sphinx.cmd.build -W docs $(PUBLIC_DIR)
 
 # Generate mypy reports.
 docs-mypy: docs-gen

README.md

@@ -1,64 +1,262 @@
-# Project
+<h4 align="center">
+  <img src="docs/_static/logo.png" alt="RA-Agent logo" style="width:70%; ">
+  
+  <a href="https://rdagent.azurewebsites.net" target="_blank">🖥️ Live Demo</a> | <a href="https://rdagent.azurewebsites.net/factor_loop" target="_blank">🎥 Demo Video</a> <a href="https://www.youtube.com/watch?v=JJ4JYO3HscM&list=PLALmKB0_N3_i52fhUmPQiL4jsO354uopR" target="_blank">▶️YouTube</a>   | <a href="https://rdagent.readthedocs.io/en/latest/index.html" target="_blank">📖 Documentation</a> | <a href="#-paperwork-list"> 📃 Papers </a>
+</h3>
 
-> This repo has been populated by an initial template to help get you started. Please
-> make sure to update the content to build a great experience for community-building.
 
-As the maintainer of this project, please make a few updates:
+[![CI](https://github.com/microsoft/RD-Agent/actions/workflows/ci.yml/badge.svg)](https://github.com/microsoft/RD-Agent/actions/workflows/ci.yml)
+[![CodeQL](https://github.com/microsoft/RD-Agent/actions/workflows/github-code-scanning/codeql/badge.svg)](https://github.com/microsoft/RD-Agent/actions/workflows/github-code-scanning/codeql)
+[![Dependabot Updates](https://github.com/microsoft/RD-Agent/actions/workflows/dependabot/dependabot-updates/badge.svg)](https://github.com/microsoft/RD-Agent/actions/workflows/dependabot/dependabot-updates)
+[![Lint PR Title](https://github.com/microsoft/RD-Agent/actions/workflows/pr.yml/badge.svg)](https://github.com/microsoft/RD-Agent/actions/workflows/pr.yml)
+[![Release.yml](https://github.com/microsoft/RD-Agent/actions/workflows/release.yml/badge.svg)](https://github.com/microsoft/RD-Agent/actions/workflows/release.yml)
+[![Platform](https://img.shields.io/badge/platform-Linux-blue)](https://pypi.org/project/rdagent/#files)
+[![PyPI](https://img.shields.io/pypi/v/rdagent)](https://pypi.org/project/rdagent/)
+[![PyPI - Python Version](https://img.shields.io/pypi/pyversions/rdagent)](https://pypi.org/project/rdagent/)
+[![Release](https://img.shields.io/github/v/release/microsoft/RD-Agent)](https://github.com/microsoft/RD-Agent/releases)
+[![GitHub](https://img.shields.io/github/license/microsoft/RD-Agent)](https://github.com/microsoft/RD-Agent/blob/main/LICENSE)
+[![pre-commit](https://img.shields.io/badge/pre--commit-enabled-brightgreen?logo=pre-commit)](https://github.com/pre-commit/pre-commit)
+[![Checked with mypy](https://www.mypy-lang.org/static/mypy_badge.svg)](http://mypy-lang.org/)
+[![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)
+[![Chat](https://img.shields.io/badge/chat-discord-blue)](https://discord.gg/ybQ97B6Jjy)
+[![Readthedocs Preview](https://github.com/microsoft/RD-Agent/actions/workflows/readthedocs-preview.yml/badge.svg)](https://github.com/microsoft/RD-Agent/actions/workflows/readthedocs-preview.yml) <!-- this badge is too long, please place it in the last one to make it pretty --> 
 
-- Improving this README.MD file to provide a great experience
-- Updating SUPPORT.MD with content about this project's support experience
-- Understanding the security reporting process in SECURITY.MD
-- Remove this section from the README
+# 📰 News
+| 🗞️ News        | 📝 Description                 |
+| --            | ------                        |
+| Official WeChat group release  | We created a WeChat group, welcome to join! (🗪[QR Code](docs/WeChat_QR_code.jpg)) |
+| Official Discord release  | We launch our first chatting channel in Discord (🗪[![Chat](https://img.shields.io/badge/chat-discord-blue)](https://discord.gg/ybQ97B6Jjy)) |
+| First release | **RDAgent** is released on GitHub |
 
-## Configuration:
 
-You can manually source the `.env` file in your shell before running the Python script:
-Most of the workflow are controlled by the environment variables.
-```sh
-# Export each variable in the .env file; Please note that it is different from `source .env` without export
-export $(grep -v '^#' .env | xargs)
-# Run the Python script
-python your_script.py
+# 🌟 Introduction
+<div align="center">
+      <img src="docs/_static/scen.png" alt="Our focused scenario" style="width:80%; ">
+</div>
+
+RDAgent aims to automate the most critical and valuable aspects of the industrial R&D process, and we begin with focusing on the data-driven scenarios to streamline the development of models and data. 
+Methodologically, we have identified a framework with two key components: 'R' for proposing new ideas and 'D' for implementing them.
+We believe that the automatic evolution of R&D will lead to solutions of significant industrial value.
+
+
+<!-- Tag Cloud -->
+R&D is a very general scenario. The advent of RDAgent can be your
+- 💰 **Automatic Quant Factory** ([🎥Demo Video](https://rdagent.azurewebsites.net/factor_loop)|[▶️YouTube](https://www.youtube.com/watch?v=X4DK2QZKaKY&t=6s))
+- 🤖 **Data Mining Agent:** Iteratively proposing data & models ([🎥Demo Video 1](https://rdagent.azurewebsites.net/model_loop)|[▶️YouTube](https://www.youtube.com/watch?v=dm0dWL49Bc0&t=104s)) ([🎥Demo Video 2](https://rdagent.azurewebsites.net/dmm)|[▶️YouTube](https://www.youtube.com/watch?v=VIaSTZuoZg4))  and implementing them by gaining knowledge from data.
+- 🦾 **Research Copilot:** Auto read research papers ([🎥Demo Video](https://rdagent.azurewebsites.net/report_model)|[▶️YouTube](https://www.youtube.com/watch?v=BiA2SfdKQ7o)) / financial reports ([🎥Demo Video](https://rdagent.azurewebsites.net/report_factor)|[▶️YouTube](https://www.youtube.com/watch?v=ECLTXVcSx-c)) and implement model structures or building datasets.
+- ...
+
+You can click the links above to view the demo. We're continuously adding more methods and scenarios to the project to enhance your R&D processes and boost productivity. 
+
+Additionally, you can take a closer look at the examples in our **[🖥️ Live Demo](https://rdagent.azurewebsites.net/)**.
+
+<div align="center">
+    <a href="https://rdagent.azurewebsites.net/" target="_blank">
+        <img src="docs/_static/demo.png" alt="Watch the demo" width="80%">
+    </a>
+</div>
+
+
+# ⚡ Quick start
+
+You can try above demos by running the following command:
+
+### 🐳 Docker installation.
+Users must ensure Docker is installed before attempting most scenarios. Please refer to the [official 🐳Docker page](https://docs.docker.com/engine/install/) for installation instructions.
+
+### 🐍 Create a Conda Environment
+- Create a new conda environment with Python (3.10 and 3.11 are well-tested in our CI):
+  ```sh
+  conda create -n rdagent python=3.10
+  ```
+- Activate the environment:
+  ```sh
+  conda activate rdagent
+  ```
+
+### 🛠️ Install the RDAgent
+- You can directly install the RDAgent package from PyPI:
+  ```sh
+  pip install rdagent
+  ```
+
+### ⚙️ Configuration
+- You have to config your GPT model in the `.env`
+  ```bash
+  cat << EOF  > .env
+  OPENAI_API_KEY=<your_api_key>
+  # EMBEDDING_MODEL=text-embedding-3-small
+  CHAT_MODEL=gpt-4-turbo
+  EOF
+  ```
+
+### 🚀 Run the Application
+
+The **[🖥️ Live Demo](https://rdagent.azurewebsites.net/)** is implemented by the following commands(each item represents one demo, you can select the one you prefer):
+
+- Run the **Automated Quantitative Trading & Iterative Factors Evolution**:  [Qlib](http://github.com/microsoft/qlib) self-loop factor proposal and implementation application
+  ```sh
+  rdagent fin_factor
+  ```
+
+- Run the **Automated Quantitative Trading & Iterative Model Evolution**: [Qlib](http://github.com/microsoft/qlib) self-loop model proposal and implementation application
+  ```sh
+  rdagent fin_model
+  ```
+
+- Run the **Automated Medical Prediction Model Evolution**: Medical self-loop model proposal and implementation application
+  >(1) Apply for an account at [PhysioNet](https://physionet.org/). <br /> (2) Request access to FIDDLE preprocessed data: [FIDDLE Dataset](https://physionet.org/content/mimic-eicu-fiddle-feature/1.0.0/). <br />
+  (3) Place your username and password in `.env`.
+  ```bash
+  cat << EOF  >> .env
+  DM_USERNAME=<your_username>
+  DM_PASSWORD=<your_password>
+  EOF
+  ```
+  ```sh
+  rdagent med_model
+  ```
+
+- Run the **Automated Quantitative Trading & Factors Extraction from Financial Reports**:  Run the [Qlib](http://github.com/microsoft/qlib) factor extraction and implementation application based on financial reports
+  ```sh
+  # 1. Generally, you can run this scenario using the following command:
+  rdagent fin_factor_report --report_folder=<Your financial reports folder path>
+
+  # 2. Specifically, you need to prepare some financial reports first. You can follow this concrete example:
+  wget https://github.com/SunsetWolf/rdagent_resource/releases/download/reports/all_reports.zip
+  unzip all_reports.zip -d git_ignore_folder/reports
+  rdagent fin_factor_report --report_folder=git_ignore_folder/reports
+  ```
+
+- Run the **Automated Model Research & Development Copilot**: model extraction and implementation application
+  ```sh
+  # 1. Generally, you can run your own papers/reports with the following command:
+  rdagent general_model <Your paper URL>
+
+  # 2. Specifically, you can do it like this. For more details and additional paper examples, use `rdagent general_model -h`:
+  rdagent general_model  "https://arxiv.org/pdf/2210.09789"
+  ```
+
+### 🖥️ Monitor the Application Results
+- You can serve our demo app to monitor the RD loop by running the following command:
+  ```sh
+  rdagent ui --port 80 --log_dir <your log folder like "log/">
+  ```
+
+# 🏭 Scenarios
+
+We have applied RD-Agent to multiple valuable data-driven industrial scenarios.
+
+
+## 🎯 Goal: Agent for Data-driven R&D
+
+In this project, we are aiming to build an Agent to automate Data-Driven R\&D that can
++ 📄 Read real-world material (reports, papers, etc.) and **extract** key formulas, descriptions of interested **features** and **models**, which are the key components of data-driven R&D .
++ 🛠️ **Implement** the extracted formulas (e.g., features, factors, and models) in runnable codes.
+   + Due to the limited ability of LLM in implementing at once, build an evolving process for the agent to improve performance by learning from feedback and knowledge.
++ 💡 Propose **new ideas** based on current knowledge and observations.
+
+<!-- ![Data-Centric R&D Overview](docs/_static/overview.png) -->
+
+## 📈 Scenarios/Demos
+
+In the two key areas of data-driven scenarios, model implementation and data building, our system aims to serve two main roles: 🦾Copilot and 🤖Agent. 
+- The 🦾Copilot follows human instructions to automate repetitive tasks. 
+- The 🤖Agent, being more autonomous, actively proposes ideas for better results in the future.
+
+The supported scenarios are listed below:
+
+| Scenario/Target | Model Implementation                   | Data Building                                                                      |
+| --              | --                                     | --                                                                                 |
+| **💹 Finance**      | 🤖 [Iteratively Proposing Ideas & Evolving](https://rdagent.azurewebsites.net/model_loop)[▶️YouTube](https://www.youtube.com/watch?v=dm0dWL49Bc0&t=104s) |  🤖 [Iteratively Proposing Ideas & Evolving](https://rdagent.azurewebsites.net/factor_loop) [▶️YouTube](https://www.youtube.com/watch?v=X4DK2QZKaKY&t=6s) <br/>   🦾 [Auto reports reading & implementation](https://rdagent.azurewebsites.net/report_factor)[▶️YouTube](https://www.youtube.com/watch?v=ECLTXVcSx-c)  |
+| **🩺 Medical**      | 🤖 [Iteratively Proposing Ideas & Evolving](https://rdagent.azurewebsites.net/dmm)[▶️YouTube](https://www.youtube.com/watch?v=VIaSTZuoZg4) | -                                                                                  |
+| **🏭 General**      | 🦾 [Auto paper reading & implementation](https://rdagent.azurewebsites.net/report_model)[▶️YouTube](https://www.youtube.com/watch?v=BiA2SfdKQ7o)    | -                                                                                  |
+
+Different scenarios vary in entrance and configuration. Please check the detailed setup tutorial in the scenarios documents.
+
+Here is a gallery of [successful explorations](https://github.com/SunsetWolf/rdagent_resource/releases/download/demo_traces/demo_traces.zip) (5 traces showed in **[🖥️ Live Demo](https://rdagent.azurewebsites.net/)**). You can download and view the execution trace using the command below:
+
+```bash
+rdagent ui --port 80 --log_dir ./demo_traces
 ```
 
-## Naming convention
+Please refer to **[📖readthedocs_scen](https://rdagent.readthedocs.io/en/latest/scens/catalog.html)** for more details of the scenarios.
 
-### File naming convention
+# ⚙️ Framework
 
-| Name      | Description       |
-| --        | --                |
-| `conf.py` | The configuration for the module & app & project  | 
+<div align="center">
+    <img src="docs/_static/Framework-RDAgent.png" alt="Framework-RDAgent" width="85%">
+</div>
 
-<!-- TODO: renaming files -->
- 
 
-## Contributing
+Automating the R&D process in data science is a highly valuable yet underexplored area in industry. We propose a framework to push the boundaries of this important research field.
 
-### Guidance
+The research questions within this framework can be divided into three main categories:
+| Research Area | Paper/Work List |
+|--------------------|-----------------|
+| **Benchmark the R&D abilities** | [Benchmark](#benchmark) |
+| **Idea proposal:** Explore new ideas or refine existing ones | [Research](#research) |
+| **Ability to realize ideas:** Implement and execute ideas | [Development](#development) |
+
+We believe that the key to delivering high-quality solutions lies in the ability to evolve R&D capabilities. Agents should learn like human experts, continuously improving their R&D skills.
+
+More documents can be found in the **[📖 readthedocs](https://rdagent.readthedocs.io/)**.
+
+# 📃 Paper/Work list
+
+## 📊 Benchmark
+- [Towards Data-Centric Automatic R&D](https://arxiv.org/abs/2404.11276)
+```BibTeX
+@misc{chen2024datacentric,
+    title={Towards Data-Centric Automatic R&D},
+    author={Haotian Chen and Xinjie Shen and Zeqi Ye and Wenjun Feng and Haoxue Wang and Xiao Yang and Xu Yang and Weiqing Liu and Jiang Bian},
+    year={2024},
+    eprint={2404.11276},
+    archivePrefix={arXiv},
+    primaryClass={cs.AI}
+}
+```
+![image](https://github.com/user-attachments/assets/494f55d3-de9e-4e73-ba3d-a787e8f9e841)
+
+## 🔍 Research
+
+In a data mining expert's daily research and development process, they propose a hypothesis (e.g., a model structure like RNN can capture patterns in time-series data), design experiments (e.g., finance data contains time-series and we can verify the hypothesis in this scenario), implement the experiment as code (e.g., Pytorch model structure), and then execute the code to get feedback (e.g., metrics, loss curve, etc.). The experts learn from the feedback and improve in the next iteration.
+
+Based on the principles above, we have established a basic method framework that continuously proposes hypotheses, verifies them, and gets feedback from the real-world practice. This is the first scientific research automation framework that supports linking with real-world verification.
+
+For more detail, please refer to our **[🖥️ Live Demo page](https://rdagent.azurewebsites.net)**.
+
+## 🛠️ Development
+
+- [Collaborative Evolving Strategy for Automatic Data-Centric Development](https://arxiv.org/abs/2407.18690)
+```BibTeX
+@misc{yang2024collaborative,
+    title={Collaborative Evolving Strategy for Automatic Data-Centric Development},
+    author={Xu Yang and Haotian Chen and Wenjun Feng and Haoxue Wang and Zeqi Ye and Xinjie Shen and Xiao Yang and Shizhao Sun and Weiqing Liu and Jiang Bian},
+    year={2024},
+    eprint={2407.18690},
+    archivePrefix={arXiv},
+    primaryClass={cs.AI}
+}
+```
+![image](https://github.com/user-attachments/assets/75d9769b-0edd-4caf-9d45-57d1e577054b)
+
+
+# 🤝 Contributing
+
+## 📝 Guidelines
 This project welcomes contributions and suggestions.
-You can find issues in the issues list or simply running `grep -r "TODO:"`.
+Contributing to this project is straightforward and rewarding. Whether it's solving an issue, addressing a bug, enhancing documentation, or even correcting a typo, every contribution is valuable and helps improve RDAgent.
 
-Making contributions is not a hard thing. Solving an issue(maybe just answering a question raised in issues list ), fixing/issuing a bug, improving the documents and even fixing a typo are important contributions to RDAgent.
+To get started, you can explore the issues list, or search for `TODO:` comments in the codebase by running the command `grep -r "TODO:"`.
 
+<img src="https://img.shields.io/github/contributors-anon/microsoft/RD-Agent"/>
 
-### Policy
+<a href="https://github.com/microsoft/RD-Agent/graphs/contributors">
+  <img src="https://contrib.rocks/image?repo=microsoft/RD-Agent&max=100&columns=15" />
+</a>
 
-This project welcomes contributions and suggestions.  Most contributions require you to agree to a
-Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us
-the rights to use your contribution. For details, visit https://cla.opensource.microsoft.com.
+Before we released RD-Agent as an open-source project on GitHub, it was an internal project within our group. Unfortunately, the internal commit history was not preserved when we removed some confidential code. As a result, some contributions from our group members, including Haotian Chen, Wenjun Feng, Haoxue Wang, Zeqi Ye, Xinjie Shen, and Jinhui Li, were not included in the public commits.
 
-When you submit a pull request, a CLA bot will automatically determine whether you need to provide
-a CLA and decorate the PR appropriately (e.g., status check, comment). Simply follow the instructions
-provided by the bot. You will only need to do this once across all repos using our CLA.
-
-This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/).
-For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or
-contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments.
-
-## Trademarks
-
-This project may contain trademarks or logos for projects, products, or services. Authorized use of Microsoft 
-trademarks or logos is subject to and must follow 
-[Microsoft's Trademark & Brand Guidelines](https://www.microsoft.com/en-us/legal/intellectualproperty/trademarks/usage/general).
-Use of Microsoft trademarks or logos in modified versions of this project must not cause confusion or imply Microsoft sponsorship.
-Any use of third-party trademarks or logos are subject to those third-party's policies.
+# ⚖️ Legal disclaimer
+<p style="line-height: 1; font-style: italic;">The RD-agent is provided “as is”, without warranty of any kind, express or implied, including but not limited to the warranties of merchantability, fitness for a particular purpose and noninfringement. The RD-agent is aimed to facilitate research and development process in the financial industry and not ready-to-use for any financial investment or advice. Users shall independently assess and test the risks of the RD-agent in a specific use scenario, ensure the responsible use of AI technology, including but not limited to developing and integrating risk mitigation measures, and comply with all applicable laws and regulations in all applicable jurisdictions. The RD-agent does not provide financial opinions or reflect the opinions of Microsoft, nor is it designed to replace the role of qualified financial professionals in formulating, assessing, and approving finance products. The inputs and outputs of the RD-agent belong to the users and users shall assume all liability under any theory of liability, whether in contract, torts, regulatory, negligence, products liability, or otherwise, associated with use of the RD-agent and any inputs and outputs thereof.</p>

TODO.md

@@ -0,0 +1,10 @@
+We encourage to set the TODOs in code. But some TODOs are more global.
+So we place it here.
+
+
+- [ ] Aligning the naming of files in components & scenarios.
+  - We would like to have the same logic for naming convention in components(reusable components for all scenarios) and scenarios (componets for specific scenario).
+  - But now we have following mismatch
+    - `coder` in `components` & `developer` in `components`
+- [ ] The name of the folders mismatch with the content in them.
+  - Why are scenarios in experiments?

constraints/3.10.txt

@@ -4,30 +4,51 @@ alabaster==0.7.13
 annotated-types==0.6.0
 anyio==4.2.0
 appdirs==1.4.4
+argon2-cffi==23.1.0
+argon2-cffi-bindings==21.2.0
+arrow==1.3.0
+asttokens==2.4.1
+async-lru==2.0.4
 async-timeout==4.0.3
 attrs==23.2.0
 autodoc-pydantic==2.0.1
 azure-ai-formrecognizer==3.3.2
 azure-common==1.1.28
 azure-core==1.29.6
+azure-identity==1.17.1
 Babel==2.14.0
 beautifulsoup4==4.12.2
 black==23.12.1
+bleach==6.1.0
+blosc2==2.7.1
 build==1.0.3
 certifi==2023.11.17
 cffi==1.16.0
 charset-normalizer==3.3.2
 click==8.1.7
 colorama==0.4.6
+comm==0.2.2
+contourpy==1.2.1
 coverage==7.4.0
 cryptography==41.0.7
+cycler==0.12.1
 Cython==3.0.7
 dataclasses-json==0.6.3
+debugpy==1.8.2
+decorator==5.1.1
+defusedxml==0.7.1
+dill==0.3.8
 distro==1.9.0
+docker==7.1.0
 docutils==0.20.1
 exceptiongroup==1.2.0
+executing==2.0.1
+fastjsonschema==2.20.0
+feedparser==6.0.11
 filelock==3.13.1
 fire==0.5.0
+fonttools==4.53.1
+fqdn==1.5.1
 frozenlist==1.4.1
 fsspec==2023.12.2
 furo==2023.9.10
@@ -41,81 +62,133 @@ idna==3.6
 imagesize==1.4.1
 importlib-metadata==7.0.1
 iniconfig==2.0.0
+ipykernel==6.29.5
+ipython==8.26.0
+ipywidgets==8.1.3
 isodate==0.6.1
+isoduration==20.11.0
 isort==5.13.2
 jaraco.classes==3.3.0
+jedi==0.19.1
 jeepney==0.8.0
-Jinja2==3.1.2
+joblib==1.4.2
+json5==0.9.25
 jsonpatch==1.33
 jsonpointer==2.4
+jsonschema==4.23.0
+jsonschema-specifications==2023.12.1
+jupyter==1.0.0
+jupyter-console==6.6.3
+jupyter-events==0.10.0
+jupyter-lsp==2.2.5
+jupyter_client==8.6.2
+jupyter_core==5.7.2
+jupyter_server==2.14.2
+jupyter_server_terminals==0.5.3
+jupyterlab==4.2.4
+jupyterlab_pygments==0.3.0
+jupyterlab_server==2.27.3
+jupyterlab_widgets==3.0.11
 keyring==24.3.0
+kiwisolver==1.4.5
 langchain==0.0.353
 langchain-community==0.0.7
 langchain-core==0.1.4
 langsmith==0.0.75
+Levenshtein==0.25.1
 livereload==2.6.3
 loguru==0.7.2
+loguru-mypy==0.0.4
 lxml==5.0.0
 markdown-it-py==3.0.0
-MarkupSafe==2.1.3
 marshmallow==3.20.1
+matplotlib==3.9.1
+matplotlib-inline==0.1.7
 mdit-py-plugins==0.4.0
 mdurl==0.1.2
+mistune==3.0.2
 more-itertools==10.1.0
-mpmath==1.3.0
+msal==1.30.0
+msal-extensions==1.2.0
+msgpack==1.0.8
 msrest==0.7.1
 multidict==6.0.4
 mypy==1.10.0
 mypy-extensions==1.0.0
 myst-parser==2.0.0
-networkx==3.2.1
+nbclient==0.10.0
+nbconvert==7.16.4
+nbformat==5.10.4
+ndindex==1.8
+nest-asyncio==1.6.0
 nh3==0.2.15
+notebook==7.2.1
+notebook_shim==0.2.4
+numexpr==2.10.1
 numpy==1.26.2
-nvidia-cublas-cu12==12.1.3.1
-nvidia-cuda-cupti-cu12==12.1.105
-nvidia-cuda-nvrtc-cu12==12.1.105
-nvidia-cuda-runtime-cu12==12.1.105
-nvidia-cudnn-cu12==8.9.2.26
-nvidia-cufft-cu12==11.0.2.54
-nvidia-curand-cu12==10.3.2.106
-nvidia-cusolver-cu12==11.4.5.107
-nvidia-cusparse-cu12==12.1.0.106
-nvidia-nccl-cu12==2.18.1
-nvidia-nvjitlink-cu12==12.3.101
-nvidia-nvtx-cu12==12.1.105
 oauthlib==3.2.2
 openai==1.6.1
+overrides==7.7.0
 packaging==23.2
+pandarallel==1.6.5
 pandas==2.1.4
+pandocfilters==1.5.1
+parso==0.8.4
 pathspec==0.12.1
+patsy==0.5.6
+pexpect==4.9.0
 pkginfo==1.9.6
 platformdirs==4.1.0
 pluggy==1.3.0
+portalocker==2.10.1
+prometheus_client==0.20.0
+prompt_toolkit==3.0.47
+psutil==6.0.0
+ptyprocess==0.7.0
+pure_eval==0.2.3
+py-cpuinfo==9.0.0
 pycparser==2.21
 pydantic==2.5.3
 pydantic-settings==2.1.0
 pydantic_core==2.14.6
 Pygments==2.17.2
+PyJWT==2.8.0
+PyMuPDF==1.24.9
+PyMuPDFb==1.24.9
+pyparsing==3.1.2
 pypdf==3.17.4
 pyproject_hooks==1.0.0
 pytest==7.4.4
 python-dateutil==2.8.2
 python-dotenv==1.0.0
+python-json-logger==2.0.7
+python-Levenshtein==0.25.1
 pytz==2023.3.post1
 PyYAML==6.0.1
+pyzmq==26.0.3
+qtconsole==5.5.2
+QtPy==2.4.1
+rapidfuzz==3.9.5
 readme-renderer==42.0
+referencing==0.35.1
+regex==2024.7.24
 requests==2.31.0
 requests-oauthlib==1.3.1
 requests-toolbelt==1.0.0
+rfc3339-validator==0.1.4
 rfc3986==2.0.0
+rfc3986-validator==0.1.1
 rich==13.7.0
+rpds-py==0.19.1
 ruamel.yaml==0.18.5
 ruamel.yaml.clib==0.2.8
 ruff==0.4.5
-scipy==1.11.4
+scikit-learn==1.5.1
 SecretStorage==3.3.3
 semver==3.0.2
+Send2Trash==1.8.3
 setuptools-scm==8.0.4
+sgmllib3k==1.0.0
 shellingham==1.5.4
 six==1.16.0
 sniffio==1.3.0
@@ -133,21 +206,38 @@ sphinxcontrib-jsmath==1.0.1
 sphinxcontrib-qthelp==1.0.6
 sphinxcontrib-serializinghtml==1.1.9
 SQLAlchemy==2.0.24
-sympy==1.12
+stack-data==0.6.3
+statsmodels==0.14.2
+tables==3.9.2
+tabulate==0.9.0
 tenacity==8.2.3
 termcolor==2.4.0
+terminado==0.18.1
+threadpoolctl==3.5.0
+tiktoken==0.7.0
+tinycss2==1.3.0
 toml-sort==0.23.1
 tomli==2.0.1
 tomlkit==0.12.3
-torch==2.1.2
 tornado==6.4
 tqdm==4.66.1
-triton==2.1.0
+traitlets==5.14.3
+tree-sitter==0.22.3
+tree-sitter-python==0.21.0
 twine==4.0.2
 typer==0.9.0
+types-psutil==6.0.0.20240621
+types-python-dateutil==2.9.0.20240316
+types-PyYAML==6.0.12.20240724
+types-tqdm==4.66.0.20240417
 typing-inspect==0.9.0
-typing_extensions==4.9.0
 tzdata==2023.4
+uri-template==1.3.0
 urllib3==2.1.0
+wcwidth==0.2.13
+webcolors==24.6.0
+webencodings==0.5.1
+websocket-client==1.8.0
+widgetsnbextension==4.0.11
 yarl==1.9.4
 zipp==3.17.0

constraints/3.11.txt

@@ -4,28 +4,49 @@ alabaster==0.7.13
 annotated-types==0.6.0
 anyio==4.2.0
 appdirs==1.4.4
+argon2-cffi==23.1.0
+argon2-cffi-bindings==21.2.0
+arrow==1.3.0
+asttokens==2.4.1
+async-lru==2.0.4
 attrs==23.2.0
 autodoc-pydantic==2.0.1
 azure-ai-formrecognizer==3.3.2
 azure-common==1.1.28
 azure-core==1.29.6
+azure-identity==1.17.1
 Babel==2.14.0
 beautifulsoup4==4.12.2
 black==23.12.1
+bleach==6.1.0
+blosc2==2.7.1
 build==1.0.3
 certifi==2023.11.17
 cffi==1.16.0
 charset-normalizer==3.3.2
 click==8.1.7
 colorama==0.4.6
+comm==0.2.2
+contourpy==1.2.1
 coverage==7.4.0
 cryptography==41.0.7
+cycler==0.12.1
 Cython==3.0.7
 dataclasses-json==0.6.3
+debugpy==1.8.2
+decorator==5.1.1
+defusedxml==0.7.1
+dill==0.3.8
 distro==1.9.0
+docker==7.1.0
 docutils==0.20.1
+executing==2.0.1
+fastjsonschema==2.20.0
+feedparser==6.0.11
 filelock==3.13.1
 fire==0.5.0
+fonttools==4.53.1
+fqdn==1.5.1
 frozenlist==1.4.1
 fsspec==2023.12.2
 furo==2023.9.10
@@ -39,81 +60,133 @@ idna==3.6
 imagesize==1.4.1
 importlib-metadata==7.0.1
 iniconfig==2.0.0
+ipykernel==6.29.5
+ipython==8.26.0
+ipywidgets==8.1.3
 isodate==0.6.1
+isoduration==20.11.0
 isort==5.13.2
 jaraco.classes==3.3.0
+jedi==0.19.1
 jeepney==0.8.0
-Jinja2==3.1.2
+joblib==1.4.2
+json5==0.9.25
 jsonpatch==1.33
 jsonpointer==2.4
+jsonschema==4.23.0
+jsonschema-specifications==2023.12.1
+jupyter==1.0.0
+jupyter-console==6.6.3
+jupyter-events==0.10.0
+jupyter-lsp==2.2.5
+jupyter_client==8.6.2
+jupyter_core==5.7.2
+jupyter_server==2.14.2
+jupyter_server_terminals==0.5.3
+jupyterlab==4.2.4
+jupyterlab_pygments==0.3.0
+jupyterlab_server==2.27.3
+jupyterlab_widgets==3.0.11
 keyring==24.3.0
+kiwisolver==1.4.5
 langchain==0.0.353
 langchain-community==0.0.7
 langchain-core==0.1.4
 langsmith==0.0.75
+Levenshtein==0.25.1
 livereload==2.6.3
 loguru==0.7.2
+loguru-mypy==0.0.4
 lxml==5.0.0
 markdown-it-py==3.0.0
-MarkupSafe==2.1.3
 marshmallow==3.20.1
+matplotlib==3.9.1
+matplotlib-inline==0.1.7
 mdit-py-plugins==0.4.0
 mdurl==0.1.2
+mistune==3.0.2
 more-itertools==10.1.0
-mpmath==1.3.0
+msal==1.30.0
+msal-extensions==1.2.0
+msgpack==1.0.8
 msrest==0.7.1
 multidict==6.0.4
 mypy==1.10.0
 mypy-extensions==1.0.0
 myst-parser==2.0.0
-networkx==3.2.1
+nbclient==0.10.0
+nbconvert==7.16.4
+nbformat==5.10.4
+ndindex==1.8
+nest-asyncio==1.6.0
 nh3==0.2.15
+notebook==7.2.1
+notebook_shim==0.2.4
+numexpr==2.10.1
 numpy==1.26.2
-nvidia-cublas-cu12==12.1.3.1
-nvidia-cuda-cupti-cu12==12.1.105
-nvidia-cuda-nvrtc-cu12==12.1.105
-nvidia-cuda-runtime-cu12==12.1.105
-nvidia-cudnn-cu12==8.9.2.26
-nvidia-cufft-cu12==11.0.2.54
-nvidia-curand-cu12==10.3.2.106
-nvidia-cusolver-cu12==11.4.5.107
-nvidia-cusparse-cu12==12.1.0.106
-nvidia-nccl-cu12==2.18.1
-nvidia-nvjitlink-cu12==12.3.101
-nvidia-nvtx-cu12==12.1.105
 oauthlib==3.2.2
 openai==1.6.1
+overrides==7.7.0
 packaging==23.2
+pandarallel==1.6.5
 pandas==2.1.4
+pandocfilters==1.5.1
+parso==0.8.4
 pathspec==0.12.1
+patsy==0.5.6
+pexpect==4.9.0
 pkginfo==1.9.6
 platformdirs==4.1.0
 pluggy==1.3.0
+portalocker==2.10.1
+prometheus_client==0.20.0
+prompt_toolkit==3.0.47
+psutil==6.0.0
+ptyprocess==0.7.0
+pure_eval==0.2.3
+py-cpuinfo==9.0.0
 pycparser==2.21
 pydantic==2.5.3
 pydantic-settings==2.1.0
 pydantic_core==2.14.6
 Pygments==2.17.2
+PyJWT==2.9.0
+PyMuPDF==1.24.9
+PyMuPDFb==1.24.9
+pyparsing==3.1.2
 pypdf==3.17.4
 pyproject_hooks==1.0.0
 pytest==7.4.4
 python-dateutil==2.8.2
 python-dotenv==1.0.0
+python-json-logger==2.0.7
+python-Levenshtein==0.25.1
 pytz==2023.3.post1
 PyYAML==6.0.1
+pyzmq==26.0.3
+qtconsole==5.5.2
+QtPy==2.4.1
+rapidfuzz==3.9.5
 readme-renderer==42.0
+referencing==0.35.1
+regex==2024.7.24
 requests==2.31.0
 requests-oauthlib==1.3.1
 requests-toolbelt==1.0.0
+rfc3339-validator==0.1.4
 rfc3986==2.0.0
+rfc3986-validator==0.1.1
 rich==13.7.0
+rpds-py==0.19.1
 ruamel.yaml==0.18.5
 ruamel.yaml.clib==0.2.8
 ruff==0.4.5
-scipy==1.11.4
+scikit-learn==1.5.1
 SecretStorage==3.3.3
 semver==3.0.2
+Send2Trash==1.8.3
 setuptools-scm==8.0.4
+sgmllib3k==1.0.0
 shellingham==1.5.4
 six==1.16.0
 sniffio==1.3.0
@@ -131,20 +204,37 @@ sphinxcontrib-jsmath==1.0.1
 sphinxcontrib-qthelp==1.0.6
 sphinxcontrib-serializinghtml==1.1.9
 SQLAlchemy==2.0.24
-sympy==1.12
+stack-data==0.6.3
+statsmodels==0.14.2
+tables==3.9.2
+tabulate==0.9.0
 tenacity==8.2.3
 termcolor==2.4.0
+terminado==0.18.1
+threadpoolctl==3.5.0
+tiktoken==0.7.0
+tinycss2==1.3.0
 toml-sort==0.23.1
 tomlkit==0.12.3
-torch==2.1.2
 tornado==6.4
 tqdm==4.66.1
-triton==2.1.0
+traitlets==5.14.3
+tree-sitter==0.22.3
+tree-sitter-python==0.21.0
 twine==4.0.2
 typer==0.9.0
+types-psutil==6.0.0.20240621
+types-python-dateutil==2.9.0.20240316
+types-PyYAML==6.0.12.20240724
+types-tqdm==4.66.0.20240417
 typing-inspect==0.9.0
-typing_extensions==4.9.0
 tzdata==2023.4
+uri-template==1.3.0
 urllib3==2.1.0
+wcwidth==0.2.13
+webcolors==24.6.0
+webencodings==0.5.1
+websocket-client==1.8.0
+widgetsnbextension==4.0.11
 yarl==1.9.4
 zipp==3.17.0

constraints/3.8.txt

@@ -1,154 +0,0 @@
-aiohttp==3.9.1
-aiosignal==1.3.1
-alabaster==0.7.13
-annotated-types==0.6.0
-anyio==4.2.0
-appdirs==1.4.4
-async-timeout==4.0.3
-attrs==23.2.0
-autodoc-pydantic==2.0.1
-azure-ai-formrecognizer==3.3.2
-azure-common==1.1.28
-azure-core==1.29.6
-Babel==2.14.0
-beautifulsoup4==4.12.2
-black==23.12.1
-build==1.0.3
-certifi==2023.11.17
-cffi==1.16.0
-charset-normalizer==3.3.2
-click==8.1.7
-colorama==0.4.6
-coverage==7.4.0
-cryptography==41.0.7
-Cython==3.0.7
-dataclasses-json==0.6.3
-distro==1.9.0
-docutils==0.20.1
-exceptiongroup==1.2.0
-filelock==3.13.1
-fire==0.5.0
-frozenlist==1.4.1
-fsspec==2023.12.2
-furo==2023.9.10
-fuzzywuzzy==0.18.0
-git-changelog==2.4.0
-greenlet==3.0.3
-h11==0.14.0
-httpcore==1.0.2
-httpx==0.26.0
-idna==3.6
-imagesize==1.4.1
-importlib-metadata==7.0.1
-importlib-resources==6.1.1
-iniconfig==2.0.0
-isodate==0.6.1
-isort==5.13.2
-jaraco.classes==3.3.0
-jeepney==0.8.0
-Jinja2==3.1.2
-jsonpatch==1.33
-jsonpointer==2.4
-keyring==24.3.0
-langchain==0.0.353
-langchain-community==0.0.7
-langchain-core==0.1.4
-langsmith==0.0.75
-livereload==2.6.3
-loguru==0.7.2
-lxml==5.0.0
-markdown-it-py==3.0.0
-MarkupSafe==2.1.3
-marshmallow==3.20.1
-mdit-py-plugins==0.4.0
-mdurl==0.1.2
-more-itertools==10.1.0
-mpmath==1.3.0
-msrest==0.7.1
-multidict==6.0.4
-mypy==1.10.0
-mypy-extensions==1.0.0
-myst-parser==2.0.0
-networkx==3.1
-nh3==0.2.15
-numpy==1.24.4
-nvidia-cublas-cu12==12.1.3.1
-nvidia-cuda-cupti-cu12==12.1.105
-nvidia-cuda-nvrtc-cu12==12.1.105
-nvidia-cuda-runtime-cu12==12.1.105
-nvidia-cudnn-cu12==8.9.2.26
-nvidia-cufft-cu12==11.0.2.54
-nvidia-curand-cu12==10.3.2.106
-nvidia-cusolver-cu12==11.4.5.107
-nvidia-cusparse-cu12==12.1.0.106
-nvidia-nccl-cu12==2.18.1
-nvidia-nvjitlink-cu12==12.3.101
-nvidia-nvtx-cu12==12.1.105
-oauthlib==3.2.2
-openai==1.6.1
-packaging==23.2
-pandas==2.0.3
-pathspec==0.12.1
-pkginfo==1.9.6
-platformdirs==4.1.0
-pluggy==1.3.0
-pycparser==2.21
-pydantic==2.5.3
-pydantic-settings==2.1.0
-pydantic_core==2.14.6
-Pygments==2.17.2
-pypdf==3.17.4
-pyproject_hooks==1.0.0
-pytest==7.4.4
-python-dateutil==2.8.2
-python-dotenv==1.0.0
-pytz==2023.3.post1
-PyYAML==6.0.1
-readme-renderer==42.0
-requests==2.31.0
-requests-oauthlib==1.3.1
-requests-toolbelt==1.0.0
-rfc3986==2.0.0
-rich==13.7.0
-ruamel.yaml==0.18.5
-ruamel.yaml.clib==0.2.8
-ruff==0.4.5
-scipy==1.10.1
-SecretStorage==3.3.3
-semver==3.0.2
-setuptools-scm==8.0.4
-shellingham==1.5.4
-six==1.16.0
-sniffio==1.3.0
-snowballstemmer==2.2.0
-soupsieve==2.5
-Sphinx==7.1.2
-sphinx-autobuild==2021.3.14
-sphinx-basic-ng==1.0.0b2
-sphinx-click==5.1.0
-sphinx-togglebutton==0.3.2
-sphinxcontrib-applehelp==1.0.4
-sphinxcontrib-devhelp==1.0.2
-sphinxcontrib-htmlhelp==2.0.1
-sphinxcontrib-jsmath==1.0.1
-sphinxcontrib-qthelp==1.0.3
-sphinxcontrib-serializinghtml==1.1.5
-SQLAlchemy==2.0.24
-sympy==1.12
-tenacity==8.2.3
-termcolor==2.4.0
-toml-sort==0.23.1
-tomli==2.0.1
-tomlkit==0.12.3
-torch==2.1.2
-tornado==6.4
-tqdm==4.66.1
-triton==2.1.0
-twine==4.0.2
-typer==0.9.0
-typing-inspect==0.9.0
-typing_extensions==4.9.0
-tzdata==2023.4
-urllib3==2.1.0
-yarl==1.9.4
-zipp==3.17.0

constraints/3.9.txt

@@ -1,153 +0,0 @@
-aiohttp==3.9.1
-aiosignal==1.3.1
-alabaster==0.7.13
-annotated-types==0.6.0
-anyio==4.2.0
-appdirs==1.4.4
-async-timeout==4.0.3
-attrs==23.2.0
-autodoc-pydantic==2.0.1
-azure-ai-formrecognizer==3.3.2
-azure-common==1.1.28
-azure-core==1.29.6
-Babel==2.14.0
-beautifulsoup4==4.12.2
-black==23.12.1
-build==1.0.3
-certifi==2023.11.17
-cffi==1.16.0
-charset-normalizer==3.3.2
-click==8.1.7
-colorama==0.4.6
-coverage==7.4.0
-cryptography==41.0.7
-Cython==3.0.7
-dataclasses-json==0.6.3
-distro==1.9.0
-docutils==0.20.1
-exceptiongroup==1.2.0
-filelock==3.13.1
-fire==0.5.0
-frozenlist==1.4.1
-fsspec==2023.12.2
-furo==2023.9.10
-fuzzywuzzy==0.18.0
-git-changelog==2.4.0
-greenlet==3.0.3
-h11==0.14.0
-httpcore==1.0.2
-httpx==0.26.0
-idna==3.6
-imagesize==1.4.1
-importlib-metadata==7.0.1
-iniconfig==2.0.0
-isodate==0.6.1
-isort==5.13.2
-jaraco.classes==3.3.0
-jeepney==0.8.0
-Jinja2==3.1.2
-jsonpatch==1.33
-jsonpointer==2.4
-keyring==24.3.0
-langchain==0.0.353
-langchain-community==0.0.7
-langchain-core==0.1.4
-langsmith==0.0.75
-livereload==2.6.3
-loguru==0.7.2
-lxml==5.0.0
-markdown-it-py==3.0.0
-MarkupSafe==2.1.3
-marshmallow==3.20.1
-mdit-py-plugins==0.4.0
-mdurl==0.1.2
-more-itertools==10.1.0
-mpmath==1.3.0
-msrest==0.7.1
-multidict==6.0.4
-mypy==1.10.0
-mypy-extensions==1.0.0
-myst-parser==2.0.0
-networkx==3.2.1
-nh3==0.2.15
-numpy==1.26.2
-nvidia-cublas-cu12==12.1.3.1
-nvidia-cuda-cupti-cu12==12.1.105
-nvidia-cuda-nvrtc-cu12==12.1.105
-nvidia-cuda-runtime-cu12==12.1.105
-nvidia-cudnn-cu12==8.9.2.26
-nvidia-cufft-cu12==11.0.2.54
-nvidia-curand-cu12==10.3.2.106
-nvidia-cusolver-cu12==11.4.5.107
-nvidia-cusparse-cu12==12.1.0.106
-nvidia-nccl-cu12==2.18.1
-nvidia-nvjitlink-cu12==12.3.101
-nvidia-nvtx-cu12==12.1.105
-oauthlib==3.2.2
-openai==1.6.1
-packaging==23.2
-pandas==2.1.4
-pathspec==0.12.1
-pkginfo==1.9.6
-platformdirs==4.1.0
-pluggy==1.3.0
-pycparser==2.21
-pydantic==2.5.3
-pydantic-settings==2.1.0
-pydantic_core==2.14.6
-Pygments==2.17.2
-pypdf==3.17.4
-pyproject_hooks==1.0.0
-pytest==7.4.4
-python-dateutil==2.8.2
-python-dotenv==1.0.0
-pytz==2023.3.post1
-PyYAML==6.0.1
-readme-renderer==42.0
-requests==2.31.0
-requests-oauthlib==1.3.1
-requests-toolbelt==1.0.0
-rfc3986==2.0.0
-rich==13.7.0
-ruamel.yaml==0.18.5
-ruamel.yaml.clib==0.2.8
-ruff==0.4.5
-scipy==1.11.4
-SecretStorage==3.3.3
-semver==3.0.2
-setuptools-scm==8.0.4
-shellingham==1.5.4
-six==1.16.0
-sniffio==1.3.0
-snowballstemmer==2.2.0
-soupsieve==2.5
-Sphinx==7.2.6
-sphinx-autobuild==2021.3.14
-sphinx-basic-ng==1.0.0b2
-sphinx-click==5.1.0
-sphinx-togglebutton==0.3.2
-sphinxcontrib-applehelp==1.0.7
-sphinxcontrib-devhelp==1.0.5
-sphinxcontrib-htmlhelp==2.0.4
-sphinxcontrib-jsmath==1.0.1
-sphinxcontrib-qthelp==1.0.6
-sphinxcontrib-serializinghtml==1.1.9
-SQLAlchemy==2.0.24
-sympy==1.12
-tenacity==8.2.3
-termcolor==2.4.0
-toml-sort==0.23.1
-tomli==2.0.1
-tomlkit==0.12.3
-torch==2.1.2
-tornado==6.4
-tqdm==4.66.1
-triton==2.1.0
-twine==4.0.2
-typer==0.9.0
-typing-inspect==0.9.0
-typing_extensions==4.9.0
-tzdata==2023.4
-urllib3==2.1.0
-yarl==1.9.4
-zipp==3.17.0

docs/Makefile

@@ -0,0 +1,20 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS    ?=
+SPHINXBUILD   ?= sphinx-build
+SOURCEDIR     = .
+BUILDDIR      = build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

docs/api_reference.rst

@@ -0,0 +1,15 @@
+=============
+API Reference
+=============
+
+Here you can find all ``RDAgent``'s interfaces.
+
+
+RD Loop
+=======
+
+Research
+--------
+
+.. automodule:: rdagent.core.proposal
+    :members:

docs/changelog.md

@@ -0,0 +1,4 @@
+# Changelog
+
+## [Unreleased]
+<!-- insertion marker -->

docs/conf.py

@@ -0,0 +1,72 @@
+# Configuration file for the Sphinx documentation builder.
+#
+# For the full list of built-in configuration values, see the documentation:
+# https://www.sphinx-doc.org/en/master/usage/configuration.html
+
+# -- Project information -----------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information
+
+import subprocess
+
+latest_tag = subprocess.check_output(["git", "describe", "--tags", "--abbrev=0"], text=True).strip()
+
+project = "RDAgent"
+copyright = "2024, Microsoft"
+author = "Microsoft"
+
+# -- General configuration ---------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration
+
+extensions = ["sphinx.ext.autodoc", "sphinxcontrib.autodoc_pydantic"]
+
+autodoc_member_order = "bysource"
+
+# The suffix of source filenames.
+source_suffix = {".rst": "restructuredtext"}
+
+# The encoding of source files.
+source_encoding = "utf-8"
+
+# The main toctree document.
+master_doc = "index"
+
+# The version info for the project you're documenting, acts as replacement for
+# |version| and |release|, also used in various other places throughout the
+# built documents.
+#
+# The short X.Y version.
+version = latest_tag
+release = latest_tag
+
+# The language for content autogenerated by Sphinx. Refer to documentation for
+# a list of supported languages.
+language = "en"
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+exclude_patterns = ["build"]
+
+# -- Options for HTML output -------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = "sphinx"
+
+try:
+    import furo
+
+    html_theme = "furo"
+    html_theme_options = {
+        "navigation_with_keys": True,
+    }
+except ImportError:
+    html_theme = "default"
+
+html_logo = "_static/logo.png"
+html_static_path = ["_static"]
+html_favicon = "_static/favicon.ico"
+
+html_theme_options = {
+    "source_repository": "https://github.com/microsoft/RD-Agent",
+    "source_branch": "main",
+    "source_directory": "docs/",
+}

docs/development.rst

@@ -0,0 +1,85 @@
+=========================
+For Development
+=========================
+
+If you want to try the latest version or contribute to RD-Agent. You can install it from the source and follow the commands in this page.
+
+   .. code-block:: bash
+
+      git clone https://github.com/microsoft/RD-Agent
+
+
+🔧Prepare for development
+=========================
+
+- Set up the development environment.
+
+   .. code-block:: bash
+
+      make dev
+
+- Run linting and checking.
+
+   .. code-block:: bash
+
+      make lint
+
+
+- Some linting issues can be fixed automatically. We have added a command in the Makefile for easy use.
+
+   .. code-block:: bash
+
+      make auto-lint
+
+
+
+Code Structure
+=========================
+
+.. code-block:: text
+
+    📂 src
+    ➥ 📂 <project name>: avoid namespace conflict
+      ➥ 📁 core
+      ➥ 📁 components/A
+      ➥ 📁 components/B
+      ➥ 📁 components/C
+      ➥ 📁 scenarios/X
+      ➥ 📁 scenarios/Y
+      ➥ 📂 app
+    ➥ 📁 scripts
+
+.. list-table::
+   :header-rows: 1
+
+   * - Folder Name
+     - Description
+   * - 📁 core
+     - The core framework of the system. All classes should be abstract and usually can't be used directly.
+   * - 📁 component/A
+     - Useful components that can be used by others (e.g., scenarios). Many subclasses of core classes are located here.
+   * - 📁 scenarios/X
+     - Concrete features for specific scenarios (usually built based on components or core). These modules are often unreusable across scenarios.
+   * - 📁 app
+     - Applications for specific scenarios (usually built based on components or scenarios). Removing any of them does not affect the system's completeness or other scenarios.
+   * - 📁 scripts
+     - Quick and dirty things. These are candidates for core, components, scenarios, and apps.
+
+
+
+Conventions
+===========
+
+
+File Naming Convention
+----------------------
+
+.. list-table::
+   :header-rows: 1
+
+   * - Name
+     - Description
+   * - `conf.py`
+     - The configuration for the module, app, and project.
+
+.. <!-- TODO: renaming files -->

docs/index.rst

@@ -0,0 +1,34 @@
+.. RDAgent documentation master file, created by
+   sphinx-quickstart on Mon Jul 15 04:27:50 2024.
+   You can adapt this file completely to your liking, but it should at least
+   contain the root `toctree` directive.
+
+Welcome to RDAgent's documentation!
+===================================
+
+.. image:: _static/logo.png
+   :alt: RD-Agent Logo
+
+.. toctree::
+   :maxdepth: 3
+   :caption: Doctree:
+
+   introduction
+   installation_and_configuration
+   scens/catalog
+   project_framework_introduction
+   ui
+   research/catalog
+   development
+   api_reference
+   policy
+
+   GitHub <https://github.com/microsoft/RD-Agent>
+
+
+Indices and tables
+==================
+
+* :ref:`genindex`
+* :ref:`modindex`
+* :ref:`search`

docs/installation_and_configuration.rst

@@ -0,0 +1,165 @@
+==============================
+Installation and Configuration
+==============================
+
+Installation
+============
+
+**Install RDAgent**: For different scenarios
+
+- for purely users: please use ``pip install rdagent`` to install RDAgent
+- for dev users: `See development <development.html>`_
+
+**Install Docker**: RDAgent is designed for research and development, acting like a human researcher and developer. It can write and run code in various environments, primarily using Docker for code execution. This keeps the remaining dependencies simple. Users must ensure Docker is installed before attempting most scenarios. Please refer to the `official 🐳Docker page <https://docs.docker.com/engine/install/>`_ for installation instructions.
+
+Configuration
+=============
+
+To run the application, please create a `.env` file in the root directory of the project and add environment variables according to your requirements.
+
+The standard configuration options for the user using the OpenAI API are provided in the `.env.example` file.
+
+Here are some other configuration options that you can use:
+
+OpenAI API
+------------
+
+Here is a standard configuration for the user using the OpenAI API.
+
+   .. code-block:: Properties
+
+      OPENAI_API_KEY=<your_api_key>
+      EMBEDDING_MODEL=text-embedding-3-small
+      CHAT_MODEL=gpt-4-turbo
+
+Azure OpenAI
+------------
+
+The following environment variables are standard configuration options for the user using the OpenAI API.
+
+   .. code-block:: Properties
+      
+      USE_AZURE=True
+
+      OPENAI_API_KEY=<replace_with_your_openai_api_key>
+      
+      EMBEDDING_MODEL=text-embedding-3-small
+      EMBEDDING_AZURE_API_BASE= # The base URL for the Azure OpenAI API.
+      EMBEDDING_AZURE_API_VERSION = # The version of the Azure OpenAI API.
+
+      CHAT_MODEL=gpt-4-turbo
+      CHAT_AZURE_API_VERSION = # The version of the Azure OpenAI API.
+
+Use Azure Token Provider
+------------------------
+
+If you are using the Azure token provider, you need to set the `USE_AZURE_TOKEN_PROVIDER` environment variable to `True`. then 
+use the environment variables provided in the `Azure Configuration section <installation_and_configuration.html#azure-openai>`_.
+
+
+☁️ Azure Configuration
+- Install Azure CLI:
+
+   ```sh
+   curl -L https://aka.ms/InstallAzureCli | bash
+   ```
+
+- Log in to Azure:
+
+   ```sh
+   az login --use-device-code
+   ```
+
+- `exit` and re-login to your environment (this step may not be necessary).
+
+
+Configuration List
+------------------
+
+.. TODO: use `autodoc-pydantic` .
+
+- OpenAI API Setting
+
++-----------------------------+--------------------------------------------------+-------------------------+
+| Configuration Option        | Meaning                                          | Default Value           |
++=============================+==================================================+=========================+
+| OPENAI_API_KEY              | API key for both chat and embedding models       | None                    |
++-----------------------------+--------------------------------------------------+-------------------------+
+| EMBEDDING_OPENAI_API_KEY    | Use a different API key for embedding model      | None                    |
++-----------------------------+--------------------------------------------------+-------------------------+
+| CHAT_OPENAI_API_KEY         | Set to use a different API key for chat model    | None                    |
++-----------------------------+--------------------------------------------------+-------------------------+
+| EMBEDDING_MODEL             | Name of the embedding model                      | text-embedding-3-small  |
++-----------------------------+--------------------------------------------------+-------------------------+
+| CHAT_MODEL                  | Name of the chat model                           | gpt-4-turbo             |
++-----------------------------+--------------------------------------------------+-------------------------+
+| EMBEDDING_AZURE_API_BASE    | Base URL for the Azure OpenAI API                | None                    |
++-----------------------------+--------------------------------------------------+-------------------------+
+| EMBEDDING_AZURE_API_VERSION | Version of the Azure OpenAI API                  | None                    |
++-----------------------------+--------------------------------------------------+-------------------------+
+| CHAT_AZURE_API_BASE         | Base URL for the Azure OpenAI API                | None                    |
++-----------------------------+--------------------------------------------------+-------------------------+
+| CHAT_AZURE_API_VERSION      | Version of the Azure OpenAI API                  | None                    |
++-----------------------------+--------------------------------------------------+-------------------------+
+| USE_AZURE                   | True if you are using Azure OpenAI               | False                   |
++-----------------------------+--------------------------------------------------+-------------------------+
+| USE_AZURE_TOKEN_PROVIDER    | True if you are using a Azure Token Provider     | False                   |
++-----------------------------+--------------------------------------------------+-------------------------+
+
+- Globol Setting
+
++-----------------------------+--------------------------------------------------+-------------------------+
+| Configuration Option        | Meaning                                          | Default Value           |
++=============================+==================================================+=========================+
+| max_retry                   | Maximum number of times to retry                 | 10                      |
++-----------------------------+--------------------------------------------------+-------------------------+
+| retry_wait_seconds          | Number of seconds to wait before retrying        | 1                       |
++-----------------------------+--------------------------------------------------+-------------------------+
++ log_trace_path              | Path to log trace file                           | None                    |
++-----------------------------+--------------------------------------------------+-------------------------+
++ log_llm_chat_content        | Flag to indicate if chat content is logged       | True                    |
++-----------------------------+--------------------------------------------------+-------------------------+
+
+
+- Cache Setting
+
+.. TODO: update Meaning for caches
+
++------------------------------+--------------------------------------------------+-------------------------+
+| Configuration Option         | Meaning                                          | Default Value           |
++==============================+==================================================+=========================+
+| dump_chat_cache              | Flag to indicate if chat cache is dumped         | False                   |
++------------------------------+--------------------------------------------------+-------------------------+
+| dump_embedding_cache         | Flag to indicate if embedding cache is dumped    | False                   |
++------------------------------+--------------------------------------------------+-------------------------+
+| use_chat_cache               | Flag to indicate if chat cache is used           | False                   |
++------------------------------+--------------------------------------------------+-------------------------+
+| use_embedding_cache          | Flag to indicate if embedding cache is used      | False                   |
++------------------------------+--------------------------------------------------+-------------------------+
+| prompt_cache_path            | Path to prompt cache                             | ./prompt_cache.db       |
++------------------------------+--------------------------------------------------+-------------------------+
+| max_past_message_include     | Maximum number of past messages to include       | 10                      |
++------------------------------+--------------------------------------------------+-------------------------+
+
+
+
+
+Loading Configuration
+---------------------
+
+For users' convenience, we provide a CLI interface called `rdagent`, which automatically runs `load_dotenv()` to load environment variables from the `.env` file.
+However, this feature is not enabled by default for other scripts. We recommend users load the environment with the following steps:
+
+
+- ⚙️ Environment Configuration
+    - Place the `.env` file in the same directory as the `.env.example` file.
+        - The `.env.example` file contains the environment variables required for users using the OpenAI API (Please note that `.env.example` is an example file. `.env` is the one that will be finally used.)
+
+    - Export each variable in the .env file:
+
+      .. code-block:: sh
+
+          export $(grep -v '^#' .env | xargs)
+    
+    - If you want to change the default environment variables, you can refer to the above configuration and edith the `.env` file.
+

docs/introduction.rst

@@ -0,0 +1,18 @@
+=========================
+Introduction
+=========================
+
+
+
+In modern industry, research and development (R&D) is crucial for the enhancement of industrial productivity, especially in the AI era, where the core aspects of R&D are mainly focused on data and models. We are committed to automate these high-value generic R&D processes through our open source R&D automation tool RDAgent, which let AI drive data-driven AI.
+
+.. image:: _static/scen.png
+   :alt: Our focused scenario
+
+
+Our RDAgent is designed to automate the most critical industrial R&D processes, focusing first on data-driven scenarios, to greatly boost the development productivity of models and data. 
+
+Methodologically, we propose an autonomous agent framework that consists of two key parts: (R)esearch stands for actively exploring by proposing new ideas, and (D)evelopment stands for realizing these ideas. The effectiveness of these two components will ultimately get feedbacks through practice, and both research and development capabilities can continuously learn and grow in the process.
+
+
+For a quick start, visit `our GitHub home page <https://github.com/microsoft/RD-Agent>`_ ⚡. If you've already checked it out and want more details, please keep reading.

docs/make.bat

@@ -0,0 +1,35 @@
+@ECHO OFF
+
+pushd %~dp0
+
+REM Command file for Sphinx documentation
+
+if "%SPHINXBUILD%" == "" (
+	set SPHINXBUILD=sphinx-build
+)
+set SOURCEDIR=.
+set BUILDDIR=build
+
+%SPHINXBUILD% >NUL 2>NUL
+if errorlevel 9009 (
+	echo.
+	echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
+	echo.installed, then set the SPHINXBUILD environment variable to point
+	echo.to the full path of the 'sphinx-build' executable. Alternatively you
+	echo.may add the Sphinx directory to PATH.
+	echo.
+	echo.If you don't have Sphinx installed, grab it from
+	echo.https://www.sphinx-doc.org/
+	exit /b 1
+)
+
+if "%1" == "" goto help
+
+%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+goto end
+
+:help
+%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+
+:end
+popd

docs/policy.rst

@@ -0,0 +1,24 @@
+======
+Policy
+======
+
+This project welcomes contributions and suggestions.  Most contributions require you to agree to a
+Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us
+the rights to use your contribution. For details, visit https://cla.opensource.microsoft.com.
+
+When you submit a pull request, a CLA bot will automatically determine whether you need to provide
+a CLA and decorate the PR appropriately (e.g., status check, comment). Simply follow the instructions
+provided by the bot. You will only need to do this once across all repos using our CLA.
+
+This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/).
+For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or
+contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments.
+
+Trademarks
+==========
+
+This project may contain trademarks or logos for projects, products, or services. Authorized use of Microsoft 
+trademarks or logos is subject to and must follow 
+[Microsoft's Trademark & Brand Guidelines](https://www.microsoft.com/en-us/legal/intellectualproperty/trademarks/usage/general).
+Use of Microsoft trademarks or logos in modified versions of this project must not cause confusion or imply Microsoft sponsorship.
+Any use of third-party trademarks or logos are subject to those third-party's policies.

docs/project_framework_introduction.rst

@@ -0,0 +1,27 @@
+===============================
+Framework Design & Components
+===============================
+
+Framework & Components
+=========================
+
+.. NOTE: This depends on the correctness of `c-v` of github.
+
+.. image:: _static/Framework-RDAgent.png
+    :alt: Components & Feature Level
+
+The image above shows the overall framework of RDAgent.
+
+In a data mining expert's daily research and development process, they propose a hypothesis (e.g., a model structure like RNN can capture patterns in time-series data), design experiments (e.g., finance data contains time-series and we can verify the hypothesis in this scenario), implement the experiment as code (e.g., Pytorch model structure), and then execute the code to get feedback (e.g., metrics, loss curve, etc.). The experts learn from the feedback and improve in the next iteration.
+
+We have established a basic method framework that continuously proposes hypotheses, verifies them, and gets feedback from the real world. This is the first scientific research automation framework that supports linking with real-world verification.
+
+
+.. image:: https://github.com/user-attachments/assets/60cc2712-c32a-4492-a137-8aec59cdc66e
+    :alt: Class Level Figure
+
+The figure above shows the main classes and how they fit into the workflow for those interested in the detailed code.
+
+
+.. Detailed Design
+.. ===============

docs/requirements.txt

@@ -0,0 +1,4 @@
+sphinx
+sphinx_rtd_theme
+furo
+importlib.metadata

docs/research/benchmark.rst

@@ -0,0 +1,118 @@
+==============================
+Benchmark
+==============================
+
+Introduction
+=============
+
+
+Benchmarking the capabilities of the R&D is a very important research problem of the research area.
+
+Currently we are continuously exploring how to benchmark them.
+
+The current benchmarks are listed in this page
+
+
+Development Capability Benchmarking
+===================================
+
+
+Benchmark is used to evaluate the effectiveness of factors with fixed data.
+
+It mainly includes the following steps:
+
+1. :ref:`read and prepare the eval_data <data>`
+
+2. :ref:`declare the method to be tested and pass the arguments <config>`
+
+3. :ref:`declare the eval method and pass the arguments <config>`
+
+4. :ref:`run the eval <run>` 
+
+5. :ref:`save and show the result <show>` 
+
+Configuration 
+-------------
+.. _config:
+
+.. autopydantic_settings:: rdagent.components.benchmark.conf.BenchmarkSettings
+
+Example
+++++++++
+.. _example:
+
+The default value for ``bench_test_round`` is 10, and it will take about 2 hours to run 10 rounds.
+To modify it from ``10`` to ``2`` you can adjust this by adding environment variables in the .env file as shown below.
+
+.. code-block:: Properties
+
+      BENCHMARK_BENCH_TEST_ROUND=1
+
+Data Format
+-------------
+.. _data:
+
+The sample data in ``bench_data_path`` is a dictionary where each key represents a factor name. 
+
+The value associated with each key is factor data containing the following information:
+
+- **description**: A textual description of the factor.
+- **formulation**: A LaTeX formula representing the model's formulation.
+- **variables**: A dictionary of variables involved in the factor.
+- **Category**: The category or classification of the factor.
+- **Difficulty**: The difficulty level of implementing or understanding the factor.
+- **gt_code**: A piece of code associated with the factor.
+
+Here is the example of this data format:
+
+.. literalinclude:: ../../rdagent/components/benchmark/example.json
+   :language: json
+
+Run Benchmark
+-------------
+.. _run:
+
+Start benchmark after finishing the :doc:`../installation_and_configuration`.
+
+.. code-block:: Properties
+
+      python rdagent/app/quant_factor_benchmark/eval.py
+
+
+
+Once completed, a pkl file will be generated, and its path will be printed on the last line of the console.
+
+Show Result
+-------------
+.. _show:
+
+The ``analysis.py`` script is used to read data from pkl and convert it to an image.
+Modify the python code in ``rdagent/app/quant_factor_benchmark/analysis.py`` to specify the path to the pkl file and the output path for the png file.
+
+.. code-block:: Properties
+
+      python rdagent/app/quant_factor_benchmark/analysis.py
+
+A png file will be saved to the designated path as shown below.
+
+.. image:: ../_static/benchmark.png
+
+
+Related Paper
+-------------
+
+- `Towards Data-Centric Automatic R&D <https://arxiv.org/abs/2404.11276>`_:
+  We have developed a comprehensive benchmark called RD2Bench to assess data and model R&D capabilities. This benchmark includes a series of tasks that outline the features or structures of models. These tasks are used to evaluate the ability of LLM-Agents to implement them.
+
+.. code-block:: bibtex
+
+    @misc{chen2024datacentric,
+        title={Towards Data-Centric Automatic R&D},
+        author={Haotian Chen and Xinjie Shen and Zeqi Ye and Wenjun Feng and Haoxue Wang and Xiao Yang and Xu Yang and Weiqing Liu and Jiang Bian},
+        year={2024},
+        eprint={2404.11276},
+        archivePrefix={arXiv},
+        primaryClass={cs.AI}
+    }
+
+.. image:: https://github.com/user-attachments/assets/494f55d3-de9e-4e73-ba3d-a787e8f9e841

docs/research/catalog.rst

@@ -0,0 +1,34 @@
+===========
+Research
+===========
+
+To achieve the good effects and improve R&D capabilities, we face multiple challenges, the most important of which is the continuous evolution capability. Existing large language models (LLMs) find it difficult to continue growing their capabilities after training is completed. Moreover, the training process of LLMs focuses more on general knowledge, and the lack of depth in more specialized knowledge becomes an obstacle to solving professional R&D problems within the industry. This specialized knowledge needs to be learned and acquired from in-depth industry practice.
+
+
+Our RD-Agent, on the other hand, can continuously acquire in-depth domain knowledge through deep exploration during the R&D phase, allowing its R&D capabilities to keep growing.
+
+To address these key challenges and achieve industrial value, a series of research work needs to be completed.
+
+
+.. list-table:: Research Areas and Descriptions
+   :header-rows: 1
+
+   * - Research Area
+     - Description
+   * - :doc:`Benchmark <benchmark>`
+     - Benchmark the R&D abilities
+   * - Research
+     - Idea proposal: Explore new ideas or refine existing ones
+   * - :doc:`Development <dev>`
+     - Ability to realize ideas: Implement and execute ideas
+
+
+
+
+.. toctree::
+   :maxdepth: 1
+   :caption: Doctree:
+   :hidden:
+
+   benchmark
+   dev

docs/research/dev.rst

@@ -0,0 +1,25 @@
+==============================
+Development
+==============================
+
+
+Related Paper
+-------------
+
+- `Collaborative Evolving Strategy for Automatic Data-Centric Development <https://arxiv.org/abs/2407.18690>`_
+  Co-STEER is a method to tackle data-centric development (AD2) tasks and highlight its main challenges, which need expert-like implementation (i.e., learning domain knowledge from practice) and task scheduling capability (e.g., starting with easier tasks for better overall efficiency), areas that previous work has largely overlooked. Our Co-STEER agent enhances its domain knowledge through our evolving strategy and improves both its scheduling and implementation skills by gathering and using domain-specific practical experience. With a better schedule, implementation becomes faster. At the same time, as implementation feedback becomes more detailed, scheduling accuracy improves. These two capabilities grow together through practical feedback, enabling a collaborative evolution process.
+
+.. code-block:: bibtex
+
+    @misc{yang2024collaborative,
+        title={Collaborative Evolving Strategy for Automatic Data-Centric Development},
+        author={Xu Yang and Haotian Chen and Wenjun Feng and Haoxue Wang and Zeqi Ye and Xinjie Shen and Xiao Yang and Shizhao Sun and Weiqing Liu and Jiang Bian},
+        year={2024},
+        eprint={2407.18690},
+        archivePrefix={arXiv},
+        primaryClass={cs.AI}
+    }
+
+.. image:: https://github.com/user-attachments/assets/75d9769b-0edd-4caf-9d45-57d1e577054b
+   :alt: Collaborative Evolving Strategy for Automatic Data-Centric Development
+

docs/scens/catalog.rst

@@ -0,0 +1,46 @@
+=========================
+Scenarios
+=========================
+
+Scenario lists
+=========================
+
+In the two key areas of data-driven scenarios, model implementation and data building, our system aims to serve two main roles: 🦾copilot and 🤖agent.
+
+- The 🦾copilot follows human instructions to automate repetitive tasks.
+- The 🤖agent, being more autonomous, actively proposes ideas for better results in the future.
+
+The supported scenarios are listed below:
+
+
+
+.. list-table:: 
+    :header-rows: 1
+
+    * - Scenario/Target
+      - Model Implementation
+      - Data Building
+    * - 💹 Finance
+      - :ref:`🤖Iteratively Proposing Ideas & Evolving <model_agent_fin>`
+      - :ref:`🦾Auto reports reading & implementation <data_copilot_fin>`
+
+        :ref:`🤖Iteratively Proposing Ideas & Evolving <data_agent_fin>`
+    * - 🩺 Medical
+      - :ref:`🤖Iteratively Proposing Ideas & Evolving <model_agent_med>`
+      - 
+    * - 🏭 General
+      - :ref:`🦾Auto paper reading & implementation <model_copilot_general>`
+      - 
+
+
+.. toctree::
+   :maxdepth: 1
+   :caption: Doctree:
+   :hidden:
+
+   data_agent_fin
+   data_copilot_fin
+   model_agent_fin
+   model_agent_med
+   model_copilot_general
+

docs/scens/data_agent_fin.rst

@@ -0,0 +1,138 @@
+.. _data_agent_fin:
+
+=====================
+Finance Data Agent
+=====================
+
+
+**🤖 Automated Quantitative Trading & Iterative Factors Evolution**
+-------------------------------------------------------------------
+
+📖 Background
+~~~~~~~~~~~~~~
+In the dynamic world of quantitative trading, **factors** serve as the strategic tools that enable traders to exploit market inefficiencies. 
+These factors—ranging from simple metrics like price-to-earnings ratios to complex models like discounted cash flows—are the key to predicting stock prices with a high degree of accuracy.
+
+By leveraging these factors, quantitative traders can develop sophisticated strategies that not only identify market patterns but also significantly enhance trading efficiency and precision. 
+The ability to systematically analyze and apply these factors is what separates ordinary trading from truly strategic market outmaneuvering.
+And this is where the **Finance Model Agent** comes into play.
+
+🎥 `Demo <https://rdagent.azurewebsites.net/factor_loop>`_
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. raw:: html
+
+    <div style="display: flex; justify-content: center; align-items: center;">
+      <video width="600" controls>
+        <source src="https://rdagent.azurewebsites.net/media/65bb598f1372c1857ccbf09b2acf5d55830911625048c03102291098.mp4" type="video/mp4">
+        Your browser does not support the video tag.
+      </video>
+    </div>
+
+
+🌟 Introduction
+~~~~~~~~~~~~~~~~
+In this scenario, our agent illustrates the iterative process of hypothesis generation, knowledge construction, and decision-making. 
+
+It highlights how financial factors evolve through continuous feedback and refinement. 
+
+Here's an enhanced outline of the steps:
+
+**Step 1 : Hypothesis Generation 🔍**
+
+- Generate and propose initial hypotheses based on previous experiment analysis and domain expertise, with thorough reasoning and financial justification.
+
+**Step 2 : Factor Creation ✨**
+
+- Based on the hypothesis, divide the tasks.
+- Each task involves developing, defining, and implementing a new financial factor, including its name, description, formulation, and variables.
+
+**Step 3 : Factor Implementation 👨‍💻**
+
+- Implement the factor code based on the description, evolving it as a developer would.
+- Quantitatively validate the newly created factors.
+
+**Step 4 : Backtesting with Qlib 📉**
+
+- Integrate the full dataset into the factor implementation code and prepare the factor library.
+- Conduct backtesting using the Alpha158 plus newly developed factors and LGBModel in Qlib to evaluate the new factors' effectiveness and performance.
+
++----------------+------------+----------------+----------------------------------------------------+
+| Dataset        | Model      | Factors        | Data Split                                         |
++================+============+================+====================================================+
+| CSI300         | LGBModel   | Alpha158 Plus  | +-----------+--------------------------+           |
+|                |            |                | | Train     | 2008-01-01 to 2014-12-31 |           |
+|                |            |                | +-----------+--------------------------+           |
+|                |            |                | | Valid     | 2015-01-01 to 2016-12-31 |           |
+|                |            |                | +-----------+--------------------------+           |
+|                |            |                | | Test      | 2017-01-01 to 2020-08-01 |           |
+|                |            |                | +-----------+--------------------------+           |
++----------------+------------+----------------+----------------------------------------------------+
+
+
+**Step 5 : Feedback Analysis 🔍**
+
+- Analyze backtest results to assess performance.
+- Incorporate feedback to refine hypotheses and improve the model.
+
+**Step 6 :Hypothesis Refinement ♻️**
+
+- Refine hypotheses based on feedback from backtesting.
+- Repeat the process to continuously improve the model.
+
+⚡ Quick Start
+~~~~~~~~~~~~~~~~~
+
+Please refer to the installation part in :doc:`../installation_and_configuration` to prepare your system dependency.
+
+You can try our demo by running the following command:
+
+- 🐍 Create a Conda Environment
+
+  - Create a new conda environment with Python (3.10 and 3.11 are well tested in our CI):
+
+    .. code-block:: sh
+
+          conda create -n rdagent python=3.10
+
+  - Activate the environment:
+
+    .. code-block:: sh
+
+        conda activate rdagent
+
+- 📦 Install the RDAgent
+  
+  - You can install the RDAgent package from PyPI:
+
+    .. code-block:: sh
+
+        pip install rdagent
+
+- 🚀 Run the Application
+    
+  - You can directly run the application by using the following command:
+    
+    .. code-block:: sh
+
+        rdagent fin_factor
+
+
+🛠️ Usage of modules
+~~~~~~~~~~~~~~~~~~~~~
+
+.. _Env Config: 
+
+- **Env Config**
+
+The following environment variables can be set in the `.env` file to customize the application's behavior:
+
+.. autopydantic_settings:: rdagent.app.qlib_rd_loop.conf.FactorBasePropSetting
+    :settings-show-field-summary: False
+    :exclude-members: Config
+
+.. autopydantic_settings:: rdagent.components.coder.factor_coder.config.FactorImplementSettings
+    :settings-show-field-summary: False
+    :members: coder_use_cache, data_folder, data_folder_debug, file_based_execution_timeout, select_method, select_threshold, max_loop, knowledge_base_path, new_knowledge_base_path
+    :exclude-members: Config, fail_task_trial_limit, v1_query_former_trace_limit, v1_query_similar_success_limit, v2_query_component_limit, v2_query_error_limit, v2_query_former_trace_limit, v2_error_summary, v2_knowledge_sampler
+    :no-index:

docs/scens/data_copilot_fin.rst

@@ -0,0 +1,164 @@
+.. _data_copilot_fin:
+
+=====================
+Finance Data Copilot
+=====================
+
+
+**🤖 Automated Quantitative Trading & Factors Extraction from Financial Reports**
+---------------------------------------------------------------------------------
+
+📖 Background
+~~~~~~~~~~~~~~
+**Research reports** are treasure troves of insights, often unveiling potential **factors** that can drive successful quantitative trading strategies. 
+Yet, with the sheer volume of reports available, extracting the most valuable insights efficiently becomes a daunting task.
+
+Furthermore, rather than hastily replicating factors from a report, it's essential to delve into the underlying logic of their construction. 
+Does the factor capture the essential market dynamics? How unique is it compared to the factors already in your library?
+
+Therefore, there is an urgent need for a systematic approach to design a framework that can effectively manage this process. 
+And this is where the **Finance Data Copilot** steps in.
+
+
+🎥 `Demo <https://rdagent.azurewebsites.net/report_factor>`_
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. raw:: html
+
+    <div style="display: flex; justify-content: center; align-items: center;">
+      <video width="600" controls>
+        <source src="https://rdagent.azurewebsites.net/media/7b14b2bd3d8771da9cf7eb799b6d96729cec3d35c8d4f68060f3e2fd.mp4" type="video/mp4">
+        Your browser does not support the video tag.
+      </video>
+    </div>
+
+
+🌟 Introduction
+~~~~~~~~~~~~~~~~
+In this scenario, RDAgent demonstrates the process of extracting factors from financial research reports, implementing these factors, and analyzing their performance through Qlib backtesting. 
+This process continually expands and refines the factor library.
+
+Here's an enhanced outline of the steps:
+
+**Step 1 : Hypothesis Generation 🔍**
+
+- Generate and propose initial hypotheses based on insights from financial reports with thorough reasoning and financial justification.
+
+**Step 2 : Factor Creation ✨**
+
+- Based on the hypothesis and financial reports, divide the tasks. 
+- Each task involves developing, defining, and implementing a new financial factor, including its name, description, formulation, and variables.
+
+**Step 3 : Factor Implementation 👨‍💻**
+
+- Implement the factor code based on the description, evolving it as a developer would.
+- Quantitatively validate the newly created factors.
+
+**Step 4 : Backtesting with Qlib 📉**
+
+- Integrate the full dataset into the factor implementation code and prepare the factor library.
+- Conduct backtesting using the Alpha158 plus newly developed factors and LGBModel in Qlib to evaluate the new factors' effectiveness and performance.
+
++----------------+------------+----------------+----------------------------------------------------+
+| Dataset        | Model      | Factors        | Data Split                                         |
++================+============+================+====================================================+
+| CSI300         | LGBModel   | Alpha158 Plus  | +-----------+--------------------------+           |
+|                |            |                | | Train     | 2008-01-01 to 2014-12-31 |           |
+|                |            |                | +-----------+--------------------------+           |
+|                |            |                | | Valid     | 2015-01-01 to 2016-12-31 |           |
+|                |            |                | +-----------+--------------------------+           |
+|                |            |                | | Test      | 2017-01-01 to 2020-08-01 |           |
+|                |            |                | +-----------+--------------------------+           |
++----------------+------------+----------------+----------------------------------------------------+
+
+**Step 5 : Feedback Analysis 🔍**
+
+- Analyze backtest results to assess performance.
+- Incorporate feedback to refine hypotheses and improve the model.
+
+**Step 6 :Hypothesis Refinement ♻️**
+
+- Refine hypotheses based on feedback from backtesting.
+- Repeat the process to continuously improve the model.
+
+⚡ Quick Start
+~~~~~~~~~~~~~~~~~
+
+Please refer to the installation part in :doc:`../installation_and_configuration` to prepare your system dependency.
+
+You can try our demo by running the following command:
+
+- 🐍 Create a Conda Environment
+    
+  - Create a new conda environment with Python (3.10 and 3.11 are well tested in our CI):
+    
+    .. code-block:: sh
+    
+        conda create -n rdagent python=3.10
+
+  - Activate the environment:
+
+    .. code-block:: sh
+
+        conda activate rdagent
+
+- 📦 Install the RDAgent
+  
+  - You can install the RDAgent package from PyPI:
+
+    .. code-block:: sh
+
+        pip install rdagent
+
+- 🚀 Run the Application
+    
+  - Download the financial reports you wish to extract factors from and store them in your preferred folder.
+
+  - Specifically, you can follow this example, or use your own method:
+
+    .. code-block:: sh
+
+        wget https://github.com/SunsetWolf/rdagent_resource/releases/download/reports/all_reports.zip
+        unzip all_reports.zip -d git_ignore_folder/reports
+
+  - Run the application with the following command:
+
+    .. code-block:: sh
+
+        rdagent fin_factor_report --report_folder=git_ignore_folder/reports
+
+  - Alternatively, you can store the paths of the reports in `report_result_json_file_path`. The format should be:
+
+    .. code-block:: json
+
+        [
+            "git_ignore_folder/report/fin_report1.pdf",
+            "git_ignore_folder/report/fin_report2.pdf",
+            "git_ignore_folder/report/fin_report3.pdf"
+        ]
+
+  - Then, run the application using the following command:
+
+    .. code-block:: sh
+
+        rdagent fin_factor_report
+
+🛠️ Usage of modules
+~~~~~~~~~~~~~~~~~~~~~
+
+.. _Env Config: 
+
+- **Env Config**
+
+The following environment variables can be set in the `.env` file to customize the application's behavior:
+
+.. autopydantic_settings:: rdagent.app.qlib_rd_loop.conf.FactorFromReportPropSetting
+    :settings-show-field-summary: False
+    :show-inheritance:
+    :exclude-members: Config
+
+.. autopydantic_settings:: rdagent.components.coder.factor_coder.config.FactorImplementSettings
+    :settings-show-field-summary: False
+    :members: coder_use_cache, data_folder, data_folder_debug, file_based_execution_timeout, select_method, select_threshold, max_loop, knowledge_base_path, new_knowledge_base_path
+    :exclude-members: Config, python_bin, fail_task_trial_limit, v1_query_former_trace_limit, v1_query_similar_success_limit, v2_query_component_limit, v2_query_error_limit, v2_query_former_trace_limit, v2_error_summary, v2_knowledge_sampler
+    :no-index:

docs/scens/model_agent_fin.rst

@@ -0,0 +1,150 @@
+.. _model_agent_fin:
+
+=======================
+Finance Model Agent
+=======================
+
+**🤖 Automated Quantitative Trading & Iterative Model Evolution**
+------------------------------------------------------------------------------------------
+
+📖 Background
+~~~~~~~~~~~~~~
+In the realm of quantitative finance, both factor discovery and model development play crucial roles in driving performance. 
+While much attention is often given to the discovery of new financial factors, the **models** that leverage these factors are equally important. 
+The effectiveness of a quantitative strategy depends not only on the factors used but also on how well these factors are integrated into robust, predictive models.
+
+However, the process of developing and optimizing these models can be labor-intensive and complex, requiring continuous refinement and adaptation to ever-changing market conditions. 
+And this is where the **Finance Model Agent** steps in.
+
+
+🎥 `Demo <https://rdagent.azurewebsites.net/model_loop>`_
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. raw:: html
+
+    <div style="display: flex; justify-content: center; align-items: center;">
+      <video width="600" controls>
+        <source src="https://rdagent.azurewebsites.net/media/d85e8cab1da1cd3501d69ce837452f53a971a24911eae7bfa9237137.mp4" type="video/mp4">
+        Your browser does not support the video tag.
+      </video>
+    </div>
+
+
+🌟 Introduction
+~~~~~~~~~~~~~~~~
+
+In this scenario, our automated system proposes hypothesis, constructs model, implements code, conducts back-testing, and utilizes feedback in a continuous, iterative process.
+
+The goal is to automatically optimize performance metrics within the Qlib library, ultimately discovering the most efficient code through autonomous research and development.
+
+Here's an enhanced outline of the steps:
+
+**Step 1 : Hypothesis Generation 🔍**
+
+- Generate and propose initial hypotheses based on previous experiment analysis and domain expertise, with thorough reasoning and financial justification.
+
+**Step 2 : Model Creation ✨**
+
+- Transform the hypothesis into a task.
+- Develop, define, and implement a quantitative model, including its name, description, and formulation.
+
+**Step 3 : Model Implementation 👨‍💻**
+
+- Implement the model code based on the detailed description.
+- Evolve the model iteratively as a developer would, ensuring accuracy and efficiency.
+
+**Step 4 : Backtesting with Qlib 📉**
+
+- Conduct backtesting using the newly developed model and 20 factors extracted from Alpha158 in Qlib.
+- Evaluate the model's effectiveness and performance.
+
++----------------+------------+------------------------+----------------------------------------------------+
+| Dataset        | Model      | Factors                | Data Split                                         |
++================+============+========================+====================================================+
+| CSI300         | RDAgent-dev| 20 factors (Alpha158)  | +-----------+--------------------------+           |
+|                |            |                        | | Train     | 2008-01-01 to 2014-12-31 |           |
+|                |            |                        | +-----------+--------------------------+           |
+|                |            |                        | | Valid     | 2015-01-01 to 2016-12-31 |           |
+|                |            |                        | +-----------+--------------------------+           |
+|                |            |                        | | Test      | 2017-01-01 to 2020-08-01 |           |
+|                |            |                        | +-----------+--------------------------+           |
++----------------+------------+------------------------+----------------------------------------------------+
+
+**Step 5 : Feedback Analysis 🔍**
+
+- Analyze backtest results to assess performance.
+- Incorporate feedback to refine hypotheses and improve the model.
+
+**Step 6 :Hypothesis Refinement ♻️**
+
+- Refine hypotheses based on feedback from backtesting.
+- Repeat the process to continuously improve the model.
+
+⚡ Quick Start
+~~~~~~~~~~~~~~~~~
+
+Please refer to the installation part in :doc:`../installation_and_configuration` to prepare your system dependency.
+
+You can try our demo by running the following command:
+
+- 🐍 Create a Conda Environment
+
+  - Create a new conda environment with Python (3.10 and 3.11 are well tested in our CI):
+
+    .. code-block:: sh
+    
+        conda create -n rdagent python=3.10
+
+  - Activate the environment:
+
+    .. code-block:: sh
+
+        conda activate rdagent
+
+- 📦 Install the RDAgent
+    
+  - You can install the RDAgent package from PyPI:
+
+    .. code-block:: sh
+
+        pip install rdagent
+
+- 🚀 Run the Application
+    
+  - You can directly run the application by using the following command:
+    
+    .. code-block:: sh
+
+        rdagent fin_model
+
+🛠️ Usage of modules
+~~~~~~~~~~~~~~~~~~~~~
+
+.. _Env Config: 
+
+- **Env Config**
+
+The following environment variables can be set in the `.env` file to customize the application's behavior:
+
+.. autopydantic_settings:: rdagent.app.qlib_rd_loop.conf.ModelBasePropSetting
+    :settings-show-field-summary: False
+    :exclude-members: Config
+
+- **Qlib Config**
+    - The `config.yaml` file located in the `model_template` folder contains the relevant configurations for running the developed model in Qlib. The default settings include key information such as:
+        - **market**: Specifies the market, which is set to `csi300`.
+        - **fields_group**: Defines the fields group, with the value `feature`.
+        - **col_list**: A list of columns used, including various indicators such as `RESI5`, `WVMA5`, `RSQR5`, and others.
+        - **start_time**: The start date for the data, set to `2008-01-01`.
+        - **end_time**: The end date for the data, set to `2020-08-01`.
+        - **fit_start_time**: The start date for fitting the model, set to `2008-01-01`.
+        - **fit_end_time**: The end date for fitting the model, set to `2014-12-31`.
+
+    - The default hyperparameters used in the configuration are as follows:
+        - **n_epochs**: The number of epochs, set to `100`.
+        - **lr**: The learning rate, set to `1e-3`.
+        - **early_stop**: The early stopping criterion, set to `10`.
+        - **batch_size**: The batch size, set to `2000`.
+        - **metric**: The evaluation metric, set to `loss`.
+        - **loss**: The loss function, set to `mse`.
+        - **n_jobs**: The number of parallel jobs, set to `20`.

docs/scens/model_agent_med.rst

@@ -0,0 +1,128 @@
+.. _model_agent_med:
+
+=======================
+Medical Model Agent
+=======================
+
+**🤖 Automated Medical Predtion Model Evolution**
+------------------------------------------------------------------------------------------
+
+📖 Background
+~~~~~~~~~~~~~~
+In this scenario, we consider the problem of risk prediction from patients' ICU monitoring data. We use the a public EHR dataset - MIMIC-III and extract a binary classification task for evaluating the framework.
+In this task, we aim at predicting the whether the patients will suffer from Acute Respiratory Failure (ARF) based their first 12 hours ICU monitoring data. 
+
+🎥 `Demo <https://rdagent.azurewebsites.net/dmm>`_
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. raw:: html
+
+    <div style="display: flex; justify-content: center; align-items: center;">
+      <video width="600" controls>
+        <source src="https://rdagent.azurewebsites.net/media/1653542fc1b9fa14a306c35c1b1fc48288f980793f38abe82b023af9.mp4" type="video/mp4">
+        Your browser does not support the video tag.
+      </video>
+    </div>
+
+
+🌟 Introduction
+~~~~~~~~~~~~~~~~
+
+In this scenario, our automated system proposes hypothesis, constructs model, implements code, receives back-testing, and uses feedbacks. 
+Hypothesis is iterated in this continuous process. 
+The system aims to automatically optimise performance metrics of medical prediction thereby finding the optimised code through autonomous research and development.
+
+Here's an enhanced outline of the steps:
+
+**Step 1 : Hypothesis Generation 🔍**
+
+- Generate and propose initial hypotheses based on previous experiment analysis and domain expertise, with thorough reasoning and justification.
+
+**Step 2 : Model Creation ✨**
+
+- Transform the hypothesis into a model.
+- Develop, define, and implement a machine learning model, including its name, description, and formulation.
+
+**Step 3 : Model Implementation 👨‍💻**
+
+- Implement the model code based on the detailed description.
+- Evolve the model iteratively as a developer would, ensuring accuracy and efficiency.
+
+**Step 4 : Backtesting with MIMIC-III 📉**
+
+- Conduct backtesting using the newly developed model on the extracted task from MIMIC-III.
+- Evaluate the model's effectiveness and performance in terms of AUROC score.
+
+**Step 5 : Feedback Analysis 🔍**
+
+- Analyze backtest results to assess performance.
+- Incorporate feedback to refine hypotheses and improve the model.
+
+**Step 6 :Hypothesis Refinement ♻️**
+
+- Refine hypotheses based on feedback from backtesting.
+- Repeat the process to continuously improve the model.
+
+⚡ Quick Start
+~~~~~~~~~~~~~~~~~
+
+Please refer to the installation part in :doc:`../installation_and_configuration` to prepare your system dependency.
+
+You can try our demo by running the following command:
+
+- 🐍 Create a Conda Environment
+  
+  - Create a new conda environment with Python (3.10 and 3.11 are well tested in our CI):
+
+    .. code-block:: sh
+    
+        conda create -n rdagent python=3.10
+
+  - Activate the environment:
+
+    .. code-block:: sh
+
+        conda activate rdagent
+
+- 📦 Install the RDAgent
+    
+  - You can install the RDAgent package from PyPI:
+
+    .. code-block:: sh
+
+        pip install rdagent
+
+- 📦 Request PhysioNet Account
+    
+  - Apply for an account at `PhysioNet <https://physionet.org/>`_.
+  - Request access to FIDDLE preprocessed data: `FIDDLE Dataset <https://physionet.org/content/mimic-eicu-fiddle-feature/1.0.0/>`_.
+  - Place your username and password in `.env`.
+
+    .. code-block:: bash
+
+        cat << EOF  >> .env
+        DM_USERNAME=<your_username>
+        DM_PASSWORD=<your_password>
+        EOF
+
+
+- 🚀 Run the Application
+    
+  - You can directly run the application by using the following command:
+    
+    .. code-block:: sh
+
+        rdagent med_model
+
+🛠️ Usage of modules
+~~~~~~~~~~~~~~~~~~~~~
+
+.. _Env Config: 
+
+- **Env Config**
+
+The following environment variables can be set in the `.env` file to customize the application's behavior:
+
+.. autopydantic_settings:: rdagent.app.data_mining.conf.MedBasePropSetting
+    :settings-show-field-summary: False
+    :exclude-members: Config

docs/scens/model_copilot_general.rst

@@ -0,0 +1,99 @@
+.. _model_copilot_general:
+
+======================
+General Model Copilot
+======================
+
+**🤖 Automated Model Research & Development Co-Pilot**
+--------------------------------------------------------
+
+📖 Background
+~~~~~~~~~~~~~~
+In the fast-paced field of artificial intelligence, the number of academic papers published each year is skyrocketing. 
+These papers introduce new models, techniques, and approaches that can significantly advance the state of the art. 
+However, reproducing and implementing these models can be a daunting task, requiring substantial time and expertise. 
+Researchers often face challenges in extracting the essential details from these papers and converting them into functional code.
+And this is where the **General Model Copilot** steps in.
+
+🎥 `Demo <https://rdagent.azurewebsites.net/report_model>`_
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. raw:: html
+
+    <div style="display: flex; justify-content: center; align-items: center;">
+      <video width="600" controls>
+        <source src="https://rdagent.azurewebsites.net/media/b35f904765b05099b0fcddbebe041a04f4d7bde239657e5fc24bf0cc.mp4" type="video/mp4">
+        Your browser does not support the video tag.
+      </video>
+    </div>
+
+🌟 Introduction
+~~~~~~~~~~~~~~~~
+In this scenario, our automated system proposes hypotheses, constructs models, implements code, performs back-testing, and uses feedback to iterate continuously. The system aims to automatically optimize performance metrics from the Qlib library, finding the best code through autonomous research and development.
+
+Model R&D CoPilot Scenario
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+**Overview**
+
+This demo automates the extraction and iterative development of models from academic papers, ensuring functionality and correctness. This scenario automates the development of PyTorch models by reading academic papers or other sources. It supports various data types, including tabular, time-series, and graph data. The primary workflow involves two main components: the Reader and the Coder.
+
+**Workflow Components**
+
+1. **Reader**
+   - Parses and extracts relevant model information from academic papers or sources, including architectures, parameters, and implementation details.
+   - Uses Large Language Models to convert content into a structured format for the Coder.
+
+2. **Evolving Coder**
+   - Translates structured information from the Reader into executable PyTorch code.
+   - Utilizes an evolving coding mechanism to ensure correct tensor shapes, verified with sample input tensors.
+   - Iteratively refines the code to align with source material specifications.
+
+**Supported Data Types**
+
+- **Tabular Data:** Structured data with rows and columns, such as spreadsheets or databases.
+- **Time-Series Data:** Sequential data points indexed in time order, useful for forecasting and temporal pattern recognition.
+- **Graph Data:** Data structured as nodes and edges, suitable for network analysis and relational tasks.
+
+⚡ Quick Start
+~~~~~~~~~~~~~~~~~
+
+Please refer to the installation part in :doc:`../installation_and_configuration` to prepare your system dependency.
+
+You can try our demo by running the following command:
+
+- 🐍 Create a Conda Environment
+  
+  - Create a new conda environment with Python (3.10 and 3.11 are well tested in our CI):
+
+    .. code-block:: sh
+    
+        conda create -n rdagent python=3.10
+
+  - Activate the environment:
+
+    .. code-block:: sh
+
+        conda activate rdagent
+
+- 📦 Install the RDAgent
+    
+  - You can install the RDAgent package from PyPI:
+
+    .. code-block:: sh
+
+        pip install rdagent
+
+
+- 🚀 Run the Application
+    
+  - Prepare relevant files (in pdf format) by uploading papers to the directory below and copy the path as report_file_path.
+      
+    .. code-block:: sh
+
+        rdagent/scenarios/general_model
+    
+  - Run the following command in your terminal within the same virtual environment:
+  
+    .. code-block:: sh
+
+        rdagent general_model --report_file_path=<path_to_pdf_file>

docs/ui.rst

@@ -0,0 +1,48 @@
+==============
+User Interface
+==============
+
+
+Introduction
+============
+
+RD-Agent will generate some logs during the R&D process. These logs are very useful for debugging and understanding the R&D process. However, just viewing the terminal log is not intuitive enough. RD-Agent provides a web app as UI to visualize the R&D process. You can easily view the R&D process and understand the R&D process better.
+
+A Quick Demo
+============
+
+Start Web App
+-------------
+
+In `RD-Agent/` folder, run:
+
+.. code-block:: bash
+
+    rdagent ui --port <port> --log_dir <log_dir like "log/"> [--debug]
+
+This will start a web app on `http://localhost:<port>`.
+
+**NOTE**: The log_dir parameter is not required. You can manually enter the log_path in the web app. If you set the log_dir parameter, you can easily select a different log_path in the web app.
+
+--debug is optional, it will show a "Single Step Run" button in sidebar and saved objects info in the web app.
+
+Use Web App
+-----------
+
+1. Open the sidebar.
+
+.. TODO: update these
+
+2. Select the scenario you want to show. There are some pre-defined scenarios:
+    - Qlib Model
+    - Qlib Factor
+    - Data Mining
+    - Model from Paper
+
+3. Click the `Config⚙️` button and input the log path (if you set the log_dir parameter, you can select a log_path in the dropdown list).
+
+4. Click the buttons below Config⚙️ to show the scenario execution process. Buttons are:
+    - All Loops: Show complete scenario execution process.
+    - Next Loop: Show one success **R&D Loop**.
+    - One Evolving: Show one **evolving** step of **development** part.
+    - refresh logs: clear shown logs.

pyproject.toml

@@ -4,7 +4,6 @@ requires = [
   "setuptools",
   "setuptools-scm",
 ]
-root= "rdagent"
 
 [project]
 authors = [
@@ -16,8 +15,6 @@ classifiers = [
   "Operating System :: OS Independent",
   "Programming Language :: Python :: 3.10",
   "Programming Language :: Python :: 3.11",
-  "Programming Language :: Python :: 3.8",
-  "Programming Language :: Python :: 3.9",
 ]
 description = "Research & Development Agent"
 dynamic = [
@@ -27,13 +24,15 @@ dynamic = [
 ]
 keywords = [
   "Autonomous Agents",
-  "Research and Development",
   "Large Language Models",
+  "Research and Development",
 ]
 name = "rdagent"
 readme = "README.md"
-requires-python = ">=3.8"
+requires-python = ">=3.10"
 
+[project.scripts]
+rdagent = "rdagent.app.cli:app"
 
 [project.urls]
 homepage = "https://github.com/microsoft/RD-Agent/"
@@ -52,13 +51,13 @@ color_output = true
 profile = "black"
 
 [tool.mypy]
-explicit_package_bases = true
 check_untyped_defs = true
 disallow_any_unimported = true
 disallow_untyped_defs = true
 enable_error_code = [
   "ignore-without-code",
 ]
+explicit_package_bases = true
 warn_return_any = true
 warn_unused_ignores = true
 
@@ -111,6 +110,8 @@ package = {file = ["requirements/package.txt"]}
 test = {file = ["requirements/test.txt"]}
 
 [tool.setuptools_scm]
+local_scheme = "no-local-version"
+version_scheme = "guess-next-dev"
 
 [tool.tomlsort]
 all = true

rdagent/app/CI/README.md

@@ -0,0 +1,38 @@
+# CI 检查
+
+`.github/workflows/ci.yml`配置了提交时自动运行`Makefile`: 91~103行的命令，可以在这调整执行的命令
+
+在`.env`中设置`USE_CHAT_CACHE=True`可以让第二次修复快一些
+
+# Rules
+
+`pyproject.toml`中配置全局屏蔽的规则
+- ruff: `[tool.ruff.lint].ignore`
+- mypy: `[tool.mypy]`
+
+## ruff rules
+ruff rules 比较好修改, 大多可以自动修复
+
+对于一些规则可以在代码中添加注释来局部屏蔽, 例如添加 `# noqa E234,ANN001`
+遇到的不好修改的规则:
+- 捕获异常时应该处理每一种异常，不应该统一当作`Exception`处理
+- `subprogress()` 调用命令应该先判断命令是否安全
+- ...
+
+规则列表: [ruff rules](https://docs.astral.sh/ruff/rules/)
+
+## mypy rules
+
+Mypy检查Python中类型标注, 常遇到需要修改结构/同时修改其他文件的情况, 自动修复效果不好
+
+局部屏蔽: `# type: ignore`
+
+规则列表: [mypy rules](https://mypy.readthedocs.io/en/stable/error_code_list.html)
+
+# Optimization (Maybe)
+
+- 添加指定文件夹检查的功能
+- 增加一个修改选项: 调用`vim`, 用户直接修改此部分代码
+- 显示时把`Original Code`部分去掉, 直接在输出的表示修改的diff部分用`^^^^^^`在代码行下标注出错误位置，这样能更直观地观察错误修复情况
+- 当前为线性执行完所有修复后交给用户检查, 可修改成 后台多线程 / 进程处理修复的任务, 终端实时展示处理完的修复让用户检查
+- ...

rdagent/app/CI/run.py

@@ -13,17 +13,6 @@ from pathlib import Path
 from typing import Any, Literal
 
 import tree_sitter_python
-from rdagent.core.evolving_framework import (
-    Evaluator,
-    EvoAgent,
-    EvolvableSubjects,
-    EvolvingStrategy,
-    EvoStep,
-    Feedback,
-    Knowledge,
-)
-from rdagent.core.prompts import Prompts
-from rdagent.oai.llm_utils import APIBackend
 from rich import print
 from rich.panel import Panel
 from rich.progress import Progress, SpinnerColumn, TimeElapsedColumn
@@ -34,6 +23,18 @@ from rich.table import Table
 from rich.text import Text
 from tree_sitter import Language, Node, Parser
 
+from rdagent.core.evaluation import Evaluator
+from rdagent.core.evolving_agent import EvoAgent
+from rdagent.core.evolving_framework import (
+    EvolvableSubjects,
+    EvolvingStrategy,
+    EvoStep,
+    Feedback,
+    Knowledge,
+)
+from rdagent.core.prompts import Prompts
+from rdagent.oai.llm_utils import APIBackend
+
 py_parser = Parser(Language(tree_sitter_python.language()))
 CI_prompts = Prompts(file_path=Path(__file__).parent / "prompts.yaml")
 
@@ -303,7 +304,7 @@ class RuffEvaluator(Evaluator):
 
         return RuffRule(**json.loads(out))
 
-    def evaluate(self, evo: Repo, **kwargs: Any) -> CIFeedback:  # noqa: ARG002
+    def evaluate(self, evo: Repo, **kwargs: dict) -> CIFeedback:
         """Simply run ruff to get the feedbacks."""
         try:
             out = subprocess.check_output(
@@ -355,14 +356,13 @@ class RuffEvaluator(Evaluator):
 
 
 class MypyEvaluator(Evaluator):
-
     def __init__(self, command: str | None = None) -> None:
         if command is None:
             self.command = "mypy . --pretty --no-error-summary --show-column-numbers"
         else:
             self.command = command
 
-    def evaluate(self, evo: Repo, **kwargs: Any) -> CIFeedback:  # noqa: ARG002
+    def evaluate(self, evo: Repo, **kwargs: dict) -> CIFeedback:
         try:
             out = subprocess.check_output(
                 shlex.split(self.command),  # noqa: S603
@@ -411,12 +411,10 @@ class MypyEvaluator(Evaluator):
 
 
 class MultiEvaluator(Evaluator):
-
     def __init__(self, *evaluators: Evaluator) -> None:
         self.evaluators = evaluators
 
-    def evaluate(self, evo: Repo, **kwargs: Any) -> CIFeedback:
-
+    def evaluate(self, evo: Repo, **kwargs: dict) -> CIFeedback:
         all_errors = defaultdict(list)
         for evaluator in self.evaluators:
             feedback: CIFeedback = evaluator.evaluate(evo, **kwargs)
@@ -435,10 +433,9 @@ class CIEvoStr(EvolvingStrategy):
         self,
         evo: Repo,
         evolving_trace: list[EvoStep] | None = None,
-        knowledge_l: list[Knowledge] | None = None,  # noqa: ARG002
-        **kwargs: Any,  # noqa: ARG002
+        knowledge_l: list[Knowledge] | None = None,
+        **kwargs: dict,
     ) -> Repo:
-
         @dataclass
         class CodeFixGroup:
             start_line: int
@@ -552,7 +549,7 @@ class CIEvoStr(EvolvingStrategy):
                         style="bright_blue",
                         align="left",
                         characters=".",
-                    )
+                    ),
                 )
 
                 file = evo.files[evo.project_path / Path(file_path)]
@@ -633,12 +630,16 @@ class CIEvoStr(EvolvingStrategy):
                         for i in diff:
                             if i.startswith("+"):
                                 table.add_row(
-                                    "", Text(str(diff_new_lineno), style="green bold"), Text(i, style="green")
+                                    "",
+                                    Text(str(diff_new_lineno), style="green bold"),
+                                    Text(i, style="green"),
                                 )
                                 diff_new_lineno += 1
                             elif i.startswith("-"):
                                 table.add_row(
-                                    Text(str(diff_original_lineno), style="red bold"), "", Text(i, style="red")
+                                    Text(str(diff_original_lineno), style="red bold"),
+                                    "",
+                                    Text(i, style="red"),
                                 )
                                 diff_original_lineno += 1
                             elif i.startswith("?"):
@@ -658,7 +659,7 @@ class CIEvoStr(EvolvingStrategy):
                         operation = Prompt.ask(
                             "Input your operation [ [red]([bold]s[/bold])kip[/red] / "
                             "[green]([bold]a[/bold])pply[/green] / "
-                            "[yellow]manual instruction[/yellow] ]"
+                            "[yellow]manual instruction[/yellow] ]",
                         )
                         print()
                         if operation in ("s", "skip"):
@@ -692,6 +693,22 @@ class CIEvoStr(EvolvingStrategy):
         return evo
 
 
+class CIEvoAgent(EvoAgent):
+    def __init__(self, evolving_strategy: CIEvoStr) -> None:
+        super().__init__(max_loop=1, evolving_strategy=evolving_strategy)
+        self.evolving_trace = []
+
+    def multistep_evolve(self, evo: Repo, eva: Evaluator) -> Repo:
+        evo = self.evolving_strategy.evolve(
+            evo=evo,
+            evolving_trace=self.evolving_trace,
+        )
+
+        self.evolving_trace.append(EvoStep(evo, feedback=eva.evaluate(evo)))
+
+        return evo
+
+
 DIR = None
 while DIR is None or not DIR.exists():
     DIR = Prompt.ask("Please input the [cyan]project directory[/cyan]")
@@ -710,12 +727,11 @@ repo = Repo(DIR, excludes=excludes)
 # evaluator = MultiEvaluator(MypyEvaluator(), RuffEvaluator())
 evaluator = RuffEvaluator()
 estr = CIEvoStr()
-rag = None  # RAG is not enable firstly.
-ea = EvoAgent(estr, rag=rag)
-ea.step_evolving(repo, evaluator)
+ea = CIEvoAgent(estr)
+ea.multistep_evolve(repo, evaluator)
 while True:
     print(Rule(f"Round {len(ea.evolving_trace)} repair", style="blue"))
-    repo: Repo = ea.step_evolving(repo, evaluator)
+    repo: Repo = ea.multistep_evolve(repo, evaluator)
 
     fix_records = repo.fix_records
     filename = f"{DIR.name}_{start_timestamp}_round_{len(ea.evolving_trace)}_fix_records.json"

rdagent/app/benchmark/factor/analysis.py

@@ -0,0 +1,210 @@
+import json
+import pickle
+from pathlib import Path
+
+import fire
+import matplotlib.pyplot as plt
+import numpy as np
+import pandas as pd
+import seaborn as sns
+
+from rdagent.components.benchmark.conf import BenchmarkSettings
+from rdagent.components.benchmark.eval_method import FactorImplementEval
+
+
+class BenchmarkAnalyzer:
+    def __init__(self, settings):
+        self.settings = settings
+        self.index_map = self.load_index_map()
+
+    def load_index_map(self):
+        index_map = {}
+        with open(self.settings.bench_data_path, "r") as file:
+            factor_dict = json.load(file)
+        for factor_name, data in factor_dict.items():
+            index_map[factor_name] = (factor_name, data["Category"], data["Difficulty"])
+        return index_map
+
+    def load_data(self, file_path):
+        file_path = Path(file_path)
+        if not (file_path.is_file() and file_path.suffix == ".pkl"):
+            raise ValueError("Invalid file path")
+
+        with file_path.open("rb") as f:
+            res = pickle.load(f)
+
+        return res
+
+    def process_results(self, results):
+        final_res = {}
+        for experiment, path in results.items():
+            data = self.load_data(path)
+            summarized_data = FactorImplementEval.summarize_res(data)
+            processed_data = self.analyze_data(summarized_data)
+            final_res[experiment] = processed_data.iloc[-1, :]
+        return final_res
+
+    def reformat_index(self, display_df):
+        """
+        reform the results from
+
+        .. code-block:: python
+
+                              success rate
+            High_Beta_Factor           0.2
+
+        to
+
+        .. code-block:: python
+
+                                                    success rate
+            Category Difficulty Factor
+            量价       Hard       High_Beta_Factor           0.2
+
+        """
+        new_idx = []
+        display_df = display_df[display_df.index.isin(self.index_map.keys())]
+        for idx in display_df.index:
+            new_idx.append(self.index_map[idx])
+
+        display_df.index = pd.MultiIndex.from_tuples(
+            new_idx,
+            names=["Factor", "Category", "Difficulty"],
+        )
+        display_df = display_df.swaplevel(0, 2).swaplevel(0, 1).sort_index(axis=0)
+
+        return display_df.sort_index(
+            key=lambda x: [{"Easy": 0, "Medium": 1, "Hard": 2, "New Discovery": 3}.get(i, i) for i in x]
+        )
+
+    def result_all_key_order(self, x):
+        order_v = []
+        for i in x:
+            order_v.append(
+                {
+                    "avg. Run successful rate": 0,
+                    "avg. Format successful rate": 1,
+                    "avg. Correlation (value only)": 2,
+                    "max. Correlation": 3,
+                    "max. accuracy": 4,
+                    "avg. accuracy": 5,
+                }.get(i, i),
+            )
+        return order_v
+
+    def analyze_data(self, sum_df):
+        index = [
+            "FactorSingleColumnEvaluator",
+            "FactorRowCountEvaluator",
+            "FactorIndexEvaluator",
+            "FactorEqualValueRatioEvaluator",
+            "FactorCorrelationEvaluator",
+            "run factor error",
+        ]
+        sum_df = sum_df.reindex(index, axis=0)
+        sum_df_clean = sum_df.T.groupby(level=0).apply(lambda x: x.reset_index(drop=True))
+
+        run_error = sum_df_clean["run factor error"].unstack().T.fillna(False).astype(bool)
+        succ_rate = ~run_error
+        succ_rate = succ_rate.mean(axis=0).to_frame("success rate")
+
+        succ_rate_f = self.reformat_index(succ_rate)
+
+        # if it rasis Error when running the evaluator, we will get NaN
+        # Running failures are reguarded to zero score.
+        format_issue = sum_df_clean[["FactorRowCountEvaluator", "FactorIndexEvaluator"]].apply(
+            lambda x: np.mean(x.fillna(0.0)), axis=1
+        )
+        format_succ_rate = format_issue.unstack().T.mean(axis=0).to_frame("success rate")
+        format_succ_rate_f = self.reformat_index(format_succ_rate)
+
+        corr = sum_df_clean["FactorCorrelationEvaluator"].fillna(0.0)
+        corr = corr.unstack().T.mean(axis=0).to_frame("corr(only success)")
+        corr_res = self.reformat_index(corr)
+        corr_max = sum_df_clean["FactorCorrelationEvaluator"]
+
+        corr_max = corr_max.unstack().T.max(axis=0).to_frame("corr(only success)")
+        corr_max_res = self.reformat_index(corr_max)
+
+        value_max = sum_df_clean["FactorEqualValueRatioEvaluator"]
+        value_max = value_max.unstack().T.max(axis=0).to_frame("max_value")
+        value_max_res = self.reformat_index(value_max)
+
+        value_avg = (
+            (sum_df_clean["FactorEqualValueRatioEvaluator"] * format_issue)
+            .unstack()
+            .T.mean(axis=0)
+            .to_frame("avg_value")
+        )
+        value_avg_res = self.reformat_index(value_avg)
+
+        result_all = pd.concat(
+            {
+                "avg. Correlation (value only)": corr_res.iloc[:, 0],
+                "avg. Format successful rate": format_succ_rate_f.iloc[:, 0],
+                "avg. Run successful rate": succ_rate_f.iloc[:, 0],
+                "max. Correlation": corr_max_res.iloc[:, 0],
+                "max. accuracy": value_max_res.iloc[:, 0],
+                "avg. accuracy": value_avg_res.iloc[:, 0],
+            },
+            axis=1,
+        )
+
+        df = result_all.sort_index(axis=1, key=self.result_all_key_order)
+        print(df)
+
+        # Calculate the mean of each column
+        mean_values = df.fillna(0.0).mean()
+        mean_df = pd.DataFrame(mean_values).T
+
+        # Assign the MultiIndex to the DataFrame
+        mean_df.index = pd.MultiIndex.from_tuples([("-", "-", "Average")], names=["Factor", "Category", "Difficulty"])
+
+        # Append the mean values to the end of the dataframe
+        df_w_mean = pd.concat([df, mean_df]).astype("float")
+
+        return df_w_mean
+
+
+class Plotter:
+    @staticmethod
+    def change_fs(font_size):
+        plt.rc("font", size=font_size)
+        plt.rc("axes", titlesize=font_size)
+        plt.rc("axes", labelsize=font_size)
+        plt.rc("xtick", labelsize=font_size)
+        plt.rc("ytick", labelsize=font_size)
+        plt.rc("legend", fontsize=font_size)
+        plt.rc("figure", titlesize=font_size)
+
+    @staticmethod
+    def plot_data(data, file_name, title):
+        plt.figure(figsize=(10, 6))
+        sns.barplot(x="index", y="b", hue="a", data=data)
+        plt.xlabel("Method")
+        plt.ylabel("Value")
+        plt.title(title)
+        plt.savefig(file_name)
+
+
+def main(
+    path="git_ignore_folder/eval_results/res_promptV220240724-060037.pkl",
+    round=1,
+    title="Comparison of Different Methods",
+):
+    settings = BenchmarkSettings()
+    benchmark = BenchmarkAnalyzer(settings)
+    results = {
+        f"{round} round experiment": path,
+    }
+    final_results = benchmark.process_results(results)
+    final_results_df = pd.DataFrame(final_results)
+
+    Plotter.change_fs(20)
+    plot_data = final_results_df.drop(["max. accuracy", "avg. accuracy"], axis=0).T
+    plot_data = plot_data.reset_index().melt("index", var_name="a", value_name="b")
+    Plotter.plot_data(plot_data, "./comparison_plot.png", title)
+
+
+if __name__ == "__main__":
+    fire.Fire(main)

rdagent/app/benchmark/factor/eval.py

@@ -0,0 +1,42 @@
+import os
+import pickle
+import time
+from pathlib import Path
+from pprint import pprint
+
+from rdagent.app.qlib_rd_loop.conf import FACTOR_PROP_SETTING
+from rdagent.components.benchmark.conf import BenchmarkSettings
+from rdagent.components.benchmark.eval_method import FactorImplementEval
+from rdagent.core.scenario import Scenario
+from rdagent.core.utils import import_class
+from rdagent.log import rdagent_logger as logger
+from rdagent.scenarios.qlib.experiment.factor_experiment import QlibFactorScenario
+from rdagent.scenarios.qlib.factor_experiment_loader.json_loader import (
+    FactorTestCaseLoaderFromJsonFile,
+)
+
+if __name__ == "__main__":
+    # 1.read the settings
+    bs = BenchmarkSettings()
+
+    # 2.read and prepare the eval_data
+    test_cases = FactorTestCaseLoaderFromJsonFile().load(bs.bench_data_path)
+
+    # 3.declare the method to be tested and pass the arguments.
+
+    scen: Scenario = import_class(FACTOR_PROP_SETTING.scen)()
+    generate_method = import_class(bs.bench_method_cls)(scen=scen)
+    # 4.declare the eval method and pass the arguments.
+    eval_method = FactorImplementEval(
+        method=generate_method,
+        test_cases=test_cases,
+        scen=scen,
+        catch_eval_except=True,
+        test_round=bs.bench_test_round,
+    )
+
+    # 5.run the eval
+    res = eval_method.eval()
+
+    # 6.save the result
+    logger.log_object(res)

rdagent/app/benchmark/model/README.md

@@ -0,0 +1,30 @@
+# Tasks
+
+## Task Extraction
+From paper to task.
+```bash
+# python rdagent/app/model_implementation/task_extraction.py
+# It may based on rdagent/document_reader/document_reader.py
+python rdagent/components/task_implementation/model_implementation/task_extraction.py ./PaperImpBench/raw_paper/
+```
+
+## Complete workflow
+From paper to implementation
+``` bash
+# Similar to
+# rdagent/app/factor_extraction_and_implementation/factor_extract_and_implement.py
+```
+
+## Paper benchmark
+```bash
+# TODO: it does not work well now.
+python rdagent/app/model_implementation/eval.py
+```
+
+TODO:
+- Create reasonable benchmark
+  - with uniform input
+  - manually create task
+- Create reasonable evaluation metrics
+
+## Evolving

rdagent/app/benchmark/model/eval.py

@@ -0,0 +1,42 @@
+from pathlib import Path
+
+from rdagent.components.coder.model_coder.CoSTEER import ModelCoSTEER
+from rdagent.components.loader.task_loader import ModelTaskLoaderJson, ModelWsLoader
+from rdagent.scenarios.qlib.experiment.model_experiment import (
+    QlibModelExperiment,
+    QlibModelScenario,
+)
+
+if __name__ == "__main__":
+    DIRNAME = Path(__file__).absolute().resolve().parent
+
+    from rdagent.components.coder.model_coder.benchmark.eval import ModelImpValEval
+    from rdagent.components.coder.model_coder.one_shot import ModelCodeWriter
+
+    bench_folder = DIRNAME.parent.parent / "components" / "coder" / "model_coder" / "benchmark"
+    mtl = ModelTaskLoaderJson(str(bench_folder / "model_dict.json"))
+
+    task_l = mtl.load()
+
+    task_l = [t for t in task_l if t.name == "A-DGN"]  # FIXME: other models does not work well
+
+    model_experiment = QlibModelExperiment(sub_tasks=task_l)
+    # mtg = ModelCodeWriter(scen=QlibModelScenario())
+    mtg = ModelCoSTEER(scen=QlibModelScenario())
+
+    model_experiment = mtg.develop(model_experiment)
+
+    # TODO: Align it with the benchmark framework after @wenjun's refine the evaluation part.
+    # Currently, we just handcraft a workflow for fast evaluation.
+
+    mil = ModelWsLoader(bench_folder / "gt_code")
+
+    mie = ModelImpValEval()
+    # Evaluation:
+    eval_l = []
+    for impl in model_experiment.sub_workspace_list:
+        print(impl.target_task)
+        gt_impl = mil.load(impl.target_task)
+        eval_l.append(mie.evaluate(gt_impl, impl))
+
+    print(eval_l)

rdagent/app/cli.py

@@ -0,0 +1,57 @@
+"""
+CLI entrance for all rdagent application.
+
+This will 
+- make rdagent a nice entry and
+- autoamtically load dotenv
+"""
+from dotenv import load_dotenv
+
+load_dotenv(".env")
+# 1) Make sure it is at the beginning of the script so that it will load dotenv before initializing BaseSettings.
+# 2) The ".env" argument is necessary to make sure it loads `.env` from the current directory.
+
+import subprocess
+from importlib.resources import path as rpath
+
+import fire
+
+from rdagent.app.data_mining.model import main as med_model
+from rdagent.app.general_model.general_model import (
+    extract_models_and_implement as general_model,
+)
+from rdagent.app.kaggle.loop import main as kaggle_main
+from rdagent.app.qlib_rd_loop.factor import main as fin_factor
+from rdagent.app.qlib_rd_loop.factor_from_report import main as fin_factor_report
+from rdagent.app.qlib_rd_loop.model import main as fin_model
+from rdagent.app.utils.info import collect_info
+
+
+def ui(port=80, log_dir="", debug=False):
+    """
+    start web app to show the log traces.
+    """
+    with rpath("rdagent.log.ui", "app.py") as app_path:
+        cmds = ["streamlit", "run", app_path, f"--server.port={port}"]
+        if log_dir or debug:
+            cmds.append("--")
+        if log_dir:
+            cmds.append(f"--log_dir={log_dir}")
+        if debug:
+            cmds.append("--debug")
+        subprocess.run(cmds)
+
+
+def app():
+    fire.Fire(
+        {
+            "fin_factor": fin_factor,
+            "fin_factor_report": fin_factor_report,
+            "fin_model": fin_model,
+            "med_model": med_model,
+            "general_model": general_model,
+            "ui": ui,
+            "collect_info": collect_info,
+            "kaggle": kaggle_main,
+        }
+    )

rdagent/app/data_mining/conf.py

@@ -0,0 +1,49 @@
+from pathlib import Path
+
+from pydantic_settings import BaseSettings
+
+from rdagent.components.workflow.conf import BasePropSetting
+
+
+class MedBasePropSetting(BasePropSetting):
+    class Config:
+        env_prefix = "DM_"
+        """Use `DM_` as prefix for environment variables"""
+        protected_namespaces = ()
+        """Add 'model_' to the protected namespaces"""
+
+    # 1) overriding the default
+    scen: str = "rdagent.scenarios.data_mining.experiment.model_experiment.DMModelScenario"
+    """Scenario class for data mining model"""
+
+    hypothesis_gen: str = "rdagent.scenarios.data_mining.proposal.model_proposal.DMModelHypothesisGen"
+    """Hypothesis generation class"""
+
+    hypothesis2experiment: str = "rdagent.scenarios.data_mining.proposal.model_proposal.DMModelHypothesis2Experiment"
+    """Hypothesis to experiment class"""
+
+    coder: str = "rdagent.scenarios.data_mining.developer.model_coder.DMModelCoSTEER"
+    """Coder class"""
+
+    runner: str = "rdagent.scenarios.data_mining.developer.model_runner.DMModelRunner"
+    """Runner class"""
+
+    summarizer: str = "rdagent.scenarios.data_mining.developer.feedback.DMModelHypothesisExperiment2Feedback"
+    """Summarizer class"""
+
+    evolving_n: int = 10
+    """Number of evolutions"""
+
+    evolving_n: int = 10
+
+    # 2) Extra config for the scenario
+    # physionet account
+    # NOTE: You should apply the account in https://physionet.org/
+    username: str = ""
+    """Physionet account username"""
+
+    password: str = ""
+    """Physionet account password"""
+
+
+MED_PROP_SETTING = MedBasePropSetting()

rdagent/app/data_mining/model.py

@@ -0,0 +1,31 @@
+import fire
+
+from rdagent.app.data_mining.conf import MED_PROP_SETTING
+from rdagent.components.workflow.rd_loop import RDLoop
+from rdagent.core.exception import ModelEmptyError
+
+
+class ModelRDLoop(RDLoop):
+    skip_loop_error = (ModelEmptyError,)
+
+
+def main(path=None, step_n=None):
+    """
+    Auto R&D Evolving loop for models in a medical scenario.
+
+    You can continue running session by
+
+    .. code-block:: python
+
+        dotenv run -- python rdagent/app/data_mining/model.py $LOG_PATH/__session__/1/0_propose  --step_n 1   # `step_n` is a optional paramter
+
+    """
+    if path is None:
+        model_loop = ModelRDLoop(MED_PROP_SETTING)
+    else:
+        model_loop = ModelRDLoop.load(path)
+    model_loop.run(step_n=step_n)
+
+
+if __name__ == "__main__":
+    fire.Fire(main)

rdagent/app/factor_extraction_and_implementation/factor_extract_and_implement.py

@@ -1,16 +0,0 @@
-# %%
-from dotenv import load_dotenv
-from rdagent.factor_implementation.CoSTEER import CoSTEERFG
-from rdagent.factor_implementation.task_loader.pdf_loader import FactorImplementationTaskLoaderFromPDFfiles
-
-assert load_dotenv()
-
-
-def extract_factors_and_implement(report_file_path: str) -> None:
-    factor_tasks = FactorImplementationTaskLoaderFromPDFfiles().load(report_file_path)
-    implementation_result = CoSTEERFG().generate(factor_tasks)
-    return implementation_result
-
-
-if __name__ == "__main__":
-    extract_factors_and_implement("/home/xuyang1/workspace/report.pdf")

rdagent/app/factor_implementation/eval_implement.py

@@ -1,30 +0,0 @@
-from rdagent.core.conf import BenchmarkSettings
-from rdagent.core.utils import import_class
-from rdagent.benchmark.eval_method import FactorImplementEval
-from rdagent.benchmark.data_process import load_eval_data
-
-# 1.read the settings
-bs = BenchmarkSettings()
-
-# 2.read and prepare the eval_data
-test_cases = load_eval_data(bs.bench_version)
-
-# 3.declare the method to be tested and pass the arguments.
-# TODO: Whether it is necessary to define two Eval method classes for two data type?
-method_cls = import_class(bs.bench_method_cls)
-generate_method = method_cls(bs.bench_method_extra_kwargs)
-
-# 4.declare the eval method and pass the arguments.
-eval_method = FactorImplementEval(
-    method=generate_method,
-    test_cases=test_cases,
-    catch_eval_except=True,
-    test_round=bs.bench_test_round,
-)
-
-# 5.run the eval
-eval_method.eval()
-
-# 6.save the result
-eval_method.save(output_path = bs.bench_result_path)
-

rdagent/app/general_model/general_model.py

@@ -0,0 +1,49 @@
+import fire
+
+from rdagent.components.coder.model_coder.task_loader import (
+    ModelExperimentLoaderFromPDFfiles,
+)
+from rdagent.components.document_reader.document_reader import (
+    extract_first_page_screenshot_from_pdf,
+)
+from rdagent.log import rdagent_logger as logger
+from rdagent.scenarios.general_model.scenario import GeneralModelScenario
+from rdagent.scenarios.qlib.developer.model_coder import QlibModelCoSTEER
+
+
+def extract_models_and_implement(report_file_path: str) -> None:
+    """
+    This is a research copilot to automatically implement models from a report file or paper.
+
+    It extracts models from a given PDF report file and implements the necessary operations.
+
+    Parameters:
+    report_file_path (str): The path to the report file. The file must be a PDF file.
+
+    Example URLs of PDF reports:
+    - https://arxiv.org/pdf/2210.09789
+    - https://arxiv.org/pdf/2305.10498
+    - https://arxiv.org/pdf/2110.14446
+    - https://arxiv.org/pdf/2205.12454
+    - https://arxiv.org/pdf/2210.16518
+
+    Returns:
+    None
+    """
+    with logger.tag("init"):
+        scenario = GeneralModelScenario()
+        logger.log_object(scenario, tag="scenario")
+    with logger.tag("r"):
+        # Save Relevant Images
+        img = extract_first_page_screenshot_from_pdf(report_file_path)
+        logger.log_object(img, tag="pdf_image")
+        exp = ModelExperimentLoaderFromPDFfiles().load(report_file_path)
+        logger.log_object(exp, tag="load_experiment")
+    with logger.tag("d"):
+        exp = QlibModelCoSTEER(scenario).develop(exp)
+        logger.log_object(exp, tag="developed_experiment")
+    return exp
+
+
+if __name__ == "__main__":
+    fire.Fire(extract_models_and_implement)

rdagent/app/kaggle/conf.py

@@ -0,0 +1,72 @@
+from pathlib import Path
+
+from pydantic_settings import BaseSettings
+
+from rdagent.components.workflow.conf import BasePropSetting
+
+
+class KaggleBasePropSetting(BasePropSetting):
+    class Config:
+        env_prefix = "KG_"
+        """Use `KG_` as prefix for environment variables"""
+        protected_namespaces = ()
+        """Do not allow overriding of these namespaces"""
+
+    # 1) overriding the default
+    scen: str = "rdagent.scenarios.kaggle.experiment.scenario.KGScenario"
+    """Scenario class for data mining model"""
+
+    knowledge_base: str = ""  # TODO enable this line to use the knowledge base
+    # knowledge_base: str = "rdagent.scenarios.kaggle.knowledge_management.graph.KGKnowledgeGraph"
+    """Knowledge base class"""
+
+    knowledge_base_path: str = "kg_graph.pkl"
+    """Knowledge base path"""
+
+    hypothesis_gen: str = "rdagent.scenarios.kaggle.proposal.proposal.KGHypothesisGen"
+    """Hypothesis generation class"""
+
+    hypothesis2experiment: str = "rdagent.scenarios.kaggle.proposal.proposal.KGHypothesis2Experiment"
+    """Hypothesis to experiment class"""
+
+    feature_coder: str = "rdagent.scenarios.kaggle.developer.coder.KGFactorCoSTEER"
+    """Feature Coder class"""
+
+    model_feature_selection_coder: str = "rdagent.scenarios.kaggle.developer.coder.KGModelFeatureSelectionCoder"
+    """Model Feature Selection Coder class"""
+
+    model_coder: str = "rdagent.scenarios.kaggle.developer.coder.KGModelCoSTEER"
+    """Model Coder class"""
+
+    feature_runner: str = "rdagent.scenarios.kaggle.developer.runner.KGFactorRunner"
+    """Feature Runner class"""
+
+    model_runner: str = "rdagent.scenarios.kaggle.developer.runner.KGModelRunner"
+    """Model Runner class"""
+
+    summarizer: str = "rdagent.scenarios.kaggle.developer.feedback.KGHypothesisExperiment2Feedback"
+    """Summarizer class"""
+
+    evolving_n: int = 10
+    """Number of evolutions"""
+
+    competition: str = ""
+
+    local_data_path: str = "/data/userdata/share/kaggle"
+
+    domain_knowledge_path: str = "/data/userdata/share/kaggle/domain_knowledge"
+
+    rag_path: str = "git_ignore_folder/rag"
+
+    if_action_choosing_based_on_UCB: bool = False
+
+    if_using_graph_rag: bool = False
+
+    if_using_vector_rag: bool = False
+
+    auto_submit: bool = True
+
+    mini_case: bool = False
+
+
+KAGGLE_IMPLEMENT_SETTING = KaggleBasePropSetting()

rdagent/app/kaggle/loop.py

@@ -0,0 +1,142 @@
+import subprocess
+from collections import defaultdict
+from typing import Any
+
+import fire
+
+from rdagent.app.kaggle.conf import KAGGLE_IMPLEMENT_SETTING
+from rdagent.components.workflow.conf import BasePropSetting
+from rdagent.components.workflow.rd_loop import RDLoop
+from rdagent.core.developer import Developer
+from rdagent.core.exception import FactorEmptyError, ModelEmptyError
+from rdagent.core.proposal import (
+    Hypothesis2Experiment,
+    HypothesisExperiment2Feedback,
+    HypothesisGen,
+    Trace,
+)
+from rdagent.core.scenario import Scenario
+from rdagent.core.utils import import_class
+from rdagent.log import rdagent_logger as logger
+from rdagent.log.time import measure_time
+from rdagent.scenarios.kaggle.experiment.scenario import (
+    KG_ACTION_FEATURE_ENGINEERING,
+    KG_ACTION_FEATURE_PROCESSING,
+    KG_ACTION_MODEL_FEATURE_SELECTION,
+)
+from rdagent.scenarios.kaggle.experiment.utils import python_files_to_notebook
+from rdagent.scenarios.kaggle.kaggle_crawler import download_data
+from rdagent.scenarios.kaggle.proposal.proposal import KGTrace
+
+
+class KaggleRDLoop(RDLoop):
+    @measure_time
+    def __init__(self, PROP_SETTING: BasePropSetting):
+        with logger.tag("init"):
+            scen: Scenario = import_class(PROP_SETTING.scen)(PROP_SETTING.competition)
+            logger.log_object(scen, tag="scenario")
+            knowledge_base = (
+                import_class(PROP_SETTING.knowledge_base)(PROP_SETTING.knowledge_base_path, scen)
+                if PROP_SETTING.knowledge_base != ""
+                else None
+            )
+            logger.log_object(knowledge_base, tag="knowledge_base")
+            self.hypothesis_gen: HypothesisGen = import_class(PROP_SETTING.hypothesis_gen)(scen)
+            logger.log_object(self.hypothesis_gen, tag="hypothesis generator")
+            self.hypothesis2experiment: Hypothesis2Experiment = import_class(PROP_SETTING.hypothesis2experiment)()
+            logger.log_object(self.hypothesis2experiment, tag="hypothesis2experiment")
+            self.feature_coder: Developer = import_class(PROP_SETTING.feature_coder)(scen)
+            logger.log_object(self.feature_coder, tag="feature coder")
+            self.model_feature_selection_coder: Developer = import_class(PROP_SETTING.model_feature_selection_coder)(
+                scen
+            )
+            logger.log_object(self.model_feature_selection_coder, tag="model feature selection coder")
+            self.model_coder: Developer = import_class(PROP_SETTING.model_coder)(scen)
+            logger.log_object(self.model_coder, tag="model coder")
+            self.feature_runner: Developer = import_class(PROP_SETTING.feature_runner)(scen)
+            logger.log_object(self.feature_runner, tag="feature runner")
+            self.model_runner: Developer = import_class(PROP_SETTING.model_runner)(scen)
+            logger.log_object(self.model_runner, tag="model runner")
+            self.summarizer: HypothesisExperiment2Feedback = import_class(PROP_SETTING.summarizer)(scen)
+            logger.log_object(self.summarizer, tag="summarizer")
+            self.trace = KGTrace(scen=scen, knowledge_base=knowledge_base)
+            super(RDLoop, self).__init__()
+
+    @measure_time
+    def coding(self, prev_out: dict[str, Any]):
+        with logger.tag("d"):  # develop
+            if prev_out["propose"].action in [KG_ACTION_FEATURE_ENGINEERING, KG_ACTION_FEATURE_PROCESSING]:
+                exp = self.feature_coder.develop(prev_out["exp_gen"])
+            elif prev_out["propose"].action == KG_ACTION_MODEL_FEATURE_SELECTION:
+                exp = self.model_feature_selection_coder.develop(prev_out["exp_gen"])
+            else:
+                exp = self.model_coder.develop(prev_out["exp_gen"])
+            logger.log_object(exp.sub_workspace_list, tag="coder result")
+        return exp
+
+    @measure_time
+    def running(self, prev_out: dict[str, Any]):
+        with logger.tag("ef"):  # evaluate and feedback
+            if prev_out["propose"].action in [KG_ACTION_FEATURE_ENGINEERING, KG_ACTION_FEATURE_PROCESSING]:
+                exp = self.feature_runner.develop(prev_out["coding"])
+            else:
+                exp = self.model_runner.develop(prev_out["coding"])
+            logger.log_object(exp, tag="runner result")
+            if KAGGLE_IMPLEMENT_SETTING.competition in [
+                "optiver-realized-volatility-prediction",
+                "covid19-global-forecasting-week-1",
+            ]:
+                try:
+                    python_files_to_notebook(
+                        KAGGLE_IMPLEMENT_SETTING.competition, exp.experiment_workspace.workspace_path
+                    )
+                except Exception as e:
+                    logger.error(f"Merge python files to one file failed: {e}")
+            if KAGGLE_IMPLEMENT_SETTING.auto_submit:
+                csv_path = exp.experiment_workspace.workspace_path / "submission.csv"
+                try:
+                    subprocess.run(
+                        [
+                            "kaggle",
+                            "competitions",
+                            "submit",
+                            "-f",
+                            str(csv_path.absolute()),
+                            "-m",
+                            str(csv_path.parent.absolute()),
+                            KAGGLE_IMPLEMENT_SETTING.competition,
+                        ],
+                        check=True,
+                    )
+                except subprocess.CalledProcessError as e:
+                    logger.error(f"Auto submission failed: \n{e}")
+                except Exception as e:
+                    logger.error(f"Other exception when use kaggle api:\n{e}")
+
+        return exp
+
+    skip_loop_error = (ModelEmptyError, FactorEmptyError)
+
+
+def main(path=None, step_n=None, competition=None):
+    """
+    Auto R&D Evolving loop for models in a kaggle{} scenario.
+    You can continue running session by
+    .. code-block:: bash
+        dotenv run -- python rdagent/app/kaggle/loop.py [--competition titanic] $LOG_PATH/__session__/1/0_propose  --step_n 1   # `step_n` is a optional parameter
+        rdagent kaggle --competition playground-series-s4e8  # You are encouraged to use this one.
+    """
+    if competition:
+        KAGGLE_IMPLEMENT_SETTING.competition = competition
+        download_data(competition=competition, local_path=KAGGLE_IMPLEMENT_SETTING.local_data_path)
+    else:
+        logger.error("Please specify competition name.")
+    if path is None:
+        kaggle_loop = KaggleRDLoop(KAGGLE_IMPLEMENT_SETTING)
+    else:
+        kaggle_loop = KaggleRDLoop.load(path)
+    kaggle_loop.run(step_n=step_n)
+
+
+if __name__ == "__main__":
+    fire.Fire(main)

rdagent/app/model_implementation/README.md

@@ -1,39 +0,0 @@
-
-# Preparation
-
-## Install Pytorch
-CPU CUDA will be enough for verify the implementation
-
-Please install pytorch based on your system.
-Here is an example on my system
-```bash
-pip3 install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu
-pip3 install torch_geometric
-
-```
-
-# Tasks
-
-## Task Extraction
-From paper to task.
-```bash
-python rdagent/app/model_implementation/task_extraction.py
-# It may based on rdagent/document_reader/document_reader.py
-```
-
-## Complete workflow
-From paper to implementation
-``` bash
-# Similar to
-# rdagent/app/factor_extraction_and_implementation/factor_extract_and_implement.py
-```
-
-## Paper benchmark
-```bash
-python rdagent/app/model_implementation/eval.py
-
-TODO:
-- Is evaluation reasonable
-```
-
-## Evolving

rdagent/app/model_implementation/eval.py

@@ -1,29 +0,0 @@
-from pathlib import Path
-
-DIRNAME = Path(__file__).absolute().resolve().parent
-
-from rdagent.model_implementation.benchmark.eval import ModelImpValEval
-from rdagent.model_implementation.one_shot import ModelTaskGen
-from rdagent.model_implementation.task import ModelImpLoader, ModelTaskLoderJson
-
-mtl = ModelTaskLoderJson("TODO: A Path to json")
-
-task_l = mtl.load()
-
-mtg = ModelTaskGen()
-
-impl_l = mtg.generate(task_l)
-
-# TODO: Align it with the benchmark framework after @wenjun's refine the evaluation part.
-# Currently, we just handcraft a workflow for fast evaluation.
-
-mil = ModelImpLoader(DIRNAME.parent.parent / "model_implementation" / "benchmark" /  "gt_code")
-
-mie = ModelImpValEval()
-# Evaluation:
-eval_l = []
-for impl in impl_l:
-    gt_impl = mil.load(impl.target_task)
-    eval_l.append(mie.evaluate(gt_impl, impl))
-
-print(eval_l)

rdagent/app/qlib_rd_loop/conf.py

@@ -0,0 +1,81 @@
+from pydantic_settings import BaseSettings
+
+from rdagent.components.workflow.conf import BasePropSetting
+
+
+class ModelBasePropSetting(BasePropSetting):
+    class Config:
+        env_prefix = "QLIB_MODEL_"
+        """Use `QLIB_MODEL_` as prefix for environment variables"""
+        protected_namespaces = ()
+        """Add 'model_' to the protected namespaces"""
+
+    # 1) override base settings
+    scen: str = "rdagent.scenarios.qlib.experiment.model_experiment.QlibModelScenario"
+    """Scenario class for Qlib Model"""
+
+    hypothesis_gen: str = "rdagent.scenarios.qlib.proposal.model_proposal.QlibModelHypothesisGen"
+    """Hypothesis generation class"""
+
+    hypothesis2experiment: str = "rdagent.scenarios.qlib.proposal.model_proposal.QlibModelHypothesis2Experiment"
+    """Hypothesis to experiment class"""
+
+    coder: str = "rdagent.scenarios.qlib.developer.model_coder.QlibModelCoSTEER"
+    """Coder class"""
+
+    runner: str = "rdagent.scenarios.qlib.developer.model_runner.QlibModelRunner"
+    """Runner class"""
+
+    summarizer: str = "rdagent.scenarios.qlib.developer.feedback.QlibModelHypothesisExperiment2Feedback"
+    """Summarizer class"""
+
+    evolving_n: int = 10
+    """Number of evolutions"""
+
+
+class FactorBasePropSetting(BasePropSetting):
+    class Config:
+        env_prefix = "QLIB_FACTOR_"
+        """Use `QLIB_FACTOR_` as prefix for environment variables"""
+        protected_namespaces = ()
+        """Add 'factor_' to the protected namespaces"""
+
+    # 1) override base settings
+    scen: str = "rdagent.scenarios.qlib.experiment.factor_experiment.QlibFactorScenario"
+    """Scenario class for Qlib Factor"""
+
+    hypothesis_gen: str = "rdagent.scenarios.qlib.proposal.factor_proposal.QlibFactorHypothesisGen"
+    """Hypothesis generation class"""
+
+    hypothesis2experiment: str = "rdagent.scenarios.qlib.proposal.factor_proposal.QlibFactorHypothesis2Experiment"
+    """Hypothesis to experiment class"""
+
+    coder: str = "rdagent.scenarios.qlib.developer.factor_coder.QlibFactorCoSTEER"
+    """Coder class"""
+
+    runner: str = "rdagent.scenarios.qlib.developer.factor_runner.QlibFactorRunner"
+    """Runner class"""
+
+    summarizer: str = "rdagent.scenarios.qlib.developer.feedback.QlibFactorHypothesisExperiment2Feedback"
+    """Summarizer class"""
+
+    evolving_n: int = 10
+    """Number of evolutions"""
+
+
+class FactorFromReportPropSetting(FactorBasePropSetting):
+    # 1) override the scen attribute
+    scen: str = "rdagent.scenarios.qlib.experiment.factor_from_report_experiment.QlibFactorFromReportScenario"
+    """Scenario class for Qlib Factor from Report"""
+
+    # 2) sub task specific:
+    report_result_json_file_path: str = "git_ignore_folder/report_list.json"
+    """Path to the JSON file listing research reports for factor extraction"""
+
+    max_factors_per_exp: int = 10000
+    """Maximum number of factors implemented per experiment"""
+
+
+FACTOR_PROP_SETTING = FactorBasePropSetting()
+FACTOR_FROM_REPORT_PROP_SETTING = FactorFromReportPropSetting()
+MODEL_PROP_SETTING = ModelBasePropSetting()

rdagent/app/qlib_rd_loop/factor.py

@@ -0,0 +1,49 @@
+"""
+Factor workflow with session control
+"""
+
+from typing import Any
+
+import fire
+
+from rdagent.app.qlib_rd_loop.conf import FACTOR_PROP_SETTING
+from rdagent.components.workflow.rd_loop import RDLoop
+from rdagent.core.exception import FactorEmptyError
+from rdagent.log import rdagent_logger as logger
+from rdagent.log.time import measure_time
+
+
+class FactorRDLoop(RDLoop):
+    skip_loop_error = (FactorEmptyError,)
+
+    @measure_time
+    def running(self, prev_out: dict[str, Any]):
+        with logger.tag("ef"):  # evaluate and feedback
+            exp = self.runner.develop(prev_out["coding"])
+            if exp is None:
+                logger.error(f"Factor extraction failed.")
+                raise FactorEmptyError("Factor extraction failed.")
+            logger.log_object(exp, tag="runner result")
+        return exp
+
+
+def main(path=None, step_n=None):
+    """
+    Auto R&D Evolving loop for fintech factors.
+
+    You can continue running session by
+
+    .. code-block:: python
+
+        dotenv run -- python rdagent/app/qlib_rd_loop/factor.py $LOG_PATH/__session__/1/0_propose  --step_n 1   # `step_n` is a optional paramter
+
+    """
+    if path is None:
+        model_loop = FactorRDLoop(FACTOR_PROP_SETTING)
+    else:
+        model_loop = FactorRDLoop.load(path)
+    model_loop.run(step_n=step_n)
+
+
+if __name__ == "__main__":
+    fire.Fire(main)

rdagent/app/qlib_rd_loop/factor_from_report.py

@@ -0,0 +1,172 @@
+import json
+from pathlib import Path
+from typing import Any, Tuple
+
+import fire
+from jinja2 import Environment, StrictUndefined
+
+from rdagent.app.qlib_rd_loop.conf import FACTOR_FROM_REPORT_PROP_SETTING
+from rdagent.app.qlib_rd_loop.factor import FactorRDLoop
+from rdagent.components.document_reader.document_reader import (
+    extract_first_page_screenshot_from_pdf,
+    load_and_process_pdfs_by_langchain,
+)
+from rdagent.core.prompts import Prompts
+from rdagent.core.proposal import Hypothesis
+from rdagent.log import rdagent_logger as logger
+from rdagent.log.time import measure_time
+from rdagent.oai.llm_utils import APIBackend
+from rdagent.scenarios.qlib.experiment.factor_experiment import QlibFactorExperiment
+from rdagent.scenarios.qlib.factor_experiment_loader.pdf_loader import (
+    FactorExperimentLoaderFromPDFfiles,
+)
+from rdagent.utils.workflow import LoopMeta
+
+prompts_path = Path(__file__).parent / "prompts.yaml"
+prompts = Prompts(file_path=prompts_path)
+
+
+def generate_hypothesis(factor_result: dict, report_content: str) -> str:
+    """
+    Generate a hypothesis based on factor results and report content.
+
+    Args:
+        factor_result (dict): The results of the factor analysis.
+        report_content (str): The content of the report.
+
+    Returns:
+        str: The generated hypothesis.
+    """
+    system_prompt = (
+        Environment(undefined=StrictUndefined).from_string(prompts["hypothesis_generation"]["system"]).render()
+    )
+    user_prompt = (
+        Environment(undefined=StrictUndefined)
+        .from_string(prompts["hypothesis_generation"]["user"])
+        .render(factor_descriptions=json.dumps(factor_result), report_content=report_content)
+    )
+
+    response = APIBackend().build_messages_and_create_chat_completion(
+        user_prompt=user_prompt,
+        system_prompt=system_prompt,
+        json_mode=True,
+    )
+
+    response_json = json.loads(response)
+
+    return Hypothesis(
+        hypothesis=response_json.get("hypothesis", "No hypothesis provided"),
+        reason=response_json.get("reason", "No reason provided"),
+        concise_reason=response_json.get("concise_reason", "No concise reason provided"),
+        concise_observation=response_json.get("concise_observation", "No concise observation provided"),
+        concise_justification=response_json.get("concise_justification", "No concise justification provided"),
+        concise_knowledge=response_json.get("concise_knowledge", "No concise knowledge provided"),
+    )
+
+
+def extract_hypothesis_and_exp_from_reports(report_file_path: str) -> Tuple[QlibFactorExperiment, Hypothesis]:
+    """
+    Extract hypothesis and experiment details from report files.
+
+    Args:
+        report_file_path (str): Path to the report file.
+
+    Returns:
+        Tuple[QlibFactorExperiment, Hypothesis]: The extracted experiment and generated hypothesis.
+    """
+    with logger.tag("extract_factors_and_implement"):
+        with logger.tag("load_factor_tasks"):
+            exp = FactorExperimentLoaderFromPDFfiles().load(report_file_path)
+            if exp is None or exp.sub_tasks == []:
+                return None, None
+
+        with logger.tag("load_pdf_screenshot"):
+            pdf_screenshot = extract_first_page_screenshot_from_pdf(report_file_path)
+            logger.log_object(pdf_screenshot)
+
+    docs_dict = load_and_process_pdfs_by_langchain(report_file_path)
+
+    factor_result = {
+        task.factor_name: {
+            "description": task.factor_description,
+            "formulation": task.factor_formulation,
+            "variables": task.variables,
+            "resources": task.factor_resources,
+        }
+        for task in exp.sub_tasks
+    }
+
+    report_content = "\n".join(docs_dict.values())
+    hypothesis = generate_hypothesis(factor_result, report_content)
+    return exp, hypothesis
+
+
+class FactorReportLoop(FactorRDLoop, metaclass=LoopMeta):
+    @measure_time
+    def __init__(self, report_folder: str = None):
+        super().__init__(PROP_SETTING=FACTOR_FROM_REPORT_PROP_SETTING)
+        if report_folder is None:
+            self.judge_pdf_data_items = json.load(
+                open(FACTOR_FROM_REPORT_PROP_SETTING.report_result_json_file_path, "r")
+            )
+        else:
+            self.judge_pdf_data_items = [i for i in Path(report_folder).rglob("*.pdf")]
+
+        self.pdf_file_index = 0
+        self.valid_pdf_file_count = 0
+        self.current_loop_hypothesis = None
+        self.current_loop_exp = None
+        self.steps = ["propose_hypo_exp", "propose", "exp_gen", "coding", "running", "feedback"]
+
+    @measure_time
+    def propose_hypo_exp(self, prev_out: dict[str, Any]):
+        with logger.tag("r"):
+            while True:
+                if self.valid_pdf_file_count > 15:
+                    break
+                report_file_path = self.judge_pdf_data_items[self.pdf_file_index]
+                logger.info(f"Processing number {self.pdf_file_index} report: {report_file_path}")
+                self.pdf_file_index += 1
+                exp, hypothesis = extract_hypothesis_and_exp_from_reports(str(report_file_path))
+                if exp is None:
+                    continue
+                self.valid_pdf_file_count += 1
+                exp.based_experiments = [QlibFactorExperiment(sub_tasks=[])] + [t[1] for t in self.trace.hist if t[2]]
+                exp.sub_workspace_list = exp.sub_workspace_list[: FACTOR_FROM_REPORT_PROP_SETTING.max_factors_per_exp]
+                exp.sub_tasks = exp.sub_tasks[: FACTOR_FROM_REPORT_PROP_SETTING.max_factors_per_exp]
+                logger.log_object(hypothesis, tag="hypothesis generation")
+                logger.log_object(exp.sub_tasks, tag="experiment generation")
+                self.current_loop_hypothesis = hypothesis
+                self.current_loop_exp = exp
+                return None
+
+    @measure_time
+    def propose(self, prev_out: dict[str, Any]):
+        return self.current_loop_hypothesis
+
+    @measure_time
+    def exp_gen(self, prev_out: dict[str, Any]):
+        return self.current_loop_exp
+
+
+def main(report_folder=None, path=None, step_n=None):
+    """
+    Auto R&D Evolving loop for fintech factors (the factors are extracted from finance reports).
+
+    Args:
+        report_folder (str, optional): The folder contains the report PDF files. Reports will be loaded from this folder.
+        path (str, optional): The path for loading a session. If provided, the session will be loaded.
+        step_n (int, optional): Step number to continue running a session.
+    """
+    if path is None and report_folder is None:
+        model_loop = FactorReportLoop()
+    elif path is not None:
+        model_loop = FactorReportLoop.load(path)
+    else:
+        model_loop = FactorReportLoop(report_folder=report_folder)
+
+    model_loop.run(step_n=step_n)
+
+
+if __name__ == "__main__":
+    fire.Fire(main)

rdagent/app/qlib_rd_loop/model.py

@@ -0,0 +1,35 @@
+"""
+Model workflow with session control
+"""
+
+import fire
+
+from rdagent.app.qlib_rd_loop.conf import MODEL_PROP_SETTING
+from rdagent.components.workflow.rd_loop import RDLoop
+from rdagent.core.exception import ModelEmptyError
+
+
+class ModelRDLoop(RDLoop):
+    skip_loop_error = (ModelEmptyError,)
+
+
+def main(path=None, step_n=None):
+    """
+    Auto R&D Evolving loop for fintech models
+
+    You can continue running session by
+
+    .. code-block:: python
+
+        dotenv run -- python rdagent/app/qlib_rd_loop/model.py $LOG_PATH/__session__/1/0_propose  --step_n 1   # `step_n` is a optional paramter
+
+    """
+    if path is None:
+        model_loop = ModelRDLoop(MODEL_PROP_SETTING)
+    else:
+        model_loop = ModelRDLoop.load(path)
+    model_loop.run(step_n=step_n)
+
+
+if __name__ == "__main__":
+    fire.Fire(main)

rdagent/app/qlib_rd_loop/prompts.yaml

@@ -0,0 +1,19 @@
+hypothesis_generation:
+  system: |-
+    You are an expert in financial analysis. Your task is to generate a well-reasoned hypothesis based on the provided financial factors and report content.
+    Please ensure your response is in JSON format as shown below:
+    {
+      "hypothesis": "A clear and concise hypothesis based on the provided information.",
+      "reason": "A detailed explanation supporting the generated hypothesis.",
+      "concise_reason": "One line summary that focuses on the justification for the change that leads to the hypothesis (like a part of a knowledge that we are building)",
+      "concise_observation": "One line summary. It focuses on the observation of the given scenario, data characteristics, or previous experiences (failures & succeses).",
+      "concise_justification": "One line summary. It focuses on the justification for the change in new hypothesis and the route of exploration supporting the growth of the hypothesis, based on the observation. ",
+      "concise_knowledge": "One line summary. It focuses on a transferable knowledege that comes with the new hypothesis. Use conditional grammar. eg. "If...., ..; When..., .; and etc"
+    }
+
+  user: |-
+    The following are the financial factors and their descriptions:
+    {{ factor_descriptions }}
+
+    The report content is as follows:
+    {{ report_content }}

rdagent/app/utils/info.py

@@ -0,0 +1,86 @@
+import importlib.metadata
+import platform
+import sys
+from pathlib import Path
+
+import docker
+import requests
+from setuptools_scm import get_version
+
+from rdagent.log import rdagent_logger as logger
+
+
+def sys_info():
+    """collect system related info"""
+    method_list = [
+        ["Name of current operating system: ", "system"],
+        ["Processor architecture: ", "machine"],
+        ["System, version, and hardware information: ", "platform"],
+        ["Version number of the system: ", "version"],
+    ]
+    for method in method_list:
+        logger.info(f"{method[0]}{getattr(platform, method[1])()}")
+    return None
+
+
+def python_info():
+    """collect Python related info"""
+    python_version = sys.version.replace("\n", " ")
+    logger.info(f"Python version: {python_version}")
+    return None
+
+
+def docker_info():
+    client = docker.from_env()
+    containers = client.containers.list(all=True)
+    if containers:
+        containers.sort(key=lambda c: c.attrs["Created"])
+        last_container = containers[-1]
+        logger.info(f"Container ID: {last_container.id}")
+        logger.info(f"Container Name: {last_container.name}")
+        logger.info(f"Container Status: {last_container.status}")
+        logger.info(f"Image ID used by the container: {last_container.image.id}")
+        logger.info(f"Image tag used by the container: {last_container.image.tags}")
+        logger.info(f"Container port mapping: {last_container.ports}")
+        logger.info(f"Container Label: {last_container.labels}")
+        logger.info(f"Startup Commands: {' '.join(client.containers.get(last_container.id).attrs['Config']['Cmd'])}")
+    else:
+        logger.info(f"No run containers.")
+
+
+def rdagent_info():
+    """collect rdagent related info"""
+    current_version = importlib.metadata.version("rdagent")
+    logger.info(f"RD-Agent version: {current_version}")
+    api_url = f"https://api.github.com/repos/microsoft/RD-Agent/contents/requirements.txt?ref=main"
+    response = requests.get(api_url)
+    if response.status_code == 200:
+        files = response.json()
+        file_url = files["download_url"]
+        file_response = requests.get(file_url)
+        if file_response.status_code == 200:
+            all_file_contents = file_response.text.split("\n")
+        else:
+            logger.warning(f"Failed to retrieve {files['name']}, status code: {file_response.status_code}")
+    else:
+        logger.warning(f"Failed to retrieve files in folder, status code: {response.status_code}")
+    package_list = [
+        item.split("#")[0].strip() for item in all_file_contents if item.strip() and not item.startswith("#")
+    ]
+    package_version_list = []
+    for package in package_list:
+        if package == "typer[all]":
+            package = "typer"
+        version = importlib.metadata.version(package)
+        package_version_list.append(f"{package}=={version}")
+    logger.info(f"Package version: {package_version_list}")
+    return None
+
+
+def collect_info():
+    """Prints information about the system and the installed packages."""
+    sys_info()
+    python_info()
+    docker_info()
+    rdagent_info()
+    return None

rdagent/benchmark/conf.py

@@ -1,27 +0,0 @@
-from dotenv import load_dotenv
-load_dotenv(verbose=True, override=True)
-from dataclasses import field
-from pathlib import Path
-from typing import Literal, Optional, Union
-
-from pydantic_settings import BaseSettings
-
-DIRNAME = Path(__file__).absolute().resolve().parent
-
-BENCHMARK_VERSION = Literal["paper", "amcV01", "amcV02train", "amcV02test"]
-
-class BenchmarkSettings(BaseSettings):
-
-    ground_truth_dir: Path = DIRNAME / "ground_truth"
-
-    bench_version: Union[BENCHMARK_VERSION, str] = "paper"
-
-    bench_test_round: int = 20 
-    bench_test_case_n: Optional[int] = None  # how many test cases to run; If not given, all test cases will be run
-
-    bench_method_cls: str = "scripts.factor_implementation.baselines.naive.one_shot.OneshotFactorGen"
-    bench_method_extra_kwargs: dict = field(
-        default_factory=dict,
-    )  # extra kwargs for the method to be tested except the task list
-
-    bench_result_path: Path = DIRNAME / "result"

rdagent/benchmark/eval_method.py

@@ -1,163 +0,0 @@
-from pathlib import Path
-from typing import List, Tuple, Union
-
-from tqdm import tqdm
-from collections import defaultdict
-from rdagent.core.conf import RD_AGENT_SETTINGS
-from rdagent.core.exception import ImplementRunException
-from rdagent.core.task import (
-    TaskImplementation,
-    TestCase,
-)
-from rdagent.factor_implementation.evolving.evaluators import (
-    FactorImplementationCorrelationEvaluator,
-    FactorImplementationIndexEvaluator,
-    FactorImplementationIndexFormatEvaluator,
-    FactorImplementationMissingValuesEvaluator,
-    FactorImplementationRowCountEvaluator,
-    FactorImplementationSingleColumnEvaluator,
-    FactorImplementationValuesEvaluator,
-    FactorImplementationEvaluator,
-)
-from rdagent.core.implementation import TaskGenerator
-from rdagent.core.utils import multiprocessing_wrapper
-from rdagent.factor_implementation.evolving.factor import FileBasedFactorImplementation
-
-
-class BaseEval:
-    """
-    The benchmark benchmark evaluation.
-    """
-
-    def __init__(
-        self,
-        evaluator_l: List[FactorImplementationEvaluator],
-        test_cases: List[TestCase],
-        generate_method: TaskGenerator,
-        catch_eval_except: bool = True,
-    ):
-        """Parameters
-        ----------
-        test_cases : List[TestCase]
-            cases to be evaluated, ground truth are included in the test cases.
-        evaluator_l : List[FactorImplementationEvaluator]
-            A list of evaluators to evaluate the generated code.
-        catch_eval_except : bool
-            If we want to debug the evaluators, we recommend to set the this parameter to True.
-        """
-        self.evaluator_l = evaluator_l
-        self.test_cases = test_cases
-        self.generate_method = generate_method
-        self.catch_eval_except = catch_eval_except
-
-    def load_cases_to_eval(
-        self,
-        path: Union[Path, str],
-        **kwargs,
-    ) -> List[TaskImplementation]:
-        path = Path(path)
-        fi_l = []
-        for tc in self.test_cases:
-            try:
-                fi = FileBasedFactorImplementation.from_folder(tc.task, path, **kwargs)
-                fi_l.append(fi)
-            except FileNotFoundError:
-                print("Fail to load test case for factor: ", tc.task.factor_name)
-        return fi_l
-
-    def eval_case(
-        self,
-        case_gt: TaskImplementation,
-        case_gen: TaskImplementation,
-    ) -> List[Union[Tuple[FactorImplementationEvaluator, object], Exception]]:
-        """Parameters
-        ----------
-        case_gt : FactorImplementation
-
-        case_gen : FactorImplementation
-
-
-        Returns
-        -------
-        List[Union[Tuple[FactorImplementationEvaluator, object],Exception]]
-            for each item
-                If the evaluation run successfully, return the evaluate results.  Otherwise, return the exception.
-        """
-        eval_res = []
-        for ev in self.evaluator_l:
-            try:
-                eval_res.append((ev, ev.evaluate(case_gt, case_gen)))
-                # if the corr ev is successfully evaluated and achieve the best performance, then break
-            except ImplementRunException as e:
-                return e
-            except Exception as e:
-                # exception when evaluation
-                if self.catch_eval_except:
-                    eval_res.append((ev, e))
-                else:
-                    raise e
-        return eval_res
-
-
-class FactorImplementEval(BaseEval):
-    def __init__(
-        self,
-        test_case: TestCase,
-        method: TaskGenerator,
-        test_round: int = 10,
-        *args,
-        **kwargs,
-    ):
-        # evaluator collection for online evaluation
-        online_evaluator_l = (
-            [
-                FactorImplementationCorrelationEvaluator,
-                FactorImplementationIndexEvaluator,
-                FactorImplementationIndexFormatEvaluator,
-                FactorImplementationMissingValuesEvaluator,
-                FactorImplementationRowCountEvaluator,
-                FactorImplementationSingleColumnEvaluator,
-                FactorImplementationValuesEvaluator,
-            ],
-        )
-        super().__init__(online_evaluator_l, test_case, method, *args, **kwargs)
-        self.test_round = test_round
-
-    def eval(self):
-
-        gen_factor_l_all_rounds = []
-        test_cases_all_rounds = []
-        res = defaultdict(list)
-        for _ in tqdm(range(self.test_round), desc="Rounds of Eval"):
-            print("\n========================================================")
-            print(f"Eval {_}-th times...")
-            print("========================================================\n")
-            try:
-                gen_factor_l = self.generate_method.generate(self.test_cases.target_task)
-            except KeyboardInterrupt:
-                # TODO: Why still need to save result after KeyboardInterrupt?
-                print("Manually interrupted the evaluation. Saving existing results")
-                break
-
-            if len(gen_factor_l) != len(self.test_cases):
-                raise ValueError(
-                    "The number of cases to eval should be equal to the number of test cases.",
-                )
-            gen_factor_l_all_rounds.extend(gen_factor_l)
-            test_cases_all_rounds.extend(self.test_cases)
-
-        eval_res_l = []
-
-        eval_res_list = multiprocessing_wrapper(
-            [
-                (self.eval_case, (gt_case.ground_truth, gen_factor))
-                for gt_case, gen_factor in zip(test_cases_all_rounds, gen_factor_l_all_rounds)
-            ],
-            n=RD_AGENT_SETTINGS.evo_multi_proc_n,
-        )
-
-        for gt_case, eval_res, gen_factor in tqdm(zip(test_cases_all_rounds, eval_res_list, gen_factor_l_all_rounds)):
-            res[gt_case.task.factor_name].append((gen_factor, eval_res))
-            eval_res_l.append(eval_res)
-
-        return res

rdagent/components/benchmark/conf.py

@@ -0,0 +1,36 @@
+from dataclasses import field
+from pathlib import Path
+from typing import Optional
+
+from pydantic_settings import BaseSettings
+
+DIRNAME = Path("./")
+
+
+class BenchmarkSettings(BaseSettings):
+    class Config:
+        env_prefix = "BENCHMARK_"
+        """Use `BENCHMARK_` as prefix for environment variables"""
+
+    ground_truth_dir: Path = DIRNAME / "ground_truth"
+    """ground truth dir"""
+
+    bench_data_path: Path = DIRNAME / "example.json"
+    """data for benchmark"""
+
+    bench_test_round: int = 10
+    """how many rounds to run, each round may cost 10 minutes"""
+
+    bench_test_case_n: Optional[int] = None
+    """how many test cases to run; If not given, all test cases will be run"""
+
+    bench_method_cls: str = "rdagent.components.coder.factor_coder.CoSTEER.FactorCoSTEER"
+    """method to be used for test cases"""
+
+    bench_method_extra_kwargs: dict = field(
+        default_factory=dict,
+    )
+    """extra kwargs for the method to be tested except the task list"""
+
+    bench_result_path: Path = DIRNAME / "result"
+    """result save path"""

rdagent/components/benchmark/eval_method.py

@@ -0,0 +1,222 @@
+from collections import defaultdict
+from pathlib import Path
+from typing import Dict, List, Tuple, Union
+
+import pandas as pd
+from tqdm import tqdm
+
+from rdagent.components.coder.factor_coder.config import FACTOR_IMPLEMENT_SETTINGS
+from rdagent.components.coder.factor_coder.CoSTEER.evaluators import (
+    FactorCorrelationEvaluator,
+    FactorEqualValueRatioEvaluator,
+    FactorEvaluator,
+    FactorIndexEvaluator,
+    FactorRowCountEvaluator,
+    FactorSingleColumnEvaluator,
+)
+from rdagent.components.coder.factor_coder.factor import FactorFBWorkspace
+from rdagent.core.conf import RD_AGENT_SETTINGS
+from rdagent.core.developer import Developer
+from rdagent.core.exception import CoderError
+from rdagent.core.experiment import Experiment, Task, Workspace
+from rdagent.core.scenario import Scenario
+from rdagent.core.utils import multiprocessing_wrapper
+
+EVAL_RES = Dict[
+    str,
+    List[Tuple[FactorEvaluator, Union[object, CoderError]]],
+]
+
+
+class TestCase:
+    def __init__(
+        self,
+        target_task: Task,
+        ground_truth: Workspace,
+    ):
+        self.target_task = target_task
+        self.ground_truth = ground_truth
+
+
+class TestCases:
+    def __init__(self, test_case_l: list[TestCase] = []):
+        # self.test_case_l = [TestCase(task, gt) for task, gt in zip(target_task, ground_truth)]
+        self.test_case_l = test_case_l
+
+    def __getitem__(self, item):
+        return self.test_case_l[item]
+
+    def __len__(self):
+        return len(self.test_case_l)
+
+    def get_exp(self):
+        return Experiment([case.target_task for case in self.test_case_l])
+
+    @property
+    def target_task(self):
+        return [case.target_task for case in self.test_case_l]
+
+    @property
+    def ground_truth(self):
+        return [case.ground_truth for case in self.test_case_l]
+
+
+class BaseEval:
+    """
+    The benchmark benchmark evaluation.
+    """
+
+    def __init__(
+        self,
+        evaluator_l: List[FactorEvaluator],
+        test_cases: TestCases,
+        generate_method: Developer,
+        catch_eval_except: bool = True,
+    ):
+        """Parameters
+        ----------
+        test_cases : TestCases
+            cases to be evaluated, ground truth are included in the test cases.
+        evaluator_l : List[FactorEvaluator]
+            A list of evaluators to evaluate the generated code.
+        catch_eval_except : bool
+            If we want to debug the evaluators, we recommend to set the this parameter to True.
+        """
+        self.evaluator_l = evaluator_l
+        self.test_cases = test_cases
+        self.generate_method = generate_method
+        self.catch_eval_except = catch_eval_except
+
+    def load_cases_to_eval(
+        self,
+        path: Union[Path, str],
+        **kwargs,
+    ) -> List[Workspace]:
+        path = Path(path)
+        fi_l = []
+        for tc in self.test_cases:
+            try:
+                fi = FactorFBWorkspace.from_folder(tc.task, path, **kwargs)
+                fi_l.append(fi)
+            except FileNotFoundError:
+                print("Fail to load test case for factor: ", tc.task.factor_name)
+        return fi_l
+
+    def eval_case(
+        self,
+        case_gt: Workspace,
+        case_gen: Workspace,
+    ) -> List[Union[Tuple[FactorEvaluator, object], Exception]]:
+        """Parameters
+        ----------
+        case_gt : FactorImplementation
+
+        case_gen : FactorImplementation
+
+
+        Returns
+        -------
+        List[Union[Tuple[FactorEvaluator, object],Exception]]
+            for each item
+                If the evaluation run successfully, return the evaluate results.  Otherwise, return the exception.
+        """
+        eval_res = []
+        for ev in self.evaluator_l:
+            try:
+                case_gen.raise_exception = True
+                eval_res.append((ev, ev.evaluate(implementation=case_gen, gt_implementation=case_gt)))
+                # if the corr ev is successfully evaluated and achieve the best performance, then break
+            except CoderError as e:
+                return e
+            except Exception as e:
+                # exception when evaluation
+                if self.catch_eval_except:
+                    eval_res.append((ev, e))
+                else:
+                    raise e
+        return eval_res
+
+
+class FactorImplementEval(BaseEval):
+    def __init__(
+        self,
+        test_cases: TestCases,
+        method: Developer,
+        *args,
+        scen: Scenario,
+        test_round: int = 10,
+        **kwargs,
+    ):
+        online_evaluator_l = [
+            FactorSingleColumnEvaluator(scen),
+            FactorRowCountEvaluator(scen),
+            FactorIndexEvaluator(scen),
+            FactorEqualValueRatioEvaluator(scen),
+            FactorCorrelationEvaluator(hard_check=False, scen=scen),
+        ]
+        super().__init__(online_evaluator_l, test_cases, method, *args, **kwargs)
+        self.test_round = test_round
+
+    def develop(self):
+        gen_factor_l_all_rounds = []
+        for _ in tqdm(range(self.test_round), desc="Rounds of Eval"):
+            print("\n========================================================")
+            print(f"Eval {_}-th times...")
+            print("========================================================\n")
+            try:
+                gen_factor_l = self.generate_method.develop(self.test_cases.get_exp())
+            except KeyboardInterrupt:
+                # TODO: Why still need to save result after KeyboardInterrupt?
+                print("Manually interrupted the evaluation. Saving existing results")
+                break
+
+            if len(gen_factor_l.sub_workspace_list) != len(self.test_cases.ground_truth):
+                raise ValueError(
+                    "The number of cases to eval should be equal to the number of test cases.",
+                )
+            gen_factor_l_all_rounds.extend(gen_factor_l.sub_workspace_list)
+
+        return gen_factor_l_all_rounds
+
+    def eval(self, gen_factor_l_all_rounds):
+        test_cases_all_rounds = []
+        res = defaultdict(list)
+        for _ in range(self.test_round):
+            test_cases_all_rounds.extend(self.test_cases.ground_truth)
+        eval_res_list = multiprocessing_wrapper(
+            [
+                (self.eval_case, (gt_case, gen_factor))
+                for gt_case, gen_factor in zip(test_cases_all_rounds, gen_factor_l_all_rounds)
+            ],
+            n=RD_AGENT_SETTINGS.multi_proc_n,
+        )
+
+        for gt_case, eval_res, gen_factor in tqdm(zip(test_cases_all_rounds, eval_res_list, gen_factor_l_all_rounds)):
+            res[gt_case.target_task.factor_name].append((gen_factor, eval_res))
+
+        return res
+
+    @staticmethod
+    def summarize_res(res: EVAL_RES) -> pd.DataFrame:
+        # None: indicate that it raises exception and get no results
+        sum_res = {}
+        for factor_name, runs in res.items():
+            for fi, err_or_res_l in runs:
+                # NOTE:  str(fi) may not be unique!!  Because the workspace can be skipped when hitting the cache.
+                uniq_key = f"{str(fi)},{id(fi)}"
+
+                key = (factor_name, uniq_key)
+                val = {}
+                if isinstance(err_or_res_l, Exception):
+                    val["run factor error"] = str(err_or_res_l.__class__)
+                else:
+                    val["run factor error"] = None
+                    for ev_obj, err_or_res in err_or_res_l:
+                        if isinstance(err_or_res, Exception):
+                            val[str(ev_obj)] = None
+                        else:
+                            feedback, metric = err_or_res
+                            val[str(ev_obj)] = metric
+                sum_res[key] = val
+
+        return pd.DataFrame(sum_res)

rdagent/components/benchmark/example.json

@@ -0,0 +1,36 @@
+{
+    "Turnover_Rate_Factor": {
+        "description": "A traditional factor based on 20-day average turnover rate, adjusted for market capitalization, which is further improved by applying the information distribution theory.",
+        "formulation": "\\text{Adjusted Turnover Rate} = \\frac{\\text{mean}(20\\text{-day turnover rate})}{\\text{Market Capitalization}}",
+        "variables": {
+            "20-day turnover rate": "Average turnover rate over the past 20 days.",
+            "Market Capitalization": "Total market value of a company's outstanding shares."
+        },
+        "Category": "Fundamentals",
+        "Difficulty": "Easy",
+        "gt_code": "import pandas as pd\n\ndata_f = pd.read_hdf('daily_f.h5')\n\ndata = data_f.reset_index()\nwindow_size = 20\n\nnominator=data.groupby('instrument')[['TurnoverRate_30D']].rolling(window=window_size).mean().reset_index(0, drop=True)\n# transfer to series\nnew=nominator['TurnoverRate_30D']\ndata['Turnover_Rate_Factor']=new/data['TradableACapital']\n\n# set the datetime and instrument as index and drop the original index\nresult=pd.DataFrame(data['Turnover_Rate_Factor']).set_index(data_f.index)\n\n# transfer the result to series\nresult=result['Turnover_Rate_Factor']\nresult.to_hdf(\"result.h5\", key=\"data\")" 
+    },
+    "PctTurn20": {
+        "description": "A factor representing the percentage change in turnover rate over the past 20 trading days, market-value neutralized.",
+        "formulation": "\\text{PctTurn20} = \\frac{1}{N} \\sum_{i=1}^{N} \\left( \\frac{\\text{Turnover}_{i, t} - \\text{Turnover}_{i, t-20}}{\\text{Turnover}_{i, t-20}} \\right)",
+        "variables": {
+            "N": "Number of stocks in the market.",
+            "Turnover_{i, t}": "Turnover of stock i at day t.",
+            "Turnover_{i, t-20}": "Turnover of stock i at day t-20."
+        },
+        "Category": "Volume&Price",
+        "Difficulty": "Medium",
+        "gt_code": "import pandas as pd\nfrom statsmodels import api as sm\n\ndef fill_mean(s: pd.Series) -> pd.Series:\n    return s.fillna(s.mean()).fillna(0.0)\n\ndef market_value_neutralize(s: pd.Series, mv: pd.Series) -> pd.Series:\n    s = s.groupby(\"datetime\", group_keys=False).apply(fill_mean)\n    mv = mv.groupby(\"datetime\", group_keys=False).apply(fill_mean)\n\n    df_f = mv.to_frame(\"MarketValue\")\n    df_f[\"const\"] = 1\n    X = df_f[[\"MarketValue\", \"const\"]]\n\n    # Perform the Ordinary Least Squares (OLS) regression\n    model = sm.OLS(s, X)\n    results = model.fit()\n\n    # Calculate the residuals\n    df_f[\"residual\"] = results.resid\n    df_f[\"norm_resi\"] = df_f.groupby(level=\"datetime\", group_keys=False)[\"residual\"].apply(\n        lambda x: (x - x.mean()) / x.std(),\n    )\n    return df_f[\"norm_resi\"]\n\n\n# get_turnover\ndf_pv = pd.read_hdf(\"daily_pv.h5\", key=\"data\")\ndf_f = pd.read_hdf(\"daily_f.h5\", key=\"data\")\nturnover = df_pv[\"$money\"] / df_f[\"TradableMarketValue\"]\n\nf = turnover.groupby(\"instrument\").pct_change(periods=20)\n\nf_neutralized = market_value_neutralize(f, df_f[\"TradableMarketValue\"])\n\nf_neutralized.to_hdf(\"result.h5\", key=\"data\")"
+    },
+    "PB_ROE": {
+        "description": "Constructed using the ranking difference between PB and ROE, with PB and ROE replacing original PB and ROE to obtain reconstructed factor values.",
+        "formulation": "\\text{rank}(PB\\_t) - rank(ROE_t)",
+        "variables": {
+            "\\text{rank}(PB_t)": "Ranking PB on cross-section at time t.",
+            "\\text{rank}(ROE_t)": "Ranking single-quarter ROE on cross-section at time t."
+        },
+        "Category": "High-Frequency",
+        "Difficulty": "Hard",
+        "gt_code": "#!/usr/bin/env python\n\nimport pandas as pd\n\ndata_f = pd.read_hdf('daily_f.h5')\n\ndata = data_f.reset_index()\n\n# Calculate the rank of PB and ROE\ndata['PB_rank'] = data.groupby('datetime')['B/P'].rank()\ndata['ROE_rank'] = data.groupby('datetime')['ROE'].rank()\n\n# Calculate the difference between the ranks\ndata['PB_ROE'] = data['PB_rank'] - data['ROE_rank']\n\n# set the datetime and instrument as index and drop the original index\nresult=pd.DataFrame(data['PB_ROE']).set_index(data_f.index)\n\n# transfer the result to series\nresult=result['PB_ROE']\nresult.to_hdf(\"result.h5\", key=\"data\")"
+    }
+}

rdagent/components/coder/factor_coder/CoSTEER/__init__.py

@@ -1,32 +1,43 @@
 import pickle
 from pathlib import Path
-from typing import List
-from rdagent.core.implementation import TaskGenerator
-from rdagent.core.task import TaskImplementation
-from rdagent.factor_implementation.evolving.knowledge_management import FactorImplementationKnowledgeBaseV1
-from rdagent.factor_implementation.evolving.factor import FactorImplementTask, FactorEvovlingItem
-from rdagent.factor_implementation.evolving.knowledge_management import (
-    FactorImplementationGraphKnowledgeBase,
-    FactorImplementationGraphRAGStrategy,
+
+from rdagent.components.coder.factor_coder.config import FACTOR_IMPLEMENT_SETTINGS
+from rdagent.components.coder.factor_coder.CoSTEER.evaluators import (
+    FactorEvaluatorForCoder,
+    FactorMultiEvaluator,
 )
-from rdagent.factor_implementation.evolving.evolving_strategy import FactorEvolvingStrategyWithGraph
-from rdagent.factor_implementation.evolving.evaluators import (
-    FactorImplementationsMultiEvaluator,
-    FactorImplementationEvaluatorV1,
+from rdagent.components.coder.factor_coder.CoSTEER.evolvable_subjects import (
+    FactorEvolvingItem,
 )
+from rdagent.components.coder.factor_coder.CoSTEER.evolving_agent import (
+    FactorRAGEvoAgent,
+)
+from rdagent.components.coder.factor_coder.CoSTEER.evolving_strategy import (
+    FactorEvolvingStrategyWithGraph,
+)
+from rdagent.components.coder.factor_coder.CoSTEER.knowledge_management import (
+    FactorGraphKnowledgeBase,
+    FactorGraphRAGStrategy,
+    FactorKnowledgeBaseV1,
+)
+from rdagent.components.coder.factor_coder.factor import FactorExperiment
+from rdagent.core.developer import Developer
 from rdagent.core.evolving_agent import RAGEvoAgent
-from rdagent.factor_implementation.share_modules.factor_implementation_config import (
-    FACTOR_IMPLEMENT_SETTINGS,
-)
+from rdagent.core.scenario import Scenario
+from rdagent.log import rdagent_logger as logger
 
 
-class CoSTEERFG(TaskGenerator):
+class FactorCoSTEER(Developer[FactorExperiment]):
     def __init__(
         self,
+        *args,
         with_knowledge: bool = True,
         with_feedback: bool = True,
         knowledge_self_gen: bool = True,
+        filter_final_evo: bool = True,
+        **kwargs,
     ) -> None:
+        super().__init__(*args, **kwargs)
         self.max_loop = FACTOR_IMPLEMENT_SETTINGS.max_loop
         self.knowledge_base_path = (
             Path(FACTOR_IMPLEMENT_SETTINGS.knowledge_base_path)
@@ -41,59 +52,62 @@ class CoSTEERFG(TaskGenerator):
         self.with_knowledge = with_knowledge
         self.with_feedback = with_feedback
         self.knowledge_self_gen = knowledge_self_gen
-        self.evolving_strategy = FactorEvolvingStrategyWithGraph()
+        self.filter_final_evo = filter_final_evo
+        self.evolving_strategy = FactorEvolvingStrategyWithGraph(scen=self.scen)
         # declare the factor evaluator
-        self.factor_evaluator = FactorImplementationsMultiEvaluator(FactorImplementationEvaluatorV1())
+        self.factor_evaluator = FactorMultiEvaluator(FactorEvaluatorForCoder(scen=self.scen), scen=self.scen)
         self.evolving_version = 2
 
     def load_or_init_knowledge_base(self, former_knowledge_base_path: Path = None, component_init_list: list = []):
-
         if former_knowledge_base_path is not None and former_knowledge_base_path.exists():
             factor_knowledge_base = pickle.load(open(former_knowledge_base_path, "rb"))
-            if self.evolving_version == 1 and not isinstance(
-                factor_knowledge_base, FactorImplementationKnowledgeBaseV1
-            ):
+            if self.evolving_version == 1 and not isinstance(factor_knowledge_base, FactorKnowledgeBaseV1):
                 raise ValueError("The former knowledge base is not compatible with the current version")
             elif self.evolving_version == 2 and not isinstance(
                 factor_knowledge_base,
-                FactorImplementationGraphKnowledgeBase,
+                FactorGraphKnowledgeBase,
             ):
                 raise ValueError("The former knowledge base is not compatible with the current version")
         else:
             factor_knowledge_base = (
-                FactorImplementationGraphKnowledgeBase(
+                FactorGraphKnowledgeBase(
                     init_component_list=component_init_list,
                 )
                 if self.evolving_version == 2
-                else FactorImplementationKnowledgeBaseV1()
+                else FactorKnowledgeBaseV1()
             )
         return factor_knowledge_base
 
-    def generate(self, tasks: List[FactorImplementTask]) -> List[TaskImplementation]:
+    def develop(self, exp: FactorExperiment) -> FactorExperiment:
         # init knowledge base
         factor_knowledge_base = self.load_or_init_knowledge_base(
             former_knowledge_base_path=self.knowledge_base_path,
             component_init_list=[],
         )
         # init rag method
-        self.rag = FactorImplementationGraphRAGStrategy(factor_knowledge_base)
+        self.rag = FactorGraphRAGStrategy(factor_knowledge_base)
 
-        # init indermediate items
-        factor_implementations = FactorEvovlingItem(target_factor_tasks=tasks)
+        # init intermediate items
+        factor_experiment = FactorEvolvingItem.from_experiment(exp)
 
-        self.evolve_agent = RAGEvoAgent(max_loop=self.max_loop, evolving_strategy=self.evolving_strategy, rag=self.rag)
-
-        factor_implementations = self.evolve_agent.multistep_evolve(
-            factor_implementations,
-            self.factor_evaluator,
+        self.evolve_agent = FactorRAGEvoAgent(
+            max_loop=self.max_loop,
+            evolving_strategy=self.evolving_strategy,
+            rag=self.rag,
             with_knowledge=self.with_knowledge,
             with_feedback=self.with_feedback,
             knowledge_self_gen=self.knowledge_self_gen,
         )
 
+        factor_experiment = self.evolve_agent.multistep_evolve(
+            factor_experiment,
+            self.factor_evaluator,
+            filter_final_evo=self.filter_final_evo,
+        )
+
         # save new knowledge base
         if self.new_knowledge_base_path is not None:
             pickle.dump(factor_knowledge_base, open(self.new_knowledge_base_path, "wb"))
-        self.knowledge_base = factor_knowledge_base
-        self.latest_factor_implementations = tasks
-        return factor_implementations
+            logger.info(f"New knowledge base saved to {self.new_knowledge_base_path}")
+        exp.sub_workspace_list = factor_experiment.sub_workspace_list
+        return exp

rdagent/components/coder/factor_coder/CoSTEER/evaluators.py

@@ -0,0 +1,777 @@
+import io
+import json
+import re
+from abc import abstractmethod
+from pathlib import Path
+from typing import List, Tuple
+
+import pandas as pd
+from jinja2 import Environment, StrictUndefined
+
+from rdagent.components.coder.factor_coder.CoSTEER.evolvable_subjects import (
+    FactorEvolvingItem,
+)
+from rdagent.components.coder.factor_coder.factor import FactorTask
+from rdagent.core.conf import RD_AGENT_SETTINGS
+from rdagent.core.evaluation import Evaluator, Feedback
+from rdagent.core.evolving_framework import QueriedKnowledge
+from rdagent.core.experiment import Task, Workspace
+from rdagent.core.prompts import Prompts
+from rdagent.core.utils import multiprocessing_wrapper
+from rdagent.log import rdagent_logger as logger
+from rdagent.oai.llm_conf import LLM_SETTINGS
+from rdagent.oai.llm_utils import APIBackend
+
+evaluate_prompts = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")
+
+
+class FactorEvaluator(Evaluator):
+    # TODO:
+    # I think we should have unified interface for all evaluates, for examples.
+    # So we should adjust the interface of other factors
+    @abstractmethod
+    def evaluate(
+        self,
+        target_task: Task,
+        implementation: Workspace,
+        gt_implementation: Workspace,
+        **kwargs,
+    ) -> Tuple[str, object]:
+        """You can get the dataframe by
+
+        .. code-block:: python
+
+            _, gen_df = implementation.execute()
+            _, gt_df = gt_implementation.execute()
+
+        Returns
+        -------
+        Tuple[str, object]
+            - str: the text-based description of the evaluation result
+            - object: a comparable metric (bool, integer, float ...) None for evaluator with only text-based result
+
+        """
+        raise NotImplementedError("Please implement the `evaluator` method")
+
+    def _get_df(self, gt_implementation: Workspace, implementation: Workspace):
+        if gt_implementation is not None:
+            _, gt_df = gt_implementation.execute()
+            if isinstance(gt_df, pd.Series):
+                gt_df = gt_df.to_frame("gt_factor")
+            if isinstance(gt_df, pd.DataFrame):
+                gt_df = gt_df.sort_index()
+        else:
+            gt_df = None
+
+        _, gen_df = implementation.execute()
+        if isinstance(gen_df, pd.Series):
+            gen_df = gen_df.to_frame("source_factor")
+        if isinstance(gen_df, pd.DataFrame):
+            gen_df = gen_df.sort_index()
+        return gt_df, gen_df
+
+    def __str__(self) -> str:
+        return self.__class__.__name__
+
+
+class FactorCodeEvaluator(FactorEvaluator):
+    def evaluate(
+        self,
+        target_task: FactorTask,
+        implementation: Workspace,
+        execution_feedback: str,
+        factor_value_feedback: str = "",
+        gt_implementation: Workspace = None,
+        **kwargs,
+    ):
+        factor_information = target_task.get_task_information()
+        code = implementation.code
+
+        system_prompt = (
+            Environment(undefined=StrictUndefined)
+            .from_string(evaluate_prompts["evaluator_code_feedback_v1_system"])
+            .render(
+                scenario=(
+                    self.scen.get_scenario_all_desc(target_task)
+                    if self.scen is not None
+                    else "No scenario description."
+                )
+            )
+        )
+
+        execution_feedback_to_render = execution_feedback
+        for _ in range(10):  # 10 times to split the content is enough
+            user_prompt = (
+                Environment(undefined=StrictUndefined)
+                .from_string(
+                    evaluate_prompts["evaluator_code_feedback_v1_user"],
+                )
+                .render(
+                    factor_information=factor_information,
+                    code=code,
+                    execution_feedback=execution_feedback_to_render,
+                    factor_value_feedback=factor_value_feedback,
+                    gt_code=gt_implementation.code if gt_implementation else None,
+                )
+            )
+            if (
+                APIBackend().build_messages_and_calculate_token(
+                    user_prompt=user_prompt,
+                    system_prompt=system_prompt,
+                )
+                > LLM_SETTINGS.chat_token_limit
+            ):
+                execution_feedback_to_render = execution_feedback_to_render[len(execution_feedback_to_render) // 2 :]
+            else:
+                break
+        critic_response = APIBackend().build_messages_and_create_chat_completion(
+            user_prompt=user_prompt,
+            system_prompt=system_prompt,
+            json_mode=False,
+        )
+
+        return critic_response, None
+
+
+class FactorInfEvaluator(FactorEvaluator):
+    def evaluate(
+        self,
+        implementation: Workspace,
+        gt_implementation: Workspace,
+    ) -> Tuple[str, object]:
+        _, gen_df = self._get_df(gt_implementation, implementation)
+        if gen_df is None:
+            return (
+                "The source dataframe is None. Please check the implementation.",
+                False,
+            )
+        INF_count = gen_df.isin([float("inf"), -float("inf")]).sum().sum()
+        if INF_count == 0:
+            return "The source dataframe does not have any infinite values.", True
+        else:
+            return (
+                f"The source dataframe has {INF_count} infinite values. Please check the implementation.",
+                False,
+            )
+
+
+class FactorSingleColumnEvaluator(FactorEvaluator):
+    def evaluate(
+        self,
+        implementation: Workspace,
+        gt_implementation: Workspace,
+    ) -> Tuple[str, object]:
+        _, gen_df = self._get_df(gt_implementation, implementation)
+        if gen_df is None:
+            return (
+                "The source dataframe is None. Please check the implementation.",
+                False,
+            )
+        if len(gen_df.columns) == 1:
+            return "The source dataframe has only one column which is correct.", True
+        else:
+            return (
+                "The source dataframe has more than one column. Please check the implementation. We only evaluate the first column.",
+                False,
+            )
+
+
+class FactorOutputFormatEvaluator(FactorEvaluator):
+    def evaluate(
+        self,
+        implementation: Workspace,
+        gt_implementation: Workspace,
+    ) -> Tuple[str, object]:
+        gt_df, gen_df = self._get_df(gt_implementation, implementation)
+        if gen_df is None:
+            return (
+                "The source dataframe is None. Skip the evaluation of the output format.",
+                False,
+            )
+        buffer = io.StringIO()
+        gen_df.info(buf=buffer)
+        gen_df_info_str = f"The use is currently working on a feature related task.\nThe output dataframe info is:\n{buffer.getvalue()}"
+        system_prompt = (
+            Environment(undefined=StrictUndefined)
+            .from_string(
+                evaluate_prompts["evaluator_output_format_system"],
+            )
+            .render(
+                scenario=(
+                    self.scen.get_scenario_all_desc(implementation.target_task)
+                    if self.scen is not None
+                    else "No scenario description."
+                )
+            )
+        )
+
+        # TODO: with retry_context(retry_n=3, except_list=[KeyError]):
+        max_attempts = 3
+        attempts = 0
+        final_evaluation_dict = None
+
+        while attempts < max_attempts:
+            try:
+                api = APIBackend() if attempts == 0 else APIBackend(use_chat_cache=False)
+                resp = api.build_messages_and_create_chat_completion(
+                    user_prompt=gen_df_info_str, system_prompt=system_prompt, json_mode=True
+                )
+                resp_dict = json.loads(resp)
+                resp_dict["output_format_decision"] = str(resp_dict["output_format_decision"]).lower() in ["true", "1"]
+
+                return (
+                    resp_dict["output_format_feedback"],
+                    resp_dict["output_format_decision"],
+                )
+
+            except json.JSONDecodeError as e:
+                raise ValueError("Failed to decode JSON response from API.") from e
+
+            except KeyError as e:
+                attempts += 1
+                if attempts >= max_attempts:
+                    raise KeyError(
+                        "Response from API is missing 'output_format_decision' or 'output_format_feedback' key after multiple attempts."
+                    ) from e
+
+        return "Failed to evaluate output format after multiple attempts.", False
+
+
+class FactorDatetimeDailyEvaluator(FactorEvaluator):
+    def evaluate(
+        self,
+        implementation: Workspace,
+        gt_implementation: Workspace,
+    ) -> Tuple[str | object]:
+        _, gen_df = self._get_df(gt_implementation, implementation)
+        if gen_df is None:
+            return "The source dataframe is None. Skip the evaluation of the datetime format.", False
+
+        if "datetime" not in gen_df.index.names:
+            return "The source dataframe does not have a datetime index. Please check the implementation.", False
+
+        try:
+            pd.to_datetime(gen_df.index.get_level_values("datetime"))
+        except Exception:
+            return (
+                f"The source dataframe has a datetime index but it is not in the correct format (maybe a regular string or other objects). Please check the implementation.\n The head of the output dataframe is: \n{gen_df.head()}",
+                False,
+            )
+
+        time_diff = pd.to_datetime(gen_df.index.get_level_values("datetime")).to_series().diff().dropna().unique()
+        if pd.Timedelta(minutes=1) in time_diff:
+            return (
+                "The generated dataframe is not daily. The implementation is definitely wrong. Please check the implementation.",
+                False,
+            )
+        return "The generated dataframe is daily.", True
+
+
+class FactorRowCountEvaluator(FactorEvaluator):
+    def evaluate(
+        self,
+        implementation: Workspace,
+        gt_implementation: Workspace,
+    ) -> Tuple[str, object]:
+        gt_df, gen_df = self._get_df(gt_implementation, implementation)
+        if gen_df is None:
+            return (
+                "The source dataframe is None. Please check the implementation.",
+                False,
+            )
+        ratio = min(len(gen_df), len(gt_df)) / max(len(gen_df), len(gt_df))
+        return (
+            (
+                f"The ratio of rows count in the source dataframe to the ground truth dataframe is {ratio:.2f}. "
+                + "Please verify the implementation. "
+                if ratio <= 0.99
+                else ""
+            ),
+            ratio,
+        )
+
+
+class FactorIndexEvaluator(FactorEvaluator):
+    def evaluate(
+        self,
+        implementation: Workspace,
+        gt_implementation: Workspace,
+    ) -> Tuple[str, object]:
+        gt_df, gen_df = self._get_df(gt_implementation, implementation)
+        if gen_df is None:
+            return (
+                "The source dataframe is None. Please check the implementation.",
+                False,
+            )
+        gen_index_set, gt_index_set = set(gen_df.index), set(gt_df.index)
+        similarity = len(gen_index_set.intersection(gt_index_set)) / len(gen_index_set.union(gt_index_set))
+        return (
+            (
+                f"The source dataframe and the ground truth dataframe have different index with a similarity of {similarity:.2%}. The similarity is calculated by the number of shared indices divided by the union indices. "
+                + "Please check the implementation."
+                if similarity <= 0.99
+                else ""
+            ),
+            similarity,
+        )
+
+
+class FactorMissingValuesEvaluator(FactorEvaluator):
+    def evaluate(
+        self,
+        implementation: Workspace,
+        gt_implementation: Workspace,
+    ) -> Tuple[str, object]:
+        gt_df, gen_df = self._get_df(gt_implementation, implementation)
+        if gen_df is None:
+            return (
+                "The source dataframe is None. Please check the implementation.",
+                False,
+            )
+        if gen_df.isna().sum().sum() == gt_df.isna().sum().sum():
+            return "Both dataframes have the same missing values.", True
+        else:
+            return (
+                f"The dataframes do not have the same missing values. The source dataframe has {gen_df.isna().sum().sum()} missing values, while the ground truth dataframe has {gt_df.isna().sum().sum()} missing values. Please check the implementation.",
+                False,
+            )
+
+
+class FactorEqualValueRatioEvaluator(FactorEvaluator):
+    def evaluate(
+        self,
+        implementation: Workspace,
+        gt_implementation: Workspace,
+    ) -> Tuple[str, object]:
+        gt_df, gen_df = self._get_df(gt_implementation, implementation)
+        if gen_df is None:
+            return (
+                "The source dataframe is None. Please check the implementation.",
+                -1,
+            )
+        try:
+            close_values = gen_df.sub(gt_df).abs().lt(1e-6)
+            result_int = close_values.astype(int)
+            pos_num = result_int.sum().sum()
+            acc_rate = pos_num / close_values.size
+        except:
+            close_values = gen_df
+        if close_values.all().iloc[0]:
+            return (
+                "All values in the dataframes are equal within the tolerance of 1e-6.",
+                acc_rate,
+            )
+        else:
+            return (
+                "Some values differ by more than the tolerance of 1e-6. Check for rounding errors or differences in the calculation methods.",
+                acc_rate,
+            )
+
+
+class FactorCorrelationEvaluator(FactorEvaluator):
+    def __init__(self, hard_check: bool, *args, **kwargs) -> None:
+        super().__init__(*args, **kwargs)
+        self.hard_check = hard_check
+
+    def evaluate(
+        self,
+        implementation: Workspace,
+        gt_implementation: Workspace,
+    ) -> Tuple[str, object]:
+        gt_df, gen_df = self._get_df(gt_implementation, implementation)
+        if gen_df is None:
+            return (
+                "The source dataframe is None. Please check the implementation.",
+                False,
+            )
+        concat_df = pd.concat([gen_df, gt_df], axis=1)
+        concat_df.columns = ["source", "gt"]
+        ic = concat_df.groupby("datetime").apply(lambda df: df["source"].corr(df["gt"])).dropna().mean()
+        ric = (
+            concat_df.groupby("datetime")
+            .apply(lambda df: df["source"].corr(df["gt"], method="spearman"))
+            .dropna()
+            .mean()
+        )
+
+        if self.hard_check:
+            if ic > 0.99 and ric > 0.99:
+                return (
+                    f"The dataframes are highly correlated. The ic is {ic:.6f} and the rankic is {ric:.6f}.",
+                    True,
+                )
+            else:
+                return (
+                    f"The dataframes are not sufficiently high correlated. The ic is {ic:.6f} and the rankic is {ric:.6f}. Investigate the factors that might be causing the discrepancies and ensure that the logic of the factor calculation is consistent.",
+                    False,
+                )
+        else:
+            return f"The ic is ({ic:.6f}) and the rankic is ({ric:.6f}).", ic
+
+
+class FactorValueEvaluator(FactorEvaluator):
+    def evaluate(
+        self,
+        implementation: Workspace,
+        gt_implementation: Workspace,
+        version: int = 1,  # 1 for qlib factors and 2 for kaggle factors
+        **kwargs,
+    ) -> Tuple:
+        conclusions = []
+
+        # Initialize result variables
+        row_result = 0
+        index_result = 0
+        output_format_result = None
+        equal_value_ratio_result = 0
+        high_correlation_result = False
+        row_result = None
+
+        # Check if both dataframe has only one columns Mute this since factor task might generate more than one columns now
+        if version == 1:
+            feedback_str, _ = FactorSingleColumnEvaluator(self.scen).evaluate(implementation, gt_implementation)
+            conclusions.append(feedback_str)
+        elif version == 2:
+            input_shape = self.scen.input_shape
+            _, gen_df = self._get_df(gt_implementation, implementation)
+            if gen_df.shape[-1] > input_shape[-1]:
+                conclusions.append(
+                    "Output dataframe has more columns than input feature which is not acceptable in feature processing tasks. Please check the implementation to avoid generating too many columns. Consider this implementation as a failure."
+                )
+
+        feedback_str, inf_evaluate_res = FactorInfEvaluator(self.scen).evaluate(implementation, gt_implementation)
+        conclusions.append(feedback_str)
+
+        # Check if the index of the dataframe is ("datetime", "instrument")
+        feedback_str, _ = FactorOutputFormatEvaluator(self.scen).evaluate(implementation, gt_implementation)
+        conclusions.append(feedback_str)
+        if version == 1:
+            feedback_str, daily_check_result = FactorDatetimeDailyEvaluator(self.scen).evaluate(
+                implementation, gt_implementation
+            )
+            conclusions.append(feedback_str)
+        else:
+            daily_check_result = None
+
+        # Check dataframe format
+        if gt_implementation is not None:
+            feedback_str, row_result = FactorRowCountEvaluator(self.scen).evaluate(implementation, gt_implementation)
+            conclusions.append(feedback_str)
+
+            feedback_str, index_result = FactorIndexEvaluator(self.scen).evaluate(implementation, gt_implementation)
+            conclusions.append(feedback_str)
+
+            feedback_str, output_format_result = FactorMissingValuesEvaluator(self.scen).evaluate(
+                implementation, gt_implementation
+            )
+            conclusions.append(feedback_str)
+
+            feedback_str, equal_value_ratio_result = FactorEqualValueRatioEvaluator(self.scen).evaluate(
+                implementation, gt_implementation
+            )
+            conclusions.append(feedback_str)
+
+            if index_result > 0.99:
+                feedback_str, high_correlation_result = FactorCorrelationEvaluator(
+                    hard_check=True, scen=self.scen
+                ).evaluate(implementation, gt_implementation)
+            else:
+                high_correlation_result = False
+                feedback_str = "The source dataframe and the ground truth dataframe have different index. Give up comparing the values and correlation because it's useless"
+            conclusions.append(feedback_str)
+
+        # Combine all conclusions into a single string
+        conclusion_str = "\n".join(conclusions)
+
+        if gt_implementation is not None and (equal_value_ratio_result > 0.99) or high_correlation_result:
+            decision_from_value_check = True
+        elif (
+            row_result is not None
+            and row_result <= 0.99
+            or output_format_result is False
+            or daily_check_result is False
+            or inf_evaluate_res is False
+        ):
+            decision_from_value_check = False
+        else:
+            decision_from_value_check = None
+        return conclusion_str, decision_from_value_check
+
+
+class FactorFinalDecisionEvaluator(Evaluator):
+    def evaluate(
+        self,
+        target_task: FactorTask,
+        execution_feedback: str,
+        value_feedback: str,
+        code_feedback: str,
+        **kwargs,
+    ) -> Tuple:
+        system_prompt = (
+            Environment(undefined=StrictUndefined)
+            .from_string(evaluate_prompts["evaluator_final_decision_v1_system"])
+            .render(
+                scenario=(
+                    self.scen.get_scenario_all_desc(target_task)
+                    if self.scen is not None
+                    else "No scenario description."
+                )
+            )
+        )
+        execution_feedback_to_render = execution_feedback
+
+        for _ in range(10):  # 10 times to split the content is enough
+            user_prompt = (
+                Environment(undefined=StrictUndefined)
+                .from_string(
+                    evaluate_prompts["evaluator_final_decision_v1_user"],
+                )
+                .render(
+                    factor_information=target_task.get_task_information(),
+                    execution_feedback=execution_feedback_to_render,
+                    code_feedback=code_feedback,
+                    factor_value_feedback=(
+                        value_feedback
+                        if value_feedback is not None
+                        else "No Ground Truth Value provided, so no evaluation on value is performed."
+                    ),
+                )
+            )
+            if (
+                APIBackend().build_messages_and_calculate_token(
+                    user_prompt=user_prompt,
+                    system_prompt=system_prompt,
+                )
+                > LLM_SETTINGS.chat_token_limit
+            ):
+                execution_feedback_to_render = execution_feedback_to_render[len(execution_feedback_to_render) // 2 :]
+            else:
+                break
+
+        # TODO:  with retry_context(retry_n=3, except_list=[KeyError]):
+        final_evaluation_dict = None
+        attempts = 0
+        max_attempts = 3
+
+        while attempts < max_attempts:
+            try:
+                api = APIBackend() if attempts == 0 else APIBackend(use_chat_cache=False)
+                final_evaluation_dict = json.loads(
+                    api.build_messages_and_create_chat_completion(
+                        user_prompt=user_prompt,
+                        system_prompt=system_prompt,
+                        json_mode=True,
+                        seed=attempts,  # in case of useless retrying when cache enabled.
+                    ),
+                )
+                final_decision = final_evaluation_dict["final_decision"]
+                final_feedback = final_evaluation_dict["final_feedback"]
+
+                final_decision = str(final_decision).lower() in ["true", "1"]
+                return final_decision, final_feedback
+
+            except json.JSONDecodeError as e:
+                raise ValueError("Failed to decode JSON response from API.") from e
+            except KeyError as e:
+                attempts += 1
+                if attempts >= max_attempts:
+                    raise KeyError(
+                        "Response from API is missing 'final_decision' or 'final_feedback' key after multiple attempts."
+                    ) from e
+
+        return None, None
+
+
+class FactorSingleFeedback:
+    """This class is a feedback to single implementation which is generated from an evaluator."""
+
+    def __init__(
+        self,
+        execution_feedback: str = None,
+        value_generated_flag: bool = False,
+        code_feedback: str = None,
+        factor_value_feedback: str = None,
+        final_decision: bool = None,
+        final_feedback: str = None,
+        final_decision_based_on_gt: bool = None,
+    ) -> None:
+        self.execution_feedback = execution_feedback
+        self.value_generated_flag = value_generated_flag
+        self.code_feedback = code_feedback
+        self.factor_value_feedback = factor_value_feedback
+        self.final_decision = final_decision
+        self.final_feedback = final_feedback
+        self.final_decision_based_on_gt = final_decision_based_on_gt
+
+    def __str__(self) -> str:
+        return f"""------------------Factor Execution Feedback------------------
+{self.execution_feedback}
+------------------Factor Code Feedback------------------
+{self.code_feedback}
+------------------Factor Value Feedback------------------
+{self.factor_value_feedback}
+------------------Factor Final Feedback------------------
+{self.final_feedback}
+------------------Factor Final Decision------------------
+This implementation is {'SUCCESS' if self.final_decision else 'FAIL'}.
+"""
+
+
+class FactorMultiFeedback(
+    Feedback,
+    List[FactorSingleFeedback],
+):
+    """Feedback contains a list, each element is the corresponding feedback for each factor implementation."""
+
+
+class FactorEvaluatorForCoder(FactorEvaluator):
+    """This class is the v1 version of evaluator for a single factor implementation.
+    It calls several evaluators in share modules to evaluate the factor implementation.
+    """
+
+    def __init__(self, *args, **kwargs) -> None:
+        super().__init__(*args, **kwargs)
+        self.value_evaluator = FactorValueEvaluator(self.scen)
+        self.code_evaluator = FactorCodeEvaluator(self.scen)
+        self.final_decision_evaluator = FactorFinalDecisionEvaluator(self.scen)
+
+    def evaluate(
+        self,
+        target_task: FactorTask,
+        implementation: Workspace,
+        gt_implementation: Workspace = None,
+        queried_knowledge: QueriedKnowledge = None,
+        **kwargs,
+    ) -> FactorSingleFeedback:
+        if implementation is None:
+            return None
+
+        target_task_information = target_task.get_task_information()
+        if (
+            queried_knowledge is not None
+            and target_task_information in queried_knowledge.success_task_to_knowledge_dict
+        ):
+            return queried_knowledge.success_task_to_knowledge_dict[target_task_information].feedback
+        elif queried_knowledge is not None and target_task_information in queried_knowledge.failed_task_info_set:
+            return FactorSingleFeedback(
+                execution_feedback="This task has failed too many times, skip implementation.",
+                value_generated_flag=False,
+                code_feedback="This task has failed too many times, skip code evaluation.",
+                factor_value_feedback="This task has failed too many times, skip value evaluation.",
+                final_decision=False,
+                final_feedback="This task has failed too many times, skip final decision evaluation.",
+                final_decision_based_on_gt=False,
+            )
+        else:
+            factor_feedback = FactorSingleFeedback()
+
+            # 1. Get factor execution feedback to generated implementation and remove the long list of numbers in execution feedback
+            (
+                execution_feedback,
+                gen_df,
+            ) = implementation.execute()
+
+            execution_feedback = re.sub(r"(?<=\D)(,\s+-?\d+\.\d+){50,}(?=\D)", ", ", execution_feedback)
+            factor_feedback.execution_feedback = "\n".join(
+                [line for line in execution_feedback.split("\n") if "warning" not in line.lower()]
+            )
+
+            # 2. Get factor value feedback
+            if gen_df is None:
+                factor_feedback.factor_value_feedback = "No factor value generated, skip value evaluation."
+                factor_feedback.value_generated_flag = False
+                decision_from_value_check = None
+            else:
+                factor_feedback.value_generated_flag = True
+                (
+                    factor_feedback.factor_value_feedback,
+                    decision_from_value_check,
+                ) = self.value_evaluator.evaluate(
+                    implementation=implementation, gt_implementation=gt_implementation, version=target_task.version
+                )
+
+            factor_feedback.final_decision_based_on_gt = gt_implementation is not None
+
+            if decision_from_value_check is not None and decision_from_value_check is True:
+                # To avoid confusion, when same_value_or_high_correlation is True, we do not need code feedback
+                factor_feedback.code_feedback = "Final decision is True and there are no code critics."
+                factor_feedback.final_decision = decision_from_value_check
+                factor_feedback.final_feedback = "Value evaluation passed, skip final decision evaluation."
+            elif decision_from_value_check is not None and decision_from_value_check is False:
+                factor_feedback.code_feedback, _ = self.code_evaluator.evaluate(
+                    target_task=target_task,
+                    implementation=implementation,
+                    execution_feedback=factor_feedback.execution_feedback,
+                    factor_value_feedback=factor_feedback.factor_value_feedback,
+                    gt_implementation=gt_implementation,
+                )
+                factor_feedback.final_decision = decision_from_value_check
+                factor_feedback.final_feedback = "Value evaluation failed, skip final decision evaluation."
+            else:
+                factor_feedback.code_feedback, _ = self.code_evaluator.evaluate(
+                    target_task=target_task,
+                    implementation=implementation,
+                    execution_feedback=factor_feedback.execution_feedback,
+                    factor_value_feedback=factor_feedback.factor_value_feedback,
+                    gt_implementation=gt_implementation,
+                )
+                (
+                    factor_feedback.final_decision,
+                    factor_feedback.final_feedback,
+                ) = self.final_decision_evaluator.evaluate(
+                    target_task=target_task,
+                    execution_feedback=factor_feedback.execution_feedback,
+                    value_feedback=factor_feedback.factor_value_feedback,
+                    code_feedback=factor_feedback.code_feedback,
+                )
+            return factor_feedback
+
+
+class FactorMultiEvaluator(Evaluator):
+    def __init__(self, single_evaluator, *args, **kwargs) -> None:
+        super().__init__(*args, **kwargs)
+        self.single_factor_implementation_evaluator = single_evaluator
+
+    def evaluate(
+        self,
+        evo: FactorEvolvingItem,
+        queried_knowledge: QueriedKnowledge = None,
+        **kwargs,
+    ) -> FactorMultiFeedback:
+        multi_implementation_feedback = multiprocessing_wrapper(
+            [
+                (
+                    self.single_factor_implementation_evaluator.evaluate,
+                    (
+                        evo.sub_tasks[index],
+                        evo.sub_workspace_list[index],
+                        evo.sub_gt_implementations[index] if evo.sub_gt_implementations is not None else None,
+                        queried_knowledge,
+                    ),
+                )
+                for index in range(len(evo.sub_tasks))
+            ],
+            n=RD_AGENT_SETTINGS.multi_proc_n,
+        )
+
+        final_decision = [
+            None if single_feedback is None else single_feedback.final_decision
+            for single_feedback in multi_implementation_feedback
+        ]
+        logger.info(f"Final decisions: {final_decision} True count: {final_decision.count(True)}")
+
+        for index in range(len(evo.sub_tasks)):
+            if final_decision[index]:
+                evo.sub_tasks[index].factor_implementation = True
+
+        return multi_implementation_feedback
+
+
+# TODO:
+def shorten_prompt(tpl: str, render_kwargs: dict, shorten_key: str, max_trail: int = 10) -> str:
+    """When the prompt is too long. We have to shorten it.
+    But we should not truncate the prompt directly, so we should find the key we want to shorten and then shorten it.
+    """
+    # TODO: this should replace most of code in
+    # - FactorFinalDecisionEvaluator.evaluate
+    # - FactorCodeEvaluator.evaluate

rdagent/components/coder/factor_coder/CoSTEER/evolvable_subjects.py

@@ -0,0 +1,37 @@
+from rdagent.components.coder.factor_coder.factor import (
+    FactorExperiment,
+    FactorFBWorkspace,
+    FactorTask,
+)
+from rdagent.core.evolving_framework import EvolvableSubjects
+from rdagent.log import rdagent_logger as logger
+
+
+class FactorEvolvingItem(FactorExperiment, EvolvableSubjects):
+    """
+    Intermediate item of factor implementation.
+    """
+
+    def __init__(
+        self,
+        sub_tasks: list[FactorTask],
+        sub_gt_implementations: list[FactorFBWorkspace] = None,
+    ):
+        FactorExperiment.__init__(self, sub_tasks=sub_tasks)
+        self.corresponding_selection: list = None
+        if sub_gt_implementations is not None and len(
+            sub_gt_implementations,
+        ) != len(self.sub_tasks):
+            self.sub_gt_implementations = None
+            logger.warning(
+                "The length of sub_gt_implementations is not equal to the length of sub_tasks, set sub_gt_implementations to None",
+            )
+        else:
+            self.sub_gt_implementations = sub_gt_implementations
+
+    @classmethod
+    def from_experiment(cls, exp: FactorExperiment) -> "FactorExperiment":
+        ei = cls(sub_tasks=exp.sub_tasks)
+        ei.based_experiments = exp.based_experiments
+        ei.experiment_workspace = exp.experiment_workspace
+        return ei

rdagent/components/coder/factor_coder/CoSTEER/evolving_agent.py

@@ -0,0 +1,19 @@
+from rdagent.components.coder.factor_coder.CoSTEER.evaluators import FactorMultiFeedback
+from rdagent.components.coder.factor_coder.CoSTEER.evolvable_subjects import (
+    FactorEvolvingItem,
+)
+from rdagent.core.evaluation import Feedback
+from rdagent.core.evolving_agent import RAGEvoAgent
+from rdagent.core.evolving_framework import EvolvableSubjects
+
+
+class FactorRAGEvoAgent(RAGEvoAgent):
+    def filter_evolvable_subjects_by_feedback(self, evo: EvolvableSubjects, feedback: Feedback) -> EvolvableSubjects:
+        assert isinstance(evo, FactorEvolvingItem)
+        assert isinstance(feedback, list)
+        assert len(evo.sub_workspace_list) == len(feedback)
+
+        for index in range(len(evo.sub_workspace_list)):
+            if feedback[index] and not feedback[index].final_decision:
+                evo.sub_workspace_list[index].clear()
+        return evo

rdagent/components/coder/factor_coder/CoSTEER/evolving_strategy.py

@@ -6,42 +6,29 @@ from copy import deepcopy
 from pathlib import Path
 from typing import TYPE_CHECKING
 
-from jinja2 import Template
+from jinja2 import Environment, StrictUndefined
 
+from rdagent.components.coder.factor_coder.config import FACTOR_IMPLEMENT_SETTINGS
+from rdagent.components.coder.factor_coder.CoSTEER.evolvable_subjects import (
+    FactorEvolvingItem,
+)
+from rdagent.components.coder.factor_coder.CoSTEER.scheduler import (
+    LLMSelect,
+    RandomSelect,
+)
+from rdagent.components.coder.factor_coder.factor import FactorFBWorkspace, FactorTask
 from rdagent.core.conf import RD_AGENT_SETTINGS
 from rdagent.core.evolving_framework import EvolvingStrategy, QueriedKnowledge
-from rdagent.oai.llm_utils import APIBackend
-from rdagent.factor_implementation.share_modules.factor_implementation_config import (
-    FACTOR_IMPLEMENT_SETTINGS,
-)
-
-from rdagent.core.task import (
-    TaskImplementation,
-)
+from rdagent.core.experiment import Workspace
 from rdagent.core.prompts import Prompts
-
-from pathlib import Path
-
-from rdagent.factor_implementation.evolving.scheduler import (
-    RandomSelect,
-    LLMSelect,
-)
-
-from rdagent.factor_implementation.share_modules.factor_implementation_utils import get_data_folder_intro
-from rdagent.oai.llm_utils import APIBackend
-
 from rdagent.core.utils import multiprocessing_wrapper
-
-from rdagent.factor_implementation.evolving.factor import (
-    FactorImplementTask,
-    FactorEvovlingItem,
-    FileBasedFactorImplementation,
-)
+from rdagent.oai.llm_conf import LLM_SETTINGS
+from rdagent.oai.llm_utils import APIBackend
 
 if TYPE_CHECKING:
-    from rdagent.factor_implementation.evolving.knowledge_management import (
-        FactorImplementationQueriedKnowledge,
-        FactorImplementationQueriedKnowledgeV1,
+    from rdagent.components.coder.factor_coder.CoSTEER.knowledge_management import (
+        FactorQueriedKnowledge,
+        FactorQueriedKnowledgeV1,
     )
 
 implement_prompts = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")
@@ -51,27 +38,24 @@ class MultiProcessEvolvingStrategy(EvolvingStrategy):
     @abstractmethod
     def implement_one_factor(
         self,
-        target_task: FactorImplementTask,
+        target_task: FactorTask,
         queried_knowledge: QueriedKnowledge = None,
-    ) -> TaskImplementation:
+    ) -> Workspace:
         raise NotImplementedError
 
     def evolve(
         self,
         *,
-        evo: FactorEvovlingItem,
-        queried_knowledge: FactorImplementationQueriedKnowledge | None = None,
+        evo: FactorEvolvingItem,
+        queried_knowledge: FactorQueriedKnowledge | None = None,
         **kwargs,
-    ) -> FactorEvovlingItem:
-        self.num_loop += 1
-        new_evo = deepcopy(evo)
-
+    ) -> FactorEvolvingItem:
         # 1.找出需要evolve的factor
         to_be_finished_task_index = []
-        for index, target_factor_task in enumerate(new_evo.target_factor_tasks):
-            target_factor_task_desc = target_factor_task.get_factor_information()
+        for index, target_factor_task in enumerate(evo.sub_tasks):
+            target_factor_task_desc = target_factor_task.get_task_information()
             if target_factor_task_desc in queried_knowledge.success_task_to_knowledge_dict:
-                new_evo.corresponding_implementations[index] = queried_knowledge.success_task_to_knowledge_dict[
+                evo.sub_workspace_list[index] = queried_knowledge.success_task_to_knowledge_dict[
                     target_factor_task_desc
                 ].implementation
             elif (
@@ -82,53 +66,50 @@ class MultiProcessEvolvingStrategy(EvolvingStrategy):
 
         # 2. 选择selection方法
         # if the number of factors to be implemented is larger than the limit, we need to select some of them
-        if FACTOR_IMPLEMENT_SETTINGS.select_ratio < 1:
-            # if the number of loops is equal to the select_loop, we need to select some of them
-            implementation_factors_per_round = int(
-                FACTOR_IMPLEMENT_SETTINGS.select_ratio * len(to_be_finished_task_index)
-            )
+
+        if FACTOR_IMPLEMENT_SETTINGS.select_threshold < len(to_be_finished_task_index):
+            # Select a fixed number of factors if the total exceeds the threshold
             if FACTOR_IMPLEMENT_SETTINGS.select_method == "random":
                 to_be_finished_task_index = RandomSelect(
                     to_be_finished_task_index,
-                    implementation_factors_per_round,
+                    FACTOR_IMPLEMENT_SETTINGS.select_threshold,
                 )
 
             if FACTOR_IMPLEMENT_SETTINGS.select_method == "scheduler":
                 to_be_finished_task_index = LLMSelect(
                     to_be_finished_task_index,
-                    implementation_factors_per_round,
-                    new_evo,
+                    FACTOR_IMPLEMENT_SETTINGS.select_threshold,
+                    evo,
                     queried_knowledge.former_traces,
+                    self.scen,
                 )
 
         result = multiprocessing_wrapper(
             [
-                (self.implement_one_factor, (new_evo.target_factor_tasks[target_index], queried_knowledge))
+                (self.implement_one_factor, (evo.sub_tasks[target_index], queried_knowledge))
                 for target_index in to_be_finished_task_index
             ],
-            n=FACTOR_IMPLEMENT_SETTINGS.evo_multi_proc_n,
+            n=RD_AGENT_SETTINGS.multi_proc_n,
         )
+        from rdagent.components.coder.factor_coder.factor import FactorFBWorkspace
 
         for index, target_index in enumerate(to_be_finished_task_index):
-            new_evo.corresponding_implementations[target_index] = result[index]
+            if evo.sub_workspace_list[target_index] is None:
+                evo.sub_workspace_list[target_index] = FactorFBWorkspace(target_task=evo.sub_tasks[target_index])
+            evo.sub_workspace_list[target_index].inject_code(**{"factor.py": result[index]})
 
-        # for target_index in to_be_finished_task_index:
-        #     new_evo.corresponding_implementations[target_index] = self.implement_one_factor(
-        #         new_evo.target_factor_tasks[target_index], queried_knowledge
-        #     )
+        evo.corresponding_selection = to_be_finished_task_index
 
-        new_evo.corresponding_selection = to_be_finished_task_index
-
-        return new_evo
+        return evo
 
 
 class FactorEvolvingStrategy(MultiProcessEvolvingStrategy):
     def implement_one_factor(
         self,
-        target_task: FactorImplementTask,
-        queried_knowledge: FactorImplementationQueriedKnowledgeV1 = None,
-    ) -> TaskImplementation:
-        factor_information_str = target_task.get_factor_information()
+        target_task: FactorTask,
+        queried_knowledge: FactorQueriedKnowledgeV1 = None,
+    ) -> str:
+        factor_information_str = target_task.get_task_information()
 
         if queried_knowledge is not None and factor_information_str in queried_knowledge.success_task_to_knowledge_dict:
             return queried_knowledge.success_task_to_knowledge_dict[factor_information_str].implementation
@@ -148,25 +129,31 @@ class FactorEvolvingStrategy(MultiProcessEvolvingStrategy):
 
             queried_former_failed_knowledge_to_render = queried_former_failed_knowledge
 
-            system_prompt = Template(
-                implement_prompts["evolving_strategy_factor_implementation_v1_system"],
-            ).render(
-                data_info=get_data_folder_intro(),
-                queried_former_failed_knowledge=queried_former_failed_knowledge_to_render,
+            system_prompt = (
+                Environment(undefined=StrictUndefined)
+                .from_string(
+                    implement_prompts["evolving_strategy_factor_implementation_v1_system"],
+                )
+                .render(
+                    scenario=self.scen.get_scenario_all_desc(target_task),
+                    queried_former_failed_knowledge=queried_former_failed_knowledge_to_render,
+                )
             )
-            session = APIBackend(use_chat_cache=False).build_chat_session(
+            session = APIBackend(use_chat_cache=FACTOR_IMPLEMENT_SETTINGS.coder_use_cache).build_chat_session(
                 session_system_prompt=system_prompt,
             )
 
             queried_similar_successful_knowledge_to_render = queried_similar_successful_knowledge
-            while True:
+            for _ in range(10):  # max attempt to reduce the length of user_prompt
                 user_prompt = (
-                    Template(
+                    Environment(undefined=StrictUndefined)
+                    .from_string(
                         implement_prompts["evolving_strategy_factor_implementation_v1_user"],
                     )
                     .render(
                         factor_information_str=factor_information_str,
                         queried_similar_successful_knowledge=queried_similar_successful_knowledge_to_render,
+                        queried_former_failed_knowledge=queried_former_failed_knowledge_to_render,
                     )
                     .strip("\n")
                 )
@@ -174,7 +161,7 @@ class FactorEvolvingStrategy(MultiProcessEvolvingStrategy):
                     session.build_chat_completion_message_and_calculate_token(
                         user_prompt,
                     )
-                    < RD_AGENT_SETTINGS.chat_token_limit
+                    < LLM_SETTINGS.chat_token_limit
                 ):
                     break
                 elif len(queried_former_failed_knowledge_to_render) > 1:
@@ -188,28 +175,24 @@ class FactorEvolvingStrategy(MultiProcessEvolvingStrategy):
                     json_mode=True,
                 ),
             )["code"]
-            # ast.parse(code)
-            factor_implementation = FileBasedFactorImplementation(
-                target_task,
-                code,
-            )
 
-            return factor_implementation
+            return code
 
 
 class FactorEvolvingStrategyWithGraph(MultiProcessEvolvingStrategy):
-    def __init__(self) -> None:
+    def __init__(self, *args, **kwargs) -> None:
+        super().__init__(*args, **kwargs)
         self.num_loop = 0
         self.haveSelected = False
 
     def implement_one_factor(
         self,
-        target_task: FactorImplementTask,
+        target_task: FactorTask,
         queried_knowledge,
-    ) -> TaskImplementation:
+    ) -> str:
         error_summary = FACTOR_IMPLEMENT_SETTINGS.v2_error_summary
         # 1. 提取因子的背景信息
-        target_factor_task_information = target_task.get_factor_information()
+        target_factor_task_information = target_task.get_task_information()
 
         # 2. 检查该因子是否需要继续做（是否已经作对，是否做错太多）
         if (
@@ -220,7 +203,6 @@ class FactorEvolvingStrategyWithGraph(MultiProcessEvolvingStrategy):
         elif queried_knowledge is not None and target_factor_task_information in queried_knowledge.failed_task_info_set:
             return None
         else:
-
             # 3. 取出knowledge里面的经验数据（similar success、similar error、former_trace）
             queried_similar_component_knowledge = (
                 queried_knowledge.component_with_success_task[target_factor_task_information]
@@ -235,19 +217,29 @@ class FactorEvolvingStrategyWithGraph(MultiProcessEvolvingStrategy):
             )  # A dict, {{error_type:[[error_imp_knowledge, success_imp_knowledge],...]},...}
 
             queried_former_failed_knowledge = (
-                queried_knowledge.former_traces[target_factor_task_information] if queried_knowledge is not None else []
+                queried_knowledge.former_traces[target_factor_task_information][0]
+                if queried_knowledge is not None
+                else []
             )
 
             queried_former_failed_knowledge_to_render = queried_former_failed_knowledge
 
-            system_prompt = Template(
-                implement_prompts["evolving_strategy_factor_implementation_v1_system"],
-            ).render(
-                data_info=get_data_folder_intro(),
-                queried_former_failed_knowledge=queried_former_failed_knowledge_to_render,
+            latest_attempt_to_latest_successful_execution = queried_knowledge.former_traces[
+                target_factor_task_information
+            ][1]
+
+            system_prompt = (
+                Environment(undefined=StrictUndefined)
+                .from_string(
+                    implement_prompts["evolving_strategy_factor_implementation_v1_system"],
+                )
+                .render(
+                    scenario=self.scen.get_scenario_all_desc(target_task),
+                    queried_former_failed_knowledge=queried_former_failed_knowledge_to_render,
+                )
             )
 
-            session = APIBackend(use_chat_cache=False).build_chat_session(
+            session = APIBackend(use_chat_cache=FACTOR_IMPLEMENT_SETTINGS.coder_use_cache).build_chat_session(
                 session_system_prompt=system_prompt,
             )
 
@@ -255,17 +247,18 @@ class FactorEvolvingStrategyWithGraph(MultiProcessEvolvingStrategy):
             queried_similar_error_knowledge_to_render = queried_similar_error_knowledge
             error_summary_critics = ""
             # 动态地防止prompt超长
-            while True:
+            for _ in range(10):  # max attempt to reduce the length of user_prompt
                 # 总结error（可选）
                 if (
                     error_summary
                     and len(queried_similar_error_knowledge_to_render) != 0
                     and len(queried_former_failed_knowledge_to_render) != 0
                 ):
-
                     error_summary_system_prompt = (
-                        Template(implement_prompts["evolving_strategy_error_summary_v2_system"])
+                        Environment(undefined=StrictUndefined)
+                        .from_string(implement_prompts["evolving_strategy_error_summary_v2_system"])
                         .render(
+                            scenario=self.scen.get_scenario_all_desc(target_task),
                             factor_information_str=target_factor_task_information,
                             code_and_feedback=queried_former_failed_knowledge_to_render[
                                 -1
@@ -273,12 +266,15 @@ class FactorEvolvingStrategyWithGraph(MultiProcessEvolvingStrategy):
                         )
                         .strip("\n")
                     )
-                    session_summary = APIBackend(use_chat_cache=False).build_chat_session(
+                    session_summary = APIBackend(
+                        use_chat_cache=FACTOR_IMPLEMENT_SETTINGS.coder_use_cache
+                    ).build_chat_session(
                         session_system_prompt=error_summary_system_prompt,
                     )
-                    while True:
+                    for _ in range(10):  # max attempt to reduce the length of error_summary_user_prompt
                         error_summary_user_prompt = (
-                            Template(implement_prompts["evolving_strategy_error_summary_v2_user"])
+                            Environment(undefined=StrictUndefined)
+                            .from_string(implement_prompts["evolving_strategy_error_summary_v2_user"])
                             .render(
                                 queried_similar_component_knowledge=queried_similar_component_knowledge_to_render,
                             )
@@ -286,7 +282,7 @@ class FactorEvolvingStrategyWithGraph(MultiProcessEvolvingStrategy):
                         )
                         if (
                             session_summary.build_chat_completion_message_and_calculate_token(error_summary_user_prompt)
-                            < RD_AGENT_SETTINGS.chat_token_limit
+                            < LLM_SETTINGS.chat_token_limit
                         ):
                             break
                         elif len(queried_similar_error_knowledge_to_render) > 0:
@@ -297,7 +293,8 @@ class FactorEvolvingStrategyWithGraph(MultiProcessEvolvingStrategy):
                     )
                 # 构建user_prompt。开始写代码
                 user_prompt = (
-                    Template(
+                    Environment(undefined=StrictUndefined)
+                    .from_string(
                         implement_prompts["evolving_strategy_factor_implementation_v2_user"],
                     )
                     .render(
@@ -306,6 +303,7 @@ class FactorEvolvingStrategyWithGraph(MultiProcessEvolvingStrategy):
                         queried_similar_error_knowledge=queried_similar_error_knowledge_to_render,
                         error_summary=error_summary,
                         error_summary_critics=error_summary_critics,
+                        latest_attempt_to_latest_successful_execution=latest_attempt_to_latest_successful_execution,
                     )
                     .strip("\n")
                 )
@@ -313,7 +311,7 @@ class FactorEvolvingStrategyWithGraph(MultiProcessEvolvingStrategy):
                     session.build_chat_completion_message_and_calculate_token(
                         user_prompt,
                     )
-                    < RD_AGENT_SETTINGS.chat_token_limit
+                    < LLM_SETTINGS.chat_token_limit
                 ):
                     break
                 elif len(queried_former_failed_knowledge_to_render) > 1:
@@ -330,5 +328,4 @@ class FactorEvolvingStrategyWithGraph(MultiProcessEvolvingStrategy):
                 json_mode=True,
             )
             code = json.loads(response)["code"]
-            factor_implementation = FileBasedFactorImplementation(target_task, code)
-            return factor_implementation
+            return code

rdagent/components/coder/factor_coder/CoSTEER/knowledge_management.py

@@ -6,40 +6,42 @@ import random
 import re
 from itertools import combinations
 from pathlib import Path
-from jinja2 import Template
 from typing import Union
 
-from jinja2 import Template
+from jinja2 import Environment, StrictUndefined
+
+from rdagent.components.coder.factor_coder.config import FACTOR_IMPLEMENT_SETTINGS
+from rdagent.components.coder.factor_coder.CoSTEER.evaluators import (
+    FactorSingleFeedback,
+)
+from rdagent.components.coder.factor_coder.factor import FactorTask
+from rdagent.components.knowledge_management.graph import (
+    UndirectedGraph,
+    UndirectedNode,
+)
 from rdagent.core.evolving_framework import (
     EvolvableSubjects,
+    EvolvingKnowledgeBase,
     EvoStep,
     Knowledge,
-    KnowledgeBase,
     QueriedKnowledge,
     RAGStrategy,
 )
-from rdagent.core.log import RDAgentLog
+from rdagent.core.experiment import Workspace
 from rdagent.core.prompts import Prompts
-from rdagent.factor_implementation.evolving.evaluators import FactorImplementationSingleFeedback
-from rdagent.core.task import (
-    TaskImplementation,
-)
-from rdagent.factor_implementation.evolving.evolving_strategy import FactorImplementTask
-from rdagent.core.prompts import Prompts
-from rdagent.knowledge_management.graph import UndirectedGraph, UndirectedNode
-from rdagent.oai.llm_utils import APIBackend, calculate_embedding_distance_between_str_list
-
-from rdagent.factor_implementation.share_modules.factor_implementation_config import (
-    FACTOR_IMPLEMENT_SETTINGS,
+from rdagent.log import rdagent_logger as logger
+from rdagent.oai.llm_utils import (
+    APIBackend,
+    calculate_embedding_distance_between_str_list,
 )
 
 
-class FactorImplementationKnowledge(Knowledge):
+class FactorKnowledge(Knowledge):
     def __init__(
         self,
-        target_task: FactorImplementTask,
-        implementation: TaskImplementation,
-        feedback: FactorImplementationSingleFeedback,
+        target_task: FactorTask,
+        implementation: Workspace,
+        feedback: FactorSingleFeedback,
     ) -> None:
         """
         Initialize a FactorKnowledge object. The FactorKnowledge object is used to store a factor implementation without the ground truth code and value.
@@ -51,7 +53,7 @@ class FactorImplementationKnowledge(Knowledge):
             None
         """
         self.target_task = target_task
-        self.implementation = implementation
+        self.implementation = implementation.copy()
         self.feedback = feedback
 
     def get_implementation_and_feedback_str(self) -> str:
@@ -62,18 +64,19 @@ class FactorImplementationKnowledge(Knowledge):
 """
 
 
-class FactorImplementationQueriedKnowledge(QueriedKnowledge):
+class FactorQueriedKnowledge(QueriedKnowledge):
     def __init__(self, success_task_to_knowledge_dict: dict = {}, failed_task_info_set: set = set()) -> None:
         self.success_task_to_knowledge_dict = success_task_to_knowledge_dict
         self.failed_task_info_set = failed_task_info_set
 
 
-class FactorImplementationKnowledgeBaseV1(KnowledgeBase):
-    def __init__(self) -> None:
-        self.implementation_trace: dict[str, FactorImplementationKnowledge] = dict()
+class FactorKnowledgeBaseV1(EvolvingKnowledgeBase):
+    def __init__(self, path: str | Path = None) -> None:
+        self.implementation_trace: dict[str, FactorKnowledge] = dict()
         self.success_task_info_set: set[str] = set()
 
         self.task_to_embedding = dict()
+        super().__init__(path)
 
     def query(self) -> QueriedKnowledge | None:
         """
@@ -82,15 +85,15 @@ class FactorImplementationKnowledgeBaseV1(KnowledgeBase):
         raise NotImplementedError
 
 
-class FactorImplementationQueriedKnowledgeV1(FactorImplementationQueriedKnowledge):
+class FactorQueriedKnowledgeV1(FactorQueriedKnowledge):
     def __init__(self) -> None:
         self.working_task_to_former_failed_knowledge_dict = dict()
         self.working_task_to_similar_successful_knowledge_dict = dict()
         super().__init__()
 
 
-class FactorImplementationRAGStrategyV1(RAGStrategy):
-    def __init__(self, knowledgebase: FactorImplementationKnowledgeBaseV1) -> None:
+class FactorRAGStrategyV1(RAGStrategy):
+    def __init__(self, knowledgebase: FactorKnowledgeBaseV1) -> None:
         super().__init__(knowledgebase)
         self.current_generated_trace_count = 0
 
@@ -110,14 +113,14 @@ class FactorImplementationRAGStrategyV1(RAGStrategy):
                 evo_step = evolving_trace[trace_index]
                 implementations = evo_step.evolvable_subjects
                 feedback = evo_step.feedback
-                for task_index in range(len(implementations.target_factor_tasks)):
-                    target_task = implementations.target_factor_tasks[task_index]
-                    target_task_information = target_task.get_factor_information()
-                    implementation = implementations.corresponding_implementations[task_index]
+                for task_index in range(len(implementations.sub_tasks)):
+                    target_task = implementations.sub_tasks[task_index]
+                    target_task_information = target_task.get_task_information()
+                    implementation = implementations.sub_workspace_list[task_index]
                     single_feedback = feedback[task_index]
                     if single_feedback is None:
                         continue
-                    single_knowledge = FactorImplementationKnowledge(
+                    single_knowledge = FactorKnowledge(
                         target_task=target_task,
                         implementation=implementation,
                         feedback=single_feedback,
@@ -143,13 +146,13 @@ class FactorImplementationRAGStrategyV1(RAGStrategy):
         v1_query_similar_success_limit = FACTOR_IMPLEMENT_SETTINGS.v1_query_similar_success_limit
         fail_task_trial_limit = FACTOR_IMPLEMENT_SETTINGS.fail_task_trial_limit
 
-        queried_knowledge = FactorImplementationQueriedKnowledgeV1()
-        for target_factor_task in evo.target_factor_tasks:
-            target_factor_task_information = target_factor_task.get_factor_information()
+        queried_knowledge = FactorQueriedKnowledgeV1()
+        for target_factor_task in evo.sub_tasks:
+            target_factor_task_information = target_factor_task.get_task_information()
             if target_factor_task_information in self.knowledgebase.success_task_info_set:
-                queried_knowledge.success_task_to_knowledge_dict[target_factor_task_information] = (
-                    self.knowledgebase.implementation_trace[target_factor_task_information][-1]
-                )
+                queried_knowledge.success_task_to_knowledge_dict[
+                    target_factor_task_information
+                ] = self.knowledgebase.implementation_trace[target_factor_task_information][-1]
             elif (
                 len(
                     self.knowledgebase.implementation_trace.setdefault(
@@ -161,12 +164,14 @@ class FactorImplementationRAGStrategyV1(RAGStrategy):
             ):
                 queried_knowledge.failed_task_info_set.add(target_factor_task_information)
             else:
-                queried_knowledge.working_task_to_former_failed_knowledge_dict[target_factor_task_information] = (
-                    self.knowledgebase.implementation_trace.setdefault(
-                        target_factor_task_information,
-                        [],
-                    )[-v1_query_former_trace_limit:]
-                )
+                queried_knowledge.working_task_to_former_failed_knowledge_dict[
+                    target_factor_task_information
+                ] = self.knowledgebase.implementation_trace.setdefault(
+                    target_factor_task_information,
+                    [],
+                )[
+                    -v1_query_former_trace_limit:
+                ]
 
                 knowledge_base_success_task_list = list(
                     self.knowledgebase.success_task_info_set,
@@ -187,13 +192,13 @@ class FactorImplementationRAGStrategyV1(RAGStrategy):
                     )[-1]
                     for index in similar_indexes
                 ]
-                queried_knowledge.working_task_to_similar_successful_knowledge_dict[target_factor_task_information] = (
-                    similar_successful_knowledge
-                )
+                queried_knowledge.working_task_to_similar_successful_knowledge_dict[
+                    target_factor_task_information
+                ] = similar_successful_knowledge
         return queried_knowledge
 
 
-class FactorImplementationQueriedGraphKnowledge(FactorImplementationQueriedKnowledge):
+class FactorQueriedGraphKnowledge(FactorQueriedKnowledge):
     # Aggregation of knowledge
     def __init__(
         self,
@@ -208,11 +213,12 @@ class FactorImplementationQueriedGraphKnowledge(FactorImplementationQueriedKnowl
         super().__init__(**kwargs)
 
 
-class FactorImplementationGraphRAGStrategy(RAGStrategy):
-    def __init__(self, knowledgebase: FactorImplementationGraphKnowledgeBase) -> None:
+class FactorGraphRAGStrategy(RAGStrategy):
+    prompt = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")
+
+    def __init__(self, knowledgebase: FactorGraphKnowledgeBase) -> None:
         super().__init__(knowledgebase)
         self.current_generated_trace_count = 0
-        self.prompt = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")
 
     def generate_knowledge(
         self,
@@ -228,15 +234,15 @@ class FactorImplementationGraphRAGStrategy(RAGStrategy):
                 evo_step = evolving_trace[trace_index]
                 implementations = evo_step.evolvable_subjects
                 feedback = evo_step.feedback
-                for task_index in range(len(implementations.target_factor_tasks)):
+                for task_index in range(len(implementations.sub_tasks)):
                     single_feedback = feedback[task_index]
-                    target_task = implementations.target_factor_tasks[task_index]
-                    target_task_information = target_task.get_factor_information()
-                    implementation = implementations.corresponding_implementations[task_index]
+                    target_task = implementations.sub_tasks[task_index]
+                    target_task_information = target_task.get_task_information()
+                    implementation = implementations.sub_workspace_list[task_index]
                     single_feedback = feedback[task_index]
                     if single_feedback is None:
                         continue
-                    single_knowledge = FactorImplementationKnowledge(
+                    single_knowledge = FactorKnowledge(
                         target_task=target_task,
                         implementation=implementation,
                         feedback=single_feedback,
@@ -282,7 +288,7 @@ class FactorImplementationGraphRAGStrategy(RAGStrategy):
 
     def query(self, evo: EvolvableSubjects, evolving_trace: list[EvoStep]) -> QueriedKnowledge | None:
         conf_knowledge_sampler = FACTOR_IMPLEMENT_SETTINGS.v2_knowledge_sampler
-        factor_implementation_queried_graph_knowledge = FactorImplementationQueriedGraphKnowledge(
+        factor_implementation_queried_graph_knowledge = FactorQueriedGraphKnowledge(
             success_task_to_knowledge_dict=self.knowledgebase.success_task_to_knowledge_dict,
         )
 
@@ -290,6 +296,7 @@ class FactorImplementationGraphRAGStrategy(RAGStrategy):
             evo,
             factor_implementation_queried_graph_knowledge,
             FACTOR_IMPLEMENT_SETTINGS.v2_query_former_trace_limit,
+            FACTOR_IMPLEMENT_SETTINGS.v2_add_fail_attempt_to_latest_successful_execution,
         )
         factor_implementation_queried_graph_knowledge = self.component_query(
             evo,
@@ -310,11 +317,17 @@ class FactorImplementationGraphRAGStrategy(RAGStrategy):
         target_factor_task_information,
     ) -> list[UndirectedNode]:  # Hardcode: certain component nodes
         all_component_nodes = self.knowledgebase.graph.get_all_nodes_by_label_list(["component"])
+        if not len(all_component_nodes):
+            return []
         all_component_content = ""
         for _, component_node in enumerate(all_component_nodes):
             all_component_content += f"{component_node.content}, \n"
-        analyze_component_system_prompt = Template(self.prompt["analyze_component_prompt_v1_system"]).render(
-            all_component_content=all_component_content,
+        analyze_component_system_prompt = (
+            Environment(undefined=StrictUndefined)
+            .from_string(self.prompt["analyze_component_prompt_v1_system"])
+            .render(
+                all_component_content=all_component_content,
+            )
         )
 
         analyze_component_user_prompt = target_factor_task_information
@@ -328,7 +341,7 @@ class FactorImplementationGraphRAGStrategy(RAGStrategy):
             )["component_no_list"]
             return [all_component_nodes[index - 1] for index in sorted(list(set(component_no_list)))]
         except:
-            RDAgentLog().warning("Error when analyzing components.")
+            logger.warning("Error when analyzing components.")
             analyze_component_user_prompt = "Your response is not a valid component index list."
 
         return []
@@ -378,16 +391,17 @@ class FactorImplementationGraphRAGStrategy(RAGStrategy):
     def former_trace_query(
         self,
         evo: EvolvableSubjects,
-        factor_implementation_queried_graph_knowledge: FactorImplementationQueriedGraphKnowledge,
+        factor_implementation_queried_graph_knowledge: FactorQueriedGraphKnowledge,
         v2_query_former_trace_limit: int = 5,
+        v2_add_fail_attempt_to_latest_successful_execution: bool = False,
     ) -> Union[QueriedKnowledge, set]:
         """
         Query the former trace knowledge of the working trace, and find all the failed task information which tried more than fail_task_trial_limit times
         """
         fail_task_trial_limit = FACTOR_IMPLEMENT_SETTINGS.fail_task_trial_limit
 
-        for target_factor_task in evo.target_factor_tasks:
-            target_factor_task_information = target_factor_task.get_factor_information()
+        for target_factor_task in evo.sub_tasks:
+            target_factor_task_information = target_factor_task.get_task_information()
             if (
                 target_factor_task_information not in self.knowledgebase.success_task_to_knowledge_dict
                 and target_factor_task_information in self.knowledgebase.working_trace_knowledge
@@ -417,24 +431,38 @@ class FactorImplementationGraphRAGStrategy(RAGStrategy):
                     else:
                         current_index += 1
 
+                latest_attempt = None
+                if v2_add_fail_attempt_to_latest_successful_execution:
+                    # When the last successful execution is not the last one in the working trace, it means we have tried to correct it. We should tell the agent this fail trial to avoid endless loop in the future.
+                    if (
+                        len(former_trace_knowledge) > 0
+                        and len(self.knowledgebase.working_trace_knowledge[target_factor_task_information]) > 1
+                        and self.knowledgebase.working_trace_knowledge[target_factor_task_information].index(
+                            former_trace_knowledge[-1]
+                        )
+                        < len(self.knowledgebase.working_trace_knowledge[target_factor_task_information]) - 1
+                    ):
+                        latest_attempt = self.knowledgebase.working_trace_knowledge[target_factor_task_information][-1]
+
                 factor_implementation_queried_graph_knowledge.former_traces[target_factor_task_information] = (
-                    former_trace_knowledge[-v2_query_former_trace_limit:]
+                    former_trace_knowledge[-v2_query_former_trace_limit:],
+                    latest_attempt,
                 )
             else:
-                factor_implementation_queried_graph_knowledge.former_traces[target_factor_task_information] = []
+                factor_implementation_queried_graph_knowledge.former_traces[target_factor_task_information] = ([], None)
 
         return factor_implementation_queried_graph_knowledge
 
     def component_query(
         self,
         evo: EvolvableSubjects,
-        factor_implementation_queried_graph_knowledge: FactorImplementationQueriedGraphKnowledge,
+        factor_implementation_queried_graph_knowledge: FactorQueriedGraphKnowledge,
         v2_query_component_limit: int = 5,
         knowledge_sampler: float = 1.0,
     ) -> QueriedKnowledge | None:
-        # queried_component_knowledge = FactorImplementationQueriedGraphComponentKnowledge()
-        for target_factor_task in evo.target_factor_tasks:
-            target_factor_task_information = target_factor_task.get_factor_information()
+        # queried_component_knowledge = FactorQueriedGraphComponentKnowledge()
+        for target_factor_task in evo.sub_tasks:
+            target_factor_task_information = target_factor_task.get_task_information()
             if (
                 target_factor_task_information in self.knowledgebase.success_task_to_knowledge_dict
                 or target_factor_task_information in factor_implementation_queried_graph_knowledge.failed_task_info_set
@@ -553,7 +581,7 @@ class FactorImplementationGraphRAGStrategy(RAGStrategy):
                     if knowledge.feedback is not None and knowledge.feedback.final_decision_based_on_gt == False
                 ]
                 queried_from_gt_knowledge_count = max(
-                    min(v2_query_component_limit // 2, len(queried_from_gt_knowledge_list)),
+                    min((v2_query_component_limit // 2 + 1), len(queried_from_gt_knowledge_list)),
                     v2_query_component_limit - len(queried_without_gt_knowledge_list),
                 )
                 factor_implementation_queried_graph_knowledge.component_with_success_task[
@@ -568,13 +596,13 @@ class FactorImplementationGraphRAGStrategy(RAGStrategy):
     def error_query(
         self,
         evo: EvolvableSubjects,
-        factor_implementation_queried_graph_knowledge: FactorImplementationQueriedGraphKnowledge,
+        factor_implementation_queried_graph_knowledge: FactorQueriedGraphKnowledge,
         v2_query_error_limit: int = 5,
         knowledge_sampler: float = 1.0,
     ) -> QueriedKnowledge | None:
-        # queried_error_knowledge = FactorImplementationQueriedGraphErrorKnowledge()
-        for task_index, target_factor_task in enumerate(evo.target_factor_tasks):
-            target_factor_task_information = target_factor_task.get_factor_information()
+        # queried_error_knowledge = FactorQueriedGraphErrorKnowledge()
+        for task_index, target_factor_task in enumerate(evo.sub_tasks):
+            target_factor_task_information = target_factor_task.get_task_information()
             factor_implementation_queried_graph_knowledge.error_with_success_task[target_factor_task_information] = {}
             if (
                 target_factor_task_information in self.knowledgebase.success_task_to_knowledge_dict
@@ -595,7 +623,7 @@ class FactorImplementationGraphRAGStrategy(RAGStrategy):
                 ):
                     queried_last_trace = factor_implementation_queried_graph_knowledge.former_traces[
                         target_factor_task_information
-                    ][-1]
+                    ][0][-1]
                     target_index = self.knowledgebase.working_trace_knowledge[target_factor_task_information].index(
                         queried_last_trace,
                     )
@@ -700,13 +728,13 @@ class FactorImplementationGraphRAGStrategy(RAGStrategy):
         return factor_implementation_queried_graph_knowledge
 
 
-class FactorImplementationGraphKnowledgeBase(KnowledgeBase):
-    def __init__(self, init_component_list=None) -> None:
+class FactorGraphKnowledgeBase(EvolvingKnowledgeBase):
+    def __init__(self, init_component_list=None, path: str | Path = None) -> None:
         """
         Load knowledge, offer brief information of knowledge and common handle interfaces
         """
-        self.graph: UndirectedGraph = UndirectedGraph.load(Path.cwd() / "graph.pkl")
-        RDAgentLog().info(f"Knowledge Graph loaded, size={self.graph.size()}")
+        self.graph: UndirectedGraph = UndirectedGraph(Path.cwd() / "graph.pkl")
+        logger.info(f"Knowledge Graph loaded, size={self.graph.size()}")
 
         if init_component_list:
             for component in init_component_list:
@@ -723,7 +751,7 @@ class FactorImplementationGraphKnowledgeBase(KnowledgeBase):
         # Add already success task
         self.success_task_to_knowledge_dict = {}
 
-        # key:node_id(for task trace and success implement), value:knowledge instance(aka 'FactorImplementationKnowledge')
+        # key:node_id(for task trace and success implement), value:knowledge instance(aka 'FactorKnowledge')
         self.node_to_implementation_knowledge_dict = {}
 
         # store the task description to component nodes

rdagent/components/coder/factor_coder/CoSTEER/scheduler.py

@@ -0,0 +1,88 @@
+import json
+from pathlib import Path
+from typing import Dict
+
+from jinja2 import Environment, StrictUndefined
+
+from rdagent.components.coder.factor_coder.CoSTEER.evolvable_subjects import (
+    FactorEvolvingItem,
+)
+from rdagent.core.prompts import Prompts
+from rdagent.core.scenario import Scenario
+from rdagent.log import rdagent_logger as logger
+from rdagent.oai.llm_conf import LLM_SETTINGS
+from rdagent.oai.llm_utils import APIBackend
+
+scheduler_prompts = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")
+
+
+def RandomSelect(to_be_finished_task_index, implementation_factors_per_round):
+    import random
+
+    to_be_finished_task_index = random.sample(
+        to_be_finished_task_index,
+        implementation_factors_per_round,
+    )
+
+    logger.info(f"The random selection is: {to_be_finished_task_index}")
+    return to_be_finished_task_index
+
+
+def LLMSelect(
+    to_be_finished_task_index,
+    implementation_factors_per_round,
+    evo: FactorEvolvingItem,
+    former_trace: Dict,
+    scen: Scenario,
+):
+    tasks = []
+    for i in to_be_finished_task_index:
+        # find corresponding former trace for each task
+        target_factor_task_information = evo.sub_tasks[i].get_task_information()
+        if target_factor_task_information in former_trace:
+            tasks.append((i, evo.sub_tasks[i], former_trace[target_factor_task_information][0]))
+
+    system_prompt = (
+        Environment(undefined=StrictUndefined)
+        .from_string(
+            scheduler_prompts["select_implementable_factor_system"],
+        )
+        .render(
+            scenario=scen.get_scenario_all_desc(),
+        )
+    )
+
+    for _ in range(10):  # max attempt to reduce the length of user_prompt
+        user_prompt = (
+            Environment(undefined=StrictUndefined)
+            .from_string(
+                scheduler_prompts["select_implementable_factor_user"],
+            )
+            .render(
+                factor_num=implementation_factors_per_round,
+                sub_tasks=tasks,
+            )
+        )
+        if (
+            APIBackend().build_messages_and_calculate_token(
+                user_prompt=user_prompt,
+                system_prompt=system_prompt,
+            )
+            < LLM_SETTINGS.chat_token_limit
+        ):
+            break
+
+    response = APIBackend().build_messages_and_create_chat_completion(
+        user_prompt=user_prompt,
+        system_prompt=system_prompt,
+        json_mode=True,
+    )
+    try:
+        selection = json.loads(response)["selected_factor"]
+        if not isinstance(selection, list):
+            return to_be_finished_task_index
+        selection_index = [x for x in selection if isinstance(x, int)]
+    except:
+        return to_be_finished_task_index
+
+    return selection_index

rdagent/components/coder/factor_coder/config.py

@@ -0,0 +1,60 @@
+from pathlib import Path
+from typing import Literal, Union
+
+from pydantic_settings import BaseSettings
+
+SELECT_METHOD = Literal["random", "scheduler"]
+
+
+class FactorImplementSettings(BaseSettings):
+    class Config:
+        env_prefix = "FACTOR_CODER_"
+        """Use `FACTOR_CODER_` as prefix for environment variables"""
+
+    coder_use_cache: bool = False
+    """Indicates whether to use cache for the coder"""
+
+    data_folder: str = "git_ignore_folder/factor_implementation_source_data"
+    """Path to the folder containing financial data (default is fundamental data in Qlib)"""
+
+    data_folder_debug: str = "git_ignore_folder/factor_implementation_source_data_debug"
+    """Path to the folder containing partial financial data (for debugging)"""
+
+    # TODO: the factor implement specific settings should not appear in this settings
+    # Evolving should have a method specific settings
+    # evolving related config
+    fail_task_trial_limit: int = 20
+
+    v1_query_former_trace_limit: int = 5
+    v1_query_similar_success_limit: int = 5
+
+    v2_query_component_limit: int = 1
+    v2_query_error_limit: int = 1
+    v2_query_former_trace_limit: int = 1
+    v2_add_fail_attempt_to_latest_successful_execution: bool = False
+    v2_error_summary: bool = False
+    v2_knowledge_sampler: float = 1.0
+
+    file_based_execution_timeout: int = 120
+    """Timeout in seconds for each factor implementation execution"""
+
+    select_method: str = "random"
+    """Method for the selection of factors implementation"""
+
+    select_threshold: int = 10
+    """Threshold for the number of factor selections"""
+
+    max_loop: int = 10
+    """Maximum number of task implementation loops"""
+
+    knowledge_base_path: Union[str, None] = None
+    """Path to the knowledge base"""
+
+    new_knowledge_base_path: Union[str, None] = None
+    """Path to the new knowledge base"""
+
+    python_bin: str = "python"
+    """Path to the Python binary"""
+
+
+FACTOR_IMPLEMENT_SETTINGS = FactorImplementSettings()

rdagent/components/coder/factor_coder/factor.py

@@ -0,0 +1,222 @@
+from __future__ import annotations
+
+import pickle
+import subprocess
+import uuid
+from pathlib import Path
+from typing import Tuple, Union
+
+import pandas as pd
+from filelock import FileLock
+
+from rdagent.app.kaggle.conf import KAGGLE_IMPLEMENT_SETTING
+from rdagent.components.coder.factor_coder.config import FACTOR_IMPLEMENT_SETTINGS
+from rdagent.core.exception import CodeFormatError, CustomRuntimeError, NoOutputError
+from rdagent.core.experiment import Experiment, FBWorkspace, Task
+from rdagent.core.utils import cache_with_pickle
+from rdagent.log import rdagent_logger as logger
+from rdagent.oai.llm_utils import md5_hash
+
+
+class FactorTask(Task):
+    # TODO:  generalized the attributes into the Task
+    # - factor_* -> *
+    def __init__(
+        self,
+        factor_name,
+        factor_description,
+        factor_formulation,
+        *args,
+        variables: dict = {},
+        resource: str = None,
+        factor_implementation: bool = False,
+        **kwargs,
+    ) -> None:
+        self.factor_name = (
+            factor_name  # TODO: remove it in the later version. Keep it only for pickle version compatibility
+        )
+        self.factor_description = factor_description
+        self.factor_formulation = factor_formulation
+        self.variables = variables
+        self.factor_resources = resource
+        self.factor_implementation = factor_implementation
+        super().__init__(name=factor_name, *args, **kwargs)
+
+    def get_task_information(self):
+        return f"""factor_name: {self.factor_name}
+factor_description: {self.factor_description}
+factor_formulation: {self.factor_formulation}
+variables: {str(self.variables)}"""
+
+    def get_task_information_and_implementation_result(self):
+        return {
+            "factor_name": self.factor_name,
+            "factor_description": self.factor_description,
+            "factor_formulation": self.factor_formulation,
+            "variables": str(self.variables),
+            "factor_implementation": str(self.factor_implementation),
+        }
+
+    @staticmethod
+    def from_dict(dict):
+        return FactorTask(**dict)
+
+    def __repr__(self) -> str:
+        return f"<{self.__class__.__name__}[{self.factor_name}]>"
+
+
+class FactorFBWorkspace(FBWorkspace):
+    """
+    This class is used to implement a factor by writing the code to a file.
+    Input data and output factor value are also written to files.
+    """
+
+    # TODO: (Xiao) think raising errors may get better information for processing
+    FB_EXEC_SUCCESS = "Execution succeeded without error."
+    FB_CODE_NOT_SET = "code is not set."
+    FB_EXECUTION_SUCCEEDED = "Execution succeeded without error."
+    FB_OUTPUT_FILE_NOT_FOUND = "\nExpected output file not found."
+    FB_OUTPUT_FILE_FOUND = "\nExpected output file found."
+
+    def __init__(
+        self,
+        *args,
+        raise_exception: bool = False,
+        **kwargs,
+    ) -> None:
+        super().__init__(*args, **kwargs)
+        self.raise_exception = raise_exception
+
+    def hash_func(self, data_type: str = "Debug") -> str:
+        return (
+            md5_hash(data_type + self.code_dict["factor.py"])
+            if ("factor.py" in self.code_dict and not self.raise_exception)
+            else None
+        )
+
+    @cache_with_pickle(hash_func)
+    def execute(self, data_type: str = "Debug") -> Tuple[str, pd.DataFrame]:
+        """
+        execute the implementation and get the factor value by the following steps:
+        1. make the directory in workspace path
+        2. write the code to the file in the workspace path
+        3. link all the source data to the workspace path folder
+        if call_factor_py is True:
+            4. execute the code
+        else:
+            4. generate a script from template to import the factor.py dump get the factor value to result.h5
+        5. read the factor value from the output file in the workspace path folder
+        returns the execution feedback as a string and the factor value as a pandas dataframe
+
+
+        Regarding the cache mechanism:
+        1. We will store the function's return value to ensure it behaves as expected.
+        - The cached information will include a tuple with the following: (execution_feedback, executed_factor_value_dataframe, Optional[Exception])
+
+        """
+        super().execute()
+        if self.code_dict is None or "factor.py" not in self.code_dict:
+            if self.raise_exception:
+                raise CodeFormatError(self.FB_CODE_NOT_SET)
+            else:
+                return self.FB_CODE_NOT_SET, None
+        with FileLock(self.workspace_path / "execution.lock"):
+            if self.target_task.version == 1:
+                source_data_path = (
+                    Path(
+                        FACTOR_IMPLEMENT_SETTINGS.data_folder_debug,
+                    )
+                    if data_type == "Debug"  # FIXME: (yx) don't think we should use a debug tag for this.
+                    else Path(
+                        FACTOR_IMPLEMENT_SETTINGS.data_folder,
+                    )
+                )
+            elif self.target_task.version == 2:
+                # TODO you can change the name of the data folder for a better understanding
+                source_data_path = Path(KAGGLE_IMPLEMENT_SETTING.local_data_path) / KAGGLE_IMPLEMENT_SETTING.competition
+
+            source_data_path.mkdir(exist_ok=True, parents=True)
+            code_path = self.workspace_path / f"factor.py"
+
+            self.link_all_files_in_folder_to_workspace(source_data_path, self.workspace_path)
+
+            execution_feedback = self.FB_EXECUTION_SUCCEEDED
+            execution_success = False
+            execution_error = None
+
+            if self.target_task.version == 1:
+                execution_code_path = code_path
+            elif self.target_task.version == 2:
+                execution_code_path = self.workspace_path / f"{uuid.uuid4()}.py"
+                execution_code_path.write_text((Path(__file__).parent / "factor_execution_template.txt").read_text())
+
+            try:
+                subprocess.check_output(
+                    f"{FACTOR_IMPLEMENT_SETTINGS.python_bin} {execution_code_path}",
+                    shell=True,
+                    cwd=self.workspace_path,
+                    stderr=subprocess.STDOUT,
+                    timeout=FACTOR_IMPLEMENT_SETTINGS.file_based_execution_timeout,
+                )
+                execution_success = True
+            except subprocess.CalledProcessError as e:
+                import site
+
+                execution_feedback = (
+                    e.output.decode()
+                    .replace(str(execution_code_path.parent.absolute()), r"/path/to")
+                    .replace(str(site.getsitepackages()[0]), r"/path/to/site-packages")
+                )
+                if len(execution_feedback) > 2000:
+                    execution_feedback = (
+                        execution_feedback[:1000] + "....hidden long error message...." + execution_feedback[-1000:]
+                    )
+                if self.raise_exception:
+                    raise CustomRuntimeError(execution_feedback)
+                else:
+                    execution_error = CustomRuntimeError(execution_feedback)
+            except subprocess.TimeoutExpired:
+                execution_feedback += f"Execution timeout error and the timeout is set to {FACTOR_IMPLEMENT_SETTINGS.file_based_execution_timeout} seconds."
+                if self.raise_exception:
+                    raise CustomRuntimeError(execution_feedback)
+                else:
+                    execution_error = CustomRuntimeError(execution_feedback)
+
+            workspace_output_file_path = self.workspace_path / "result.h5"
+            if workspace_output_file_path.exists() and execution_success:
+                try:
+                    executed_factor_value_dataframe = pd.read_hdf(workspace_output_file_path)
+                    execution_feedback += self.FB_OUTPUT_FILE_FOUND
+                except Exception as e:
+                    execution_feedback += f"Error found when reading hdf file: {e}"[:1000]
+                    executed_factor_value_dataframe = None
+            else:
+                execution_feedback += self.FB_OUTPUT_FILE_NOT_FOUND
+                executed_factor_value_dataframe = None
+                if self.raise_exception:
+                    raise NoOutputError(execution_feedback)
+                else:
+                    execution_error = NoOutputError(execution_feedback)
+
+        return execution_feedback, executed_factor_value_dataframe
+
+    def __str__(self) -> str:
+        # NOTE:
+        # If the code cache works, the workspace will be None.
+        return f"File Factor[{self.target_task.factor_name}]: {self.workspace_path}"
+
+    def __repr__(self) -> str:
+        return self.__str__()
+
+    @staticmethod
+    def from_folder(task: FactorTask, path: Union[str, Path], **kwargs):
+        path = Path(path)
+        code_dict = {}
+        for file_path in path.iterdir():
+            if file_path.suffix == ".py":
+                code_dict[file_path.name] = file_path.read_text()
+        return FactorFBWorkspace(target_task=task, code_dict=code_dict, **kwargs)
+
+
+FactorExperiment = Experiment
+FeatureExperiment = Experiment

rdagent/components/coder/factor_coder/factor_execution_template.txt

@@ -0,0 +1,15 @@
+import os
+
+import numpy as np
+import pandas as pd
+from factor import feature_engineering_cls
+
+if os.path.exists("X_valid.pkl"):
+    valid_df = pd.read_pickle("X_valid.pkl").head(1000)
+else:
+    raise FileNotFoundError("No valid data found.")
+
+cls = feature_engineering_cls()
+cls.fit(valid_df)
+new_feat = cls.transform(valid_df)
+new_feat.to_hdf("result.h5", key="data", mode="w")

rdagent/components/coder/factor_coder/prompts.yaml

@@ -1,8 +1,10 @@
+
 evaluator_code_feedback_v1_system: |-
-  Your job is to give critic to user's code. User's code is expected to implement some factors in quant investment. The code contains reading data from a HDF5(H5) file, calculate the factor to each instrument on each datetime, and save the result pandas dataframe to a HDF5(H5) file.
-
-  User will firstly provide you the information of the factor, which includes the name of the factor, description of the factor, the formulation of the factor and the description of the formulation. You can check whether user's code is align with the factor.
+  User is trying to implement some factors in the following scenario:
+  {{ scenario }}
+  User will provide you the information of the factor.
 
+  Your job is to check whether user's code is align with the factor and the scenario.
   The user will provide the source python code and the execution error message if execution failed.
   The user might provide you the ground truth code for you to provide the critic. You should not leak the ground truth code to the user in any form but you can use it to provide the critic.
 
@@ -11,9 +13,14 @@ evaluator_code_feedback_v1_system: |-
   If the ground truth code is provided, your critic should only consider checking whether the user's code is align with the ground truth code since the ground truth is definitely correct.
   If the ground truth code is not provided, your critic should consider checking whether the user's code is reasonable and correct.
 
+  Notice that your critics are not for user to debug the code. They are sent to the coding agent to correct the code. So don't give any following items for the user to check like "Please check the code line XXX".
+
+  You suggestion should not include any code, just some clear and short suggestions. Please point out very critical issues in your response, ignore non-important issues to avoid confusion. If no big issue found in the code, you can response "No critics found".
+  
   You should provide the suggestion to each of your critic to help the user improve the code. Please response the critic in the following format. Here is an example structure for the output:
   critic 1: The critic message to critic 1
   critic 2: The critic message to critic 2
+
 evaluator_code_feedback_v1_user: |-
   --------------Factor information:---------------
   {{ factor_information }}
@@ -29,44 +36,27 @@ evaluator_code_feedback_v1_user: |-
   --------------Ground truth Python code:---------------
   {{ gt_code }}
   {% endif %}
+
 evolving_strategy_factor_implementation_v1_system: |-
-  The user is trying to implement some factors in quant investment, and you are the one to help write the python code.
-  
-  {{ data_info }}
-
-  The user will provide you a formulation of the factor, which contains some function calls and some operators. You need to implement the function calls and operators in python. Your code is expected to align the formulation in any form which means The user needs to get the exact factor values with your code as expected.
-
-  Your code should contain the following part: the import part, the function part, and the main part. You should write a main function name: "calculate_{function_name}" and call this function in "if __name__ == __main__" part. Don't write any try-except block in your code. The user will catch the exception message and provide the feedback to you.
-
-  User will write your code into a python file and execute the file directly with "python {your_file_name}.py". You should calculate the factor values and save the result into a HDF5(H5) file named "result.h5" in the same directory as your python file. The result file is a HDF5(H5) file containing a pandas dataframe. The index of the dataframe is the "datetime" and "instrument", and the single column name is the factor name,and the value is the factor value. The result file should be saved in the same directory as your python file.
+  User is trying to implement some factors in the following scenario:
+  {{ scenario }}
+  Your code is expected to align the scenario in any form which means The user needs to get the exact factor values with your code as expected.
 
   To help you write the correct code, the user might provide multiple information that helps you write the correct code:
   1. The user might provide you the correct code to similar factors. Your should learn from these code to write the correct code.
   2. The user might provide you the failed former code and the corresponding feedback to the code. The feedback contains to the execution, the code and the factor value. You should analyze the feedback and try to correct the latest code.
   3. The user might provide you the suggestion to the latest fail code and some similar fail to correct pairs. Each pair contains the fail code with similar error and the corresponding corrected version code. You should learn from these suggestion to write the correct code.
+  
+  Your must write your code based on your former latest attempt below which consists of your former code and code feedback, you should read the former attempt carefully and must not modify the right part of your former code.
 
-  Your must write your code based on your former lastest attempt below which consists of your former code and code feedback, you should read the former attempt carefully and must not modify the right part of your former code.
   {% if queried_former_failed_knowledge|length != 0 %}
   --------------Your former latest attempt:---------------
-  {% for former_failed_knowledge in queried_former_failed_knowledge %}
-  =====Code to implementation {{ loop.index }}=====
-  {{ former_failed_knowledge.implementation.code }}
-  =====Feedback to implementation {{ loop.index }}=====
-  {{ former_failed_knowledge.feedback }}
-  {% endfor %}
+  =====Code to the former implementation=====
+  {{ queried_former_failed_knowledge[-1].implementation.code }}
+  =====Feedback to the former implementation=====
+  {{ queried_former_failed_knowledge[-1].feedback }}
   {% endif %}
 
-  A typical format of `result.h5` may be like following:
-    datetime    instrument
-    2020-01-02  SZ000001     -0.001796
-                SZ000166      0.005780
-                SZ000686      0.004228
-                SZ000712      0.001298
-                SZ000728      0.005330
-                                ...
-    2021-12-31  SZ000750      0.000000
-                SZ000776      0.002459
-
   Please response the code in the following json format. Here is an example structure for the JSON output:
   {
       "code": "The Python code as a string."
@@ -80,7 +70,7 @@ evolving_strategy_factor_implementation_v1_user: |-
   --------------Correct code to similar factors:---------------
   {% for similar_successful_knowledge in queried_similar_successful_knowledge %}
   =====Factor {{loop.index}}:=====
-  {{ similar_successful_knowledge.target_task.get_factor_information() }}
+  {{ similar_successful_knowledge.target_task.get_task_information() }}
   =====Code:=====
   {{ similar_successful_knowledge.implementation.code }}
   {% endfor %}
@@ -96,31 +86,6 @@ evolving_strategy_factor_implementation_v1_user: |-
   {% endfor %}
   {% endif %}
 
-evaluator_final_decision_v1_system: |-
-  User is trying to implement some factor in quant investment and has finished a version of implementation. User has finished evaluation and got some feedback from the evaluator.
-  The evaluator run the code and get the factor value dataframe and provide several feedback regarding user's code and code output. You should analyze the feedback and considering the factor description to give a final decision about the evaluation result. The final decision concludes whether the factor is implemented correctly and if not, detail feedback containing reason and suggestion if the final decision is False.
-
-  The implementation final decision is considered in the following logic:
-  1. If the value and the ground truth value are exactly the same under a small tolerance, the implementation is considered correct.
-  2. If the value and the ground truth value have a high correlation on ic or rank ic, the implementation is considered correct.
-  3. If no ground truth value is not provided, the implementation is considered correct if the code execution is successful and the code feedback is reasonable.
-
-  Please response the critic in the json format. Here is an example structure for the JSON output, please strictly follow the format:
-  {
-      "final_decision": True,
-      "final_feedback": "The final feedback message",
-  }
-
-evaluator_final_decision_v1_user: |-
-  --------------Factor information:---------------
-  {{ factor_information }}
-  --------------Execution feedback:---------------
-  {{ execution_feedback }}
-  --------------Code feedback:---------------
-  {{ code_feedback }}
-  --------------Factor value feedback:---------------
-  {{ factor_value_feedback }}
-
 evolving_strategy_factor_implementation_v2_user: |-
   --------------Target factor information:---------------
   {{ factor_information_str }}
@@ -131,7 +96,7 @@ evolving_strategy_factor_implementation_v2_user: |-
   When doing other tasks, you met some similar errors but you finally solve them. Here are some examples:
   {% for error_content, similar_error_knowledge in queried_similar_error_knowledge %} 
   --------------Factor information to similar error ({{error_content}}):---------------
-  {{ similar_error_knowledge[0].target_task.get_factor_information() }}
+  {{ similar_error_knowledge[0].target_task.get_task_information() }}
   =====Code with similar error ({{error_content}}):=====
   {{ similar_error_knowledge[0].implementation.code }}
   =====Success code to former code with similar error ({{error_content}}):=====
@@ -148,15 +113,23 @@ evolving_strategy_factor_implementation_v2_user: |-
   --------------Correct code to similar factors:---------------
   {% for similar_component_knowledge in queried_similar_component_knowledge %}
   =====Factor {{loop.index}}:=====
-  {{ similar_component_knowledge.target_task.get_factor_information() }}
+  {{ similar_component_knowledge.target_task.get_task_information() }}
   =====Code:=====
   {{ similar_component_knowledge.implementation.code }}
   {% endfor %}
   {% endif %}
-
+  {% if latest_attempt_to_latest_successful_execution is not none %}
+  You have tried to correct your former failed code but still met some errors. Here is the latest attempt to the latest successful execution, try not to get the same error to your new code:
+  =====Your latest attempt=====
+  {{ latest_attempt_to_latest_successful_execution.implementation.code }}
+  =====Feedback to your latest attempt=====
+  {{ latest_attempt_to_latest_successful_execution.feedback }}
+  {% endif %}
 
 evolving_strategy_error_summary_v2_system: |-
-  You are doing the following task: 
+  User is trying to implement some factors in the following scenario:
+  {{ scenario }}
+  User is doing the following task: 
   {{factor_information_str}}
 
   You have written some code but it meets errors like the following:
@@ -165,6 +138,8 @@ evolving_strategy_error_summary_v2_system: |-
   The user has found some tasks that met similar errors, and their final correct solutions.
   Please refer to these similar errors and their solutions, provide some clear, short and accurate critics that might help you solve the issues in your code.
 
+  You suggestion should not include any code, just some clear and short suggestions. Please point out very critical issues in your response, ignore non-important issues to avoid confusion. If no big issue found in the code, you can response "No critics found".
+
   Please response the critic in the following format. Here is an example structure for the output:
   critic 1: The critic message to critic 1
   critic 2: The critic message to critic 2
@@ -173,7 +148,7 @@ evolving_strategy_error_summary_v2_user: |-
   {% if queried_similar_error_knowledge|length != 0 %}
   {% for error_content, similar_error_knowledge in queried_similar_error_knowledge %} 
   --------------Factor information to similar error ({{error_content}}):---------------
-  {{ similar_error_knowledge[0].target_task.get_factor_information() }}
+  {{ similar_error_knowledge[0].target_task.get_task_information() }}
   =====Code with similar error ({{error_content}}):=====
   {{ similar_error_knowledge[0].implementation.code }}
   =====Success code to former code with similar error ({{error_content}}):=====
@@ -183,14 +158,10 @@ evolving_strategy_error_summary_v2_user: |-
 
 
 select_implementable_factor_system: |-
-  User is trying to implement some factors in quant investment using Python code, You are an assistant who helps the user select the easiest-to-implement factors and some factors may be difficult to implement due to a lack of information or excessive complexity.. 
-  The user will provide the number of factor you should pick and information about the factors, including their descriptions, formulas, and variable explanations.
-
-  At the same time, user will provide with your former attempt to implement the factor and the feedback to the implementation.You need to carefully review your previous attempts. Some factors have been repeatedly tried without success. You should consider discarding these factors.
-
-  Here is the source data that user will use to implement the factors:
-  {{ data_info }}
-
+  User is trying to implement some factors in the following scenario:
+  {{ scenario }}
+  Your job is to help the user select the easiest-to-implement factors. Some factors may be difficult to implement due to a lack of information or excessive complexity. The user will provide the number of factors you should pick and information about the factors, including their descriptions, formulas, and variable explanations.
+  User will provide you the former attempt to implement the factor and the feedback to the implementation. You need to carefully review your previous attempts. Some factors have been repeatedly tried without success. You should consider discarding these factors.
   Please analyze the difficulties of the each factors and provide the reason and response the indices of selected implementable factor in the json format. Here is an example structure for the JSON output:
   {
       "Analysis": "Analyze the difficulties of the each factors and provide the reason why the factor can be implemented or not."
@@ -199,7 +170,7 @@ select_implementable_factor_system: |-
 
 select_implementable_factor_user: |-
   Number of factor you should pick: {{ factor_num }}
-  {% for factor_info in target_factor_tasks %} 
+  {% for factor_info in sub_tasks %} 
   =============Factor index:{{factor_info[0]}}:=============
   =====Factor name:=====
   {{ factor_info[1].factor_name }}
@@ -226,4 +197,42 @@ analyze_component_prompt_v1_system: |-
   Please response the critic in the json format. Here is an example structure for the JSON output, please strictly follow the format:
   {
       "component_no_list": the list containing indices of components.
-  }
+  }
+
+evaluator_output_format_system: |-
+  User is trying to implement some factors in the following scenario:
+  {{ scenario }}
+  User will provide you the format of the output. Please help to check whether the output is align with the format.
+  Please respond in the JSON format. Here is an example structure for the JSON output:
+  {
+      "output_format_decision": True,
+      "output_format_feedback": "The output format is correct."
+  }
+
+
+evaluator_final_decision_v1_system: |-
+  User is trying to implement some factors in the following scenario:
+  {{ scenario }}
+  User has finished evaluation and got some feedback from the evaluator.
+  The evaluator run the code and get the factor value dataframe and provide several feedback regarding user's code and code output. You should analyze the feedback and considering the scenario and factor description to give a final decision about the evaluation result. The final decision concludes whether the factor is implemented correctly and if not, detail feedback containing reason and suggestion if the final decision is False.
+
+  The implementation final decision is considered in the following logic:
+  1. If the value and the ground truth value are exactly the same under a small tolerance, the implementation is considered correct.
+  2. If the value and the ground truth value have a high correlation on ic or rank ic, the implementation is considered correct.
+  3. If no ground truth value is provided, the implementation is considered correct if the code executes successfully (assuming the data provided is correct). Any exceptions, including those actively raised, are considered faults of the code. Additionally, the code feedback must align with the scenario and factor description.
+
+  Please response the critic in the json format. Here is an example structure for the JSON output, please strictly follow the format:
+  {
+      "final_decision": True,
+      "final_feedback": "The final feedback message",
+  }
+
+evaluator_final_decision_v1_user: |-
+  --------------Factor information:---------------
+  {{ factor_information }}
+  --------------Execution feedback:---------------
+  {{ execution_feedback }}
+  --------------Code feedback:---------------
+  {{ code_feedback }}
+  --------------Factor value feedback:---------------
+  {{ factor_value_feedback }}

rdagent/components/coder/model_coder/CoSTEER/__init__.py

@@ -0,0 +1,94 @@
+import pickle
+from pathlib import Path
+
+from rdagent.components.coder.model_coder.conf import MODEL_IMPL_SETTINGS
+from rdagent.components.coder.model_coder.CoSTEER.evaluators import (
+    ModelCoderMultiEvaluator,
+)
+from rdagent.components.coder.model_coder.CoSTEER.evolvable_subjects import (
+    ModelEvolvingItem,
+)
+from rdagent.components.coder.model_coder.CoSTEER.evolving_agent import ModelRAGEvoAgent
+from rdagent.components.coder.model_coder.CoSTEER.evolving_strategy import (
+    ModelCoderEvolvingStrategy,
+)
+from rdagent.components.coder.model_coder.CoSTEER.knowledge_management import (
+    ModelKnowledgeBase,
+    ModelRAGStrategy,
+)
+from rdagent.components.coder.model_coder.model import ModelExperiment
+from rdagent.core.developer import Developer
+from rdagent.core.evolving_agent import RAGEvoAgent
+
+
+class ModelCoSTEER(Developer[ModelExperiment]):
+    def __init__(
+        self,
+        *args,
+        with_knowledge: bool = True,
+        with_feedback: bool = True,
+        knowledge_self_gen: bool = True,
+        filter_final_evo: bool = True,
+        **kwargs,
+    ) -> None:
+        super().__init__(*args, **kwargs)
+        self.max_loop = MODEL_IMPL_SETTINGS.max_loop
+        self.knowledge_base_path = (
+            Path(MODEL_IMPL_SETTINGS.knowledge_base_path)
+            if MODEL_IMPL_SETTINGS.knowledge_base_path is not None
+            else None
+        )
+        self.new_knowledge_base_path = (
+            Path(MODEL_IMPL_SETTINGS.new_knowledge_base_path)
+            if MODEL_IMPL_SETTINGS.new_knowledge_base_path is not None
+            else None
+        )
+        self.with_knowledge = with_knowledge
+        self.with_feedback = with_feedback
+        self.knowledge_self_gen = knowledge_self_gen
+        self.filter_final_evo = filter_final_evo
+        self.evolving_strategy = ModelCoderEvolvingStrategy(scen=self.scen)
+        self.model_evaluator = ModelCoderMultiEvaluator(scen=self.scen)
+
+    def load_or_init_knowledge_base(self, former_knowledge_base_path: Path = None, component_init_list: list = []):
+        if former_knowledge_base_path is not None and former_knowledge_base_path.exists():
+            model_knowledge_base = pickle.load(open(former_knowledge_base_path, "rb"))
+            if not isinstance(model_knowledge_base, ModelKnowledgeBase):
+                raise ValueError("The former knowledge base is not compatible with the current version")
+        else:
+            model_knowledge_base = ModelKnowledgeBase()
+
+        return model_knowledge_base
+
+    def develop(self, exp: ModelExperiment) -> ModelExperiment:
+        # init knowledge base
+        model_knowledge_base = self.load_or_init_knowledge_base(
+            former_knowledge_base_path=self.knowledge_base_path,
+            component_init_list=[],
+        )
+        # init rag method
+        self.rag = ModelRAGStrategy(model_knowledge_base)
+
+        # init intermediate items
+        model_experiment = ModelEvolvingItem.from_experiment(exp)
+
+        self.evolve_agent = ModelRAGEvoAgent(
+            max_loop=self.max_loop,
+            evolving_strategy=self.evolving_strategy,
+            rag=self.rag,
+            with_knowledge=self.with_knowledge,
+            with_feedback=self.with_feedback,
+            knowledge_self_gen=self.knowledge_self_gen,
+        )
+
+        model_experiment = self.evolve_agent.multistep_evolve(
+            model_experiment,
+            self.model_evaluator,
+            filter_final_evo=self.filter_final_evo,
+        )
+
+        # save new knowledge base
+        if self.new_knowledge_base_path is not None:
+            pickle.dump(model_knowledge_base, open(self.new_knowledge_base_path, "wb"))
+        exp.sub_workspace_list = model_experiment.sub_workspace_list
+        return exp

rdagent/components/coder/model_coder/CoSTEER/evaluators.py

@@ -0,0 +1,343 @@
+import json
+import random
+from pathlib import Path
+from typing import List, Tuple
+
+import numpy as np
+from jinja2 import Environment, StrictUndefined
+
+from rdagent.components.coder.model_coder.conf import MODEL_IMPL_SETTINGS
+from rdagent.components.coder.model_coder.CoSTEER.evolvable_subjects import (
+    ModelEvolvingItem,
+)
+from rdagent.components.coder.model_coder.model import ModelFBWorkspace, ModelTask
+from rdagent.core.conf import RD_AGENT_SETTINGS
+from rdagent.core.evaluation import Evaluator
+from rdagent.core.evolving_framework import QueriedKnowledge
+from rdagent.core.experiment import Task, Workspace
+from rdagent.core.prompts import Prompts
+from rdagent.core.utils import multiprocessing_wrapper
+from rdagent.log import rdagent_logger as logger
+from rdagent.oai.llm_conf import LLM_SETTINGS
+from rdagent.oai.llm_utils import APIBackend
+
+evaluate_prompts = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")
+
+
+def shape_evaluator(prediction: np.ndarray, target_shape: Tuple = None) -> Tuple[str, bool]:
+    if target_shape is None or prediction is None:
+        return (
+            "No output generated from the model. No shape evaluation conducted.",
+            False,
+        )
+    pre_shape = prediction.shape
+
+    if pre_shape == target_shape:
+        return "The shape of the output is correct.", True
+    else:
+        return (
+            f"The shape of the output is incorrect. Expected {target_shape}, but got {pre_shape}.",
+            False,
+        )
+
+
+def value_evaluator(
+    prediction: np.ndarray,
+    target: np.ndarray,
+) -> Tuple[np.ndarray, bool]:
+    if prediction is None:
+        return "No output generated from the model. Skip value evaluation", False
+    elif target is None:
+        return (
+            "No ground truth output provided. Value evaluation not impractical",
+            False,
+        )
+    else:
+        # Calculate the mean absolute difference
+        diff = np.mean(np.abs(target - prediction))
+        return (
+            f"The value of the output is correct. The mean absolute difference is {diff}.",
+            diff < 0.1,
+        )
+
+
+class ModelCodeEvaluator(Evaluator):
+    def evaluate(
+        self,
+        target_task: Task,
+        implementation: Workspace,
+        gt_implementation: Workspace,
+        model_execution_feedback: str = "",
+        model_value_feedback: str = "",
+    ):
+        assert isinstance(target_task, ModelTask)
+        assert isinstance(implementation, ModelFBWorkspace)
+        if gt_implementation is not None:
+            assert isinstance(gt_implementation, ModelFBWorkspace)
+
+        model_task_information = target_task.get_task_information()
+        code = implementation.code
+
+        system_prompt = (
+            Environment(undefined=StrictUndefined)
+            .from_string(evaluate_prompts["evaluator_code_feedback"]["system"])
+            .render(
+                scenario=(
+                    self.scen.get_scenario_all_desc(target_task)
+                    if self.scen is not None
+                    else "No scenario description."
+                )
+            )
+        )
+
+        execution_feedback_to_render = model_execution_feedback
+        for _ in range(10):  # 10 times to split the content is enough
+            user_prompt = (
+                Environment(undefined=StrictUndefined)
+                .from_string(
+                    evaluate_prompts["evaluator_code_feedback"]["user"],
+                )
+                .render(
+                    model_information=model_task_information,
+                    code=code,
+                    model_execution_feedback=execution_feedback_to_render,
+                    model_value_feedback=model_value_feedback,
+                    gt_code=gt_implementation.code if gt_implementation else None,
+                )
+            )
+            if (
+                APIBackend().build_messages_and_calculate_token(
+                    user_prompt=user_prompt,
+                    system_prompt=system_prompt,
+                )
+                > LLM_SETTINGS.chat_token_limit
+            ):
+                execution_feedback_to_render = execution_feedback_to_render[len(execution_feedback_to_render) // 2 :]
+            else:
+                break
+
+        critic_response = APIBackend().build_messages_and_create_chat_completion(
+            user_prompt=user_prompt,
+            system_prompt=system_prompt,
+            json_mode=False,
+        )
+
+        return critic_response, None
+
+
+class ModelFinalEvaluator(Evaluator):
+    def evaluate(
+        self,
+        target_task: Task,
+        implementation: Workspace,
+        gt_implementation: Workspace,
+        model_execution_feedback: str,
+        model_value_feedback: str,
+        model_code_feedback: str,
+    ):
+        assert isinstance(target_task, ModelTask)
+        assert isinstance(implementation, ModelFBWorkspace)
+        if gt_implementation is not None:
+            assert isinstance(gt_implementation, ModelFBWorkspace)
+
+        system_prompt = (
+            Environment(undefined=StrictUndefined)
+            .from_string(evaluate_prompts["evaluator_final_feedback"]["system"])
+            .render(
+                scenario=(
+                    self.scen.get_scenario_all_desc(target_task)
+                    if self.scen is not None
+                    else "No scenario description."
+                )
+            )
+        )
+
+        execution_feedback_to_render = model_execution_feedback
+
+        for _ in range(10):  # 10 times to split the content is enough
+            user_prompt = (
+                Environment(undefined=StrictUndefined)
+                .from_string(
+                    evaluate_prompts["evaluator_final_feedback"]["user"],
+                )
+                .render(
+                    model_information=target_task.get_task_information(),
+                    model_execution_feedback=execution_feedback_to_render,
+                    model_code_feedback=model_code_feedback,
+                    model_value_feedback=model_value_feedback,
+                )
+            )
+            if (
+                APIBackend().build_messages_and_calculate_token(
+                    user_prompt=user_prompt,
+                    system_prompt=system_prompt,
+                )
+                > LLM_SETTINGS.chat_token_limit
+            ):
+                execution_feedback_to_render = execution_feedback_to_render[len(execution_feedback_to_render) // 2 :]
+            else:
+                break
+
+        final_evaluation_dict = json.loads(
+            APIBackend().build_messages_and_create_chat_completion(
+                user_prompt=user_prompt,
+                system_prompt=system_prompt,
+                json_mode=True,
+            ),
+        )
+        if isinstance(final_evaluation_dict["final_decision"], str) and final_evaluation_dict[
+            "final_decision"
+        ].lower() in ("true", "false"):
+            final_evaluation_dict["final_decision"] = bool(final_evaluation_dict["final_decision"])
+        return (
+            final_evaluation_dict["final_feedback"],
+            final_evaluation_dict["final_decision"],
+        )
+
+
+class ModelCoderFeedback:
+    """This feedback includes all the content to the model coder"""
+
+    def __init__(
+        self,
+        execution_feedback: str,
+        shape_feedback: str,
+        value_feedback: str,
+        code_feedback: str,
+        final_feedback: str,
+        final_decision: bool,
+    ):
+        self.execution_feedback: str = execution_feedback
+        self.shape_feedback: str = shape_feedback
+        self.value_feedback: str = value_feedback
+        self.code_feedback: str = code_feedback
+        self.final_feedback: str = final_feedback
+        self.final_decision: str = final_decision
+
+    def __str__(self) -> str:
+        return f"""------------------Model Execution Feedback------------------
+{self.execution_feedback}
+------------------Model Shape Feedback------------------
+{self.shape_feedback}
+------------------Model Value Feedback------------------
+{self.value_feedback}
+------------------Model Code Feedback------------------
+{self.code_feedback}
+------------------Model Final Feedback------------------
+{self.final_feedback}
+------------------Model Final Decision------------------
+This implementation is {'SUCCESS' if self.final_decision else 'FAIL'}.
+"""
+
+
+class ModelCoderEvaluator(Evaluator):
+    def evaluate(
+        self,
+        target_task: Task,
+        implementation: Workspace,
+        gt_implementation: Workspace,
+        queried_knowledge: QueriedKnowledge = None,
+        **kwargs,
+    ) -> ModelCoderFeedback:
+        target_task_information = target_task.get_task_information()
+        if (
+            queried_knowledge is not None
+            and target_task_information in queried_knowledge.success_task_to_knowledge_dict
+        ):
+            return queried_knowledge.success_task_to_knowledge_dict[target_task_information].feedback
+        elif queried_knowledge is not None and target_task_information in queried_knowledge.failed_task_info_set:
+            return ModelCoderFeedback(
+                execution_feedback="This task has failed too many times, skip implementation.",
+                shape_feedback="This task has failed too many times, skip implementation.",
+                value_feedback="This task has failed too many times, skip implementation.",
+                code_feedback="This task has failed too many times, skip implementation.",
+                final_feedback="This task has failed too many times, skip implementation.",
+                final_decision=False,
+            )
+        assert isinstance(target_task, ModelTask)
+
+        # NOTE: Use fixed input to test the model to avoid randomness
+        batch_size = 8
+        num_features = 30
+        num_timesteps = 40
+        input_value = 0.4
+        param_init_value = 0.6
+
+        assert isinstance(implementation, ModelFBWorkspace)
+        model_execution_feedback, gen_np_array = implementation.execute(
+            batch_size=batch_size,
+            num_features=num_features,
+            num_timesteps=num_timesteps,
+            input_value=input_value,
+            param_init_value=param_init_value,
+        )
+        if gt_implementation is not None:
+            assert isinstance(gt_implementation, ModelFBWorkspace)
+            _, gt_np_array = gt_implementation.execute(
+                batch_size=batch_size,
+                num_features=num_features,
+                num_timesteps=num_timesteps,
+                input_value=input_value,
+                param_init_value=param_init_value,
+            )
+        else:
+            gt_np_array = None
+
+        shape_feedback, shape_decision = shape_evaluator(gen_np_array, (batch_size, 1))
+        value_feedback, value_decision = value_evaluator(gen_np_array, gt_np_array)
+        code_feedback, _ = ModelCodeEvaluator(scen=self.scen).evaluate(
+            target_task=target_task,
+            implementation=implementation,
+            gt_implementation=gt_implementation,
+            model_execution_feedback=model_execution_feedback,
+            model_value_feedback="\n".join([shape_feedback, value_feedback]),
+        )
+        final_feedback, final_decision = ModelFinalEvaluator(scen=self.scen).evaluate(
+            target_task=target_task,
+            implementation=implementation,
+            gt_implementation=gt_implementation,
+            model_execution_feedback=model_execution_feedback,
+            model_value_feedback=value_feedback,
+            model_code_feedback=code_feedback,
+        )
+
+        return ModelCoderFeedback(
+            execution_feedback=model_execution_feedback,
+            shape_feedback=shape_feedback,
+            value_feedback=value_feedback,
+            code_feedback=code_feedback,
+            final_feedback=final_feedback,
+            final_decision=final_decision,
+        )
+
+
+class ModelCoderMultiEvaluator(Evaluator):
+    def evaluate(
+        self,
+        evo: ModelEvolvingItem,
+        queried_knowledge: QueriedKnowledge = None,
+        **kwargs,
+    ) -> List[ModelCoderFeedback]:
+        multi_implementation_feedback = multiprocessing_wrapper(
+            [
+                (
+                    ModelCoderEvaluator(scen=self.scen).evaluate,
+                    (
+                        evo.sub_tasks[index],
+                        evo.sub_workspace_list[index],
+                        evo.sub_gt_implementations[index] if evo.sub_gt_implementations is not None else None,
+                        queried_knowledge,
+                    ),
+                )
+                for index in range(len(evo.sub_tasks))
+            ],
+            n=RD_AGENT_SETTINGS.multi_proc_n,
+        )
+
+        final_decision = [
+            None if single_feedback is None else single_feedback.final_decision
+            for single_feedback in multi_implementation_feedback
+        ]
+        logger.info(f"Final decisions: {final_decision} True count: {final_decision.count(True)}")
+
+        return multi_implementation_feedback

rdagent/components/coder/model_coder/CoSTEER/evolvable_subjects.py

@@ -0,0 +1,36 @@
+from rdagent.components.coder.model_coder.model import (
+    ModelExperiment,
+    ModelFBWorkspace,
+    ModelTask,
+)
+from rdagent.core.evolving_framework import EvolvableSubjects
+from rdagent.log import rdagent_logger as logger
+
+
+class ModelEvolvingItem(ModelExperiment, EvolvableSubjects):
+    """
+    Intermediate item of model implementation.
+    """
+
+    def __init__(
+        self,
+        sub_tasks: list[ModelTask],
+        sub_gt_implementations: list[ModelFBWorkspace] = None,
+    ):
+        ModelExperiment.__init__(self, sub_tasks=sub_tasks)
+        if sub_gt_implementations is not None and len(
+            sub_gt_implementations,
+        ) != len(self.sub_tasks):
+            self.sub_gt_implementations = None
+            logger.warning(
+                "The length of sub_gt_implementations is not equal to the length of sub_tasks, set sub_gt_implementations to None",
+            )
+        else:
+            self.sub_gt_implementations = sub_gt_implementations
+
+    @classmethod
+    def from_experiment(cls, exp: ModelExperiment) -> "ModelEvolvingItem":
+        ei = cls(sub_tasks=exp.sub_tasks)
+        ei.based_experiments = exp.based_experiments
+        ei.experiment_workspace = exp.experiment_workspace
+        return ei

rdagent/components/coder/model_coder/CoSTEER/evolving_agent.py

@@ -0,0 +1,19 @@
+from rdagent.components.coder.model_coder.CoSTEER.evaluators import ModelCoderFeedback
+from rdagent.components.coder.model_coder.CoSTEER.evolvable_subjects import (
+    ModelEvolvingItem,
+)
+from rdagent.core.evaluation import Feedback
+from rdagent.core.evolving_agent import RAGEvoAgent
+from rdagent.core.evolving_framework import EvolvableSubjects
+
+
+class ModelRAGEvoAgent(RAGEvoAgent):
+    def filter_evolvable_subjects_by_feedback(self, evo: EvolvableSubjects, feedback: Feedback) -> EvolvableSubjects:
+        assert isinstance(evo, ModelEvolvingItem)
+        assert isinstance(feedback, list)
+        assert len(evo.sub_workspace_list) == len(feedback)
+
+        for index in range(len(evo.sub_workspace_list)):
+            if not feedback[index].final_decision:
+                evo.sub_workspace_list[index].clear()
+        return evo

rdagent/components/coder/model_coder/CoSTEER/evolving_strategy.py

@@ -0,0 +1,158 @@
+import json
+from copy import deepcopy
+from pathlib import Path
+
+from jinja2 import Environment, StrictUndefined
+
+from rdagent.components.coder.model_coder.conf import MODEL_IMPL_SETTINGS
+from rdagent.components.coder.model_coder.CoSTEER.evolvable_subjects import (
+    ModelEvolvingItem,
+)
+from rdagent.components.coder.model_coder.CoSTEER.knowledge_management import (
+    ModelQueriedKnowledge,
+)
+from rdagent.components.coder.model_coder.model import (
+    ModelExperiment,
+    ModelFBWorkspace,
+    ModelTask,
+)
+from rdagent.core.conf import RD_AGENT_SETTINGS
+from rdagent.core.evolving_framework import EvolvingStrategy
+from rdagent.core.prompts import Prompts
+from rdagent.core.utils import multiprocessing_wrapper
+from rdagent.oai.llm_conf import LLM_SETTINGS
+from rdagent.oai.llm_utils import APIBackend
+from rdagent.scenarios.kaggle.experiment.kaggle_experiment import KG_MODEL_MAPPING
+
+coder_prompts = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")
+
+
+class ModelCoderEvolvingStrategy(EvolvingStrategy):
+    def implement_one_model(
+        self,
+        target_task: ModelTask,
+        queried_knowledge: ModelQueriedKnowledge = None,
+        current_exp: ModelExperiment = None,  # Add this parameter
+    ) -> str:
+        model_information_str = target_task.get_task_information()
+        model_type = target_task.model_type
+
+        if len(current_exp.based_experiments) == 0:
+            current_code = None
+        else:
+            current_code = ""
+            sota_exp_code_dict = current_exp.based_experiments[-1].experiment_workspace.code_dict
+            if target_task.version == 2:
+                if model_type in KG_MODEL_MAPPING:
+                    current_code = sota_exp_code_dict.get(KG_MODEL_MAPPING[model_type], None)
+                elif "model.py" in sota_exp_code_dict:
+                    current_code = sota_exp_code_dict["model.py"]
+                else:
+                    current_code = None
+            elif target_task.version == 1:
+                current_code = sota_exp_code_dict.get("model.py", None)
+
+        if queried_knowledge is not None and model_information_str in queried_knowledge.success_task_to_knowledge_dict:
+            return queried_knowledge.success_task_to_knowledge_dict[model_information_str].implementation
+        elif queried_knowledge is not None and model_information_str in queried_knowledge.failed_task_info_set:
+            return None
+        else:
+            queried_similar_successful_knowledge = (
+                queried_knowledge.working_task_to_similar_successful_knowledge_dict[model_information_str]
+                if queried_knowledge is not None
+                else []
+            )
+            queried_former_failed_knowledge = (
+                queried_knowledge.working_task_to_former_failed_knowledge_dict[model_information_str]
+                if queried_knowledge is not None
+                else []
+            )
+
+            queried_former_failed_knowledge_to_render = queried_former_failed_knowledge
+
+            system_prompt = (
+                Environment(undefined=StrictUndefined)
+                .from_string(
+                    coder_prompts["evolving_strategy_model_coder"]["system"],
+                )
+                .render(
+                    scenario=self.scen.get_scenario_all_desc(),
+                    queried_former_failed_knowledge=queried_former_failed_knowledge_to_render,
+                    current_code=current_code,
+                )
+            )
+
+            queried_similar_successful_knowledge_to_render = queried_similar_successful_knowledge
+            for _ in range(10):  # max attempt to reduce the length of user_prompt
+                user_prompt = (
+                    Environment(undefined=StrictUndefined)
+                    .from_string(
+                        coder_prompts["evolving_strategy_model_coder"]["user"],
+                    )
+                    .render(
+                        model_information_str=model_information_str,
+                        queried_similar_successful_knowledge=queried_similar_successful_knowledge_to_render,
+                        queried_former_failed_knowledge=queried_former_failed_knowledge_to_render,
+                    )
+                    .strip("\n")
+                )
+                if (
+                    APIBackend().build_messages_and_calculate_token(
+                        user_prompt=user_prompt,
+                        system_prompt=system_prompt,
+                    )
+                    < LLM_SETTINGS.chat_token_limit
+                ):
+                    break
+                elif len(queried_former_failed_knowledge_to_render) > 1:
+                    queried_former_failed_knowledge_to_render = queried_former_failed_knowledge_to_render[1:]
+                elif len(queried_similar_successful_knowledge_to_render) > 1:
+                    queried_similar_successful_knowledge_to_render = queried_similar_successful_knowledge_to_render[1:]
+
+            code = json.loads(
+                APIBackend(
+                    use_chat_cache=MODEL_IMPL_SETTINGS.coder_use_cache
+                ).build_messages_and_create_chat_completion(
+                    user_prompt=user_prompt,
+                    system_prompt=system_prompt,
+                    json_mode=True,
+                ),
+            )["code"]
+            return code
+
+    def evolve(
+        self,
+        *,
+        evo: ModelEvolvingItem,
+        queried_knowledge: ModelQueriedKnowledge | None = None,
+        **kwargs,
+    ) -> ModelEvolvingItem:
+        # 1.找出需要evolve的model
+        to_be_finished_task_index = []
+        for index, target_model_task in enumerate(evo.sub_tasks):
+            target_model_task_desc = target_model_task.get_task_information()
+            if target_model_task_desc in queried_knowledge.success_task_to_knowledge_dict:
+                evo.sub_workspace_list[index] = queried_knowledge.success_task_to_knowledge_dict[
+                    target_model_task_desc
+                ].implementation
+            elif (
+                target_model_task_desc not in queried_knowledge.success_task_to_knowledge_dict
+                and target_model_task_desc not in queried_knowledge.failed_task_info_set
+            ):
+                to_be_finished_task_index.append(index)
+
+        result = multiprocessing_wrapper(
+            [
+                (self.implement_one_model, (evo.sub_tasks[target_index], queried_knowledge, evo))
+                for target_index in to_be_finished_task_index
+            ],
+            n=RD_AGENT_SETTINGS.multi_proc_n,
+        )
+
+        for index, target_index in enumerate(to_be_finished_task_index):
+            evo.sub_workspace_list[target_index] = ModelFBWorkspace(target_task=evo.sub_tasks[target_index])
+            evo.sub_workspace_list[target_index].inject_code(**{"model.py": result[index]})
+
+        evo.corresponding_selection = to_be_finished_task_index
+
+        return evo

rdagent/components/coder/model_coder/CoSTEER/knowledge_management.py

@@ -0,0 +1,173 @@
+from pathlib import Path
+
+from rdagent.components.coder.model_coder.conf import MODEL_IMPL_SETTINGS
+from rdagent.components.coder.model_coder.CoSTEER.evaluators import ModelCoderFeedback
+from rdagent.components.coder.model_coder.model import ModelTask
+from rdagent.core.evolving_framework import (
+    EvolvableSubjects,
+    EvolvingKnowledgeBase,
+    EvoStep,
+    Knowledge,
+    QueriedKnowledge,
+    RAGStrategy,
+)
+from rdagent.core.experiment import Workspace
+from rdagent.oai.llm_utils import calculate_embedding_distance_between_str_list
+
+
+class ModelKnowledge(Knowledge):
+    def __init__(
+        self,
+        target_task: ModelTask,
+        implementation: Workspace,
+        feedback: ModelCoderFeedback,
+    ) -> None:
+        """
+        Initialize a ModelKnowledge object. The ModelKnowledge object is used to store a model implementation without the ground truth code and value.
+
+        Args:
+            model (Model): The model object associated with the KnowledgeManagement.
+
+        Returns:
+            None
+        """
+        self.target_task = target_task
+        self.implementation = implementation.copy()
+        self.feedback = feedback
+
+    def get_implementation_and_feedback_str(self) -> str:
+        return f"""------------------Model implementation code:------------------
+{self.implementation.code}
+------------------Model implementation feedback:------------------
+{self.feedback!s}
+"""
+
+
+class ModelQueriedKnowledge(QueriedKnowledge):
+    def __init__(self, success_task_to_knowledge_dict: dict = {}, failed_task_info_set: set = set()) -> None:
+        self.success_task_to_knowledge_dict = success_task_to_knowledge_dict
+        self.failed_task_info_set = failed_task_info_set
+        self.working_task_to_former_failed_knowledge_dict = dict()
+        self.working_task_to_similar_successful_knowledge_dict = dict()
+
+
+class ModelKnowledgeBase(EvolvingKnowledgeBase):
+    def __init__(self, path: str | Path = None) -> None:
+        self.implementation_trace: dict[str, ModelKnowledge] = dict()
+        self.success_task_info_set: set[str] = set()
+
+        self.task_to_embedding = dict()
+
+        super().__init__(path)
+
+    def query(self) -> QueriedKnowledge | None:
+        """
+        Query the knowledge base to get the queried knowledge. So far is handled in RAG strategy.
+        """
+        raise NotImplementedError
+
+
+class ModelRAGStrategy(RAGStrategy):
+    def __init__(self, knowledgebase: ModelKnowledgeBase) -> None:
+        super().__init__(knowledgebase)
+        self.current_generated_trace_count = 0
+
+    def generate_knowledge(
+        self,
+        evolving_trace: list[EvoStep],
+        *,
+        return_knowledge: bool = False,
+    ) -> Knowledge | None:
+        if len(evolving_trace) == self.current_generated_trace_count:
+            return
+        else:
+            for trace_index in range(
+                self.current_generated_trace_count,
+                len(evolving_trace),
+            ):
+                evo_step = evolving_trace[trace_index]
+                implementations = evo_step.evolvable_subjects
+                feedback = evo_step.feedback
+                for task_index in range(len(implementations.sub_tasks)):
+                    target_task = implementations.sub_tasks[task_index]
+                    target_task_information = target_task.get_task_information()
+                    implementation = implementations.sub_workspace_list[task_index]
+                    single_feedback = feedback[task_index]
+                    if single_feedback is None:
+                        continue
+                    single_knowledge = ModelKnowledge(
+                        target_task=target_task,
+                        implementation=implementation,
+                        feedback=single_feedback,
+                    )
+                    if target_task_information not in self.knowledgebase.success_task_info_set:
+                        self.knowledgebase.implementation_trace.setdefault(
+                            target_task_information,
+                            [],
+                        ).append(single_knowledge)
+
+                        if single_feedback.final_decision == True:
+                            self.knowledgebase.success_task_info_set.add(
+                                target_task_information,
+                            )
+            self.current_generated_trace_count = len(evolving_trace)
+
+    def query(
+        self,
+        evo: EvolvableSubjects,
+        evolving_trace: list[EvoStep],
+    ) -> QueriedKnowledge | None:
+        query_former_trace_limit = MODEL_IMPL_SETTINGS.query_former_trace_limit
+        query_similar_success_limit = MODEL_IMPL_SETTINGS.query_similar_success_limit
+        fail_task_trial_limit = MODEL_IMPL_SETTINGS.fail_task_trial_limit
+
+        queried_knowledge = ModelQueriedKnowledge()
+        for target_model_task in evo.sub_tasks:
+            target_model_task_information = target_model_task.get_task_information()
+            if target_model_task_information in self.knowledgebase.success_task_info_set:
+                queried_knowledge.success_task_to_knowledge_dict[
+                    target_model_task_information
+                ] = self.knowledgebase.implementation_trace[target_model_task_information][-1]
+            elif (
+                len(
+                    self.knowledgebase.implementation_trace.setdefault(
+                        target_model_task_information,
+                        [],
+                    ),
+                )
+                >= fail_task_trial_limit
+            ):
+                queried_knowledge.failed_task_info_set.add(target_model_task_information)
+            else:
+                queried_knowledge.working_task_to_former_failed_knowledge_dict[
+                    target_model_task_information
+                ] = self.knowledgebase.implementation_trace.setdefault(
+                    target_model_task_information,
+                    [],
+                )[
+                    -query_former_trace_limit:
+                ]
+
+                knowledge_base_success_task_list = list(
+                    self.knowledgebase.success_task_info_set,
+                )
+                similarity = calculate_embedding_distance_between_str_list(
+                    [target_model_task_information],
+                    knowledge_base_success_task_list,
+                )[0]
+                similar_indexes = sorted(
+                    range(len(similarity)),
+                    key=lambda i: similarity[i],
+                    reverse=True,
+                )[:query_similar_success_limit]
+                similar_successful_knowledge = [
+                    self.knowledgebase.implementation_trace.setdefault(
+                        knowledge_base_success_task_list[index],
+                        [],
+                    )[-1]
+                    for index in similar_indexes
+                ]
+                queried_knowledge.working_task_to_similar_successful_knowledge_dict[
+                    target_model_task_information
+                ] = similar_successful_knowledge
+        return queried_knowledge

rdagent/components/coder/model_coder/benchmark/eval.py

@@ -1,6 +1,7 @@
 # TODO: inherent from the benchmark base class
 import torch
-from rdagent.model_implementation.task import ModelTaskImpl
+
+from rdagent.components.coder.model_coder.model import ModelFBWorkspace
 
 
 def get_data_conf(init_val):
@@ -15,14 +16,23 @@ def get_data_conf(init_val):
 
 class ModelImpValEval:
     """
-    Evaluate the similarity of the model structure by changing the input and observate the output.
+    Evaluate the similarity of the model structure by changing the input and observe the output.
 
     Assumption:
     - If the model structure is similar, the output will change in similar way when we change the input.
-        - we try to initialize the model param in similar value. So only the model structure is different.
+
+    Challenge:
+    - The key difference between it and implementing models is that we have parameters in the layers (Model operators often have no parameters or are given parameters).
+    - we try to initialize the model param in similar value. So only the model structure is different.
+
+    Comparing the correlation of following sequences
+    - modelA[init1](input1).hidden_out1, modelA[init1](input2).hidden_out1, ...
+    - modelB[init1](input1).hidden_out1, modelB[init1](input2).hidden_out1, ...
+
+    For each hidden output, we can calculate a correlation. The average correlation will be the metrics.
     """
 
-    def evaluate(self, gt: ModelTaskImpl, gen: ModelTaskImpl):
+    def evaluate(self, gt: ModelFBWorkspace, gen: ModelFBWorkspace):
         round_n = 10
 
         eval_pairs: list[tuple] = []
@@ -31,9 +41,8 @@ class ModelImpValEval:
         for _ in range(round_n):
             # run different model initial parameters.
             for init_val in [-0.2, -0.1, 0.1, 0.2]:
-                data, exec_config = get_data_conf(init_val)
-                gt_res = gt.execute(data=data, config=exec_config)
-                res = gen.execute(data=data, config=exec_config)
+                _, gt_res = gt.execute(input_value=init_val, param_init_value=init_val)
+                _, res = gen.execute(input_value=init_val, param_init_value=init_val)
                 eval_pairs.append((res, gt_res))
 
         # flat and concat the output
@@ -50,12 +59,13 @@ class ModelImpValEval:
         # pearson correlation of each hidden output
         def norm(x):
             return (x - x.mean(axis=0)) / x.std(axis=0)
+
         dim_corr = (norm(res_batch) * norm(gt_res_batch)).mean(axis=0)  # the correlation of each hidden output
 
         # aggregate all the correlation
         avr_corr = dim_corr.mean()
-        # FIXME: 
+        # FIXME:
         # It is too high(e.g. 0.944) .
-        # Check if it is not a good evaluation!! 
+        # Check if it is not a good evaluation!!
         # Maybe all the same initial params will results in extreamly high correlation without regard to the model structure.
         return avr_corr

rdagent/components/coder/model_coder/benchmark/gt_code/A-DGN.py

@@ -4,7 +4,6 @@ from typing import Any, Callable, Dict, Optional, Union
 import torch
 from torch import Tensor
 from torch.nn import Parameter
-
 from torch_geometric.nn.conv import GCNConv, MessagePassing
 from torch_geometric.nn.inits import zeros
 from torch_geometric.nn.resolver import activation_resolver