chore(main): release 0.3.0 (#258)

Release-As: 0.3.0

.commitlintrc.js

@@ -0,0 +1,21 @@
+module.exports = {
+    extends: ["@commitlint/config-conventional"],
+    rules: {
+        // Configuration Format: [level, applicability, value]
+        // level: Error level, usually expressed as a number:
+        //     0 - disable rule
+        //     1 - Warning (does not prevent commits)
+        //     2 - Error (will block the commit)
+        // applicability: the conditions under which the rule applies, commonly used values:
+        //     “always” - always apply the rule
+        //     “never” - never apply the rule
+        // value: the specific value of the rule, e.g. a maximum length of 100.
+        // Refs: https://commitlint.js.org/reference/rules-configuration.html
+      "header-max-length": [2, "always", 100],
+      "type-enum": [
+        2,
+        "always",
+        ["build", "chore", "ci", "docs", "feat", "fix", "perf", "refactor", "revert", "style", "test", "Release-As"]
+      ]
+    }
+  };

.github/ISSUE_TEMPLATE/bug-report.md

@@ -0,0 +1,51 @@
+---
+name: "\U0001F41B Bug Report"
+about: Submit a bug report to help us improve RD-Agent
+labels: bug
+
+---
+
+## 🐛 Bug Description
+
+<!-- A clear and concise description of what the bug is. -->
+
+## To Reproduce
+
+Steps to reproduce the behavior:
+
+1.
+2.
+3.
+
+
+## Expected Behavior
+
+<!-- A clear and concise description of what you expected to happen. -->
+
+## Screenshot
+
+<!-- A screenshot of the error message or anything shouldn't appear-->
+
+## Environment
+
+**Note**: Users can run `rdagent collect_info` to get system information and paste it directly here.
+
+ - Name of current operating system:
+ - Processor architecture:
+ - System, version, and hardware information:
+ - Version number of the system:
+ - Python version:
+ - Container ID:
+ - Container Name:
+ - Container Status:
+ - Image ID used by the container:
+ - Image tag used by the container:
+ - Container port mapping:
+ - Container Label:
+ - Startup Commands:
+ - RD-Agent version:
+ - Package version:
+
+## Additional Notes
+
+<!-- Add any other information about the problem here. -->

.github/ISSUE_TEMPLATE/documentation.md

@@ -0,0 +1,9 @@
+---
+name: "\U0001F4D6 Documentation"
+about: Report an issue related to documentation
+
+---
+
+## 📖 Documentation
+
+<!-- Please specify whether it's tutorial part or API reference part, and describe it.-->

.github/ISSUE_TEMPLATE/feature-request.md

@@ -0,0 +1,25 @@
+---
+name: "\U0001F31FFeature Request"
+about: Request for a new RD-Agent feature
+labels: enhancement
+
+---
+
+## 🌟 Feature Description
+<!-- A clear and concise description of the feature proposal -->
+
+## Motivation
+
+1. Application scenario
+2. Related works (Papers, Github repos etc.):
+3. Any other relevant and important information:
+
+<!-- Please describe why the feature is important. -->
+
+## Alternatives
+
+<!-- A short description of any alternative solutions or features you've considered. -->
+
+## Additional Notes
+
+<!-- Add any other context or screenshots about the feature request here. -->

.github/ISSUE_TEMPLATE/question.md

@@ -0,0 +1,10 @@
+---
+name: "❓Questions & Help"
+about: Have some questions? We can offer help.
+labels: question
+
+---
+
+## ❓ Questions and Help
+
+We sincerely suggest you to carefully read the [documentation](http://rdagent.readthedocs.io/). After that, if you still feel puzzled, please describe the question clearly under this issue.

.github/workflows/ci.yml

@@ -20,7 +20,7 @@ jobs:
       - run: env | sort
       - run: make dev
       - name: lint test docs and build
-        run: make lint docs-gen # test docs build
+        run: make lint docs-gen test-offline # test docs build
     strategy:
       matrix:
         python-version:

.github/workflows/pr.yml

@@ -1,18 +1,5 @@
-concurrency:
-  cancel-in-progress: true
-  group: ${{ github.workflow }}-${{ github.ref }}
-jobs:
-  lint-title:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Check PR Title for Conventional Commit Format
-        run: |
-          if ! echo "${{ github.event.pull_request.title }}" | grep -Pq '^(build|chore|ci|docs|feat|fix|perf|refactor|revert|style|test|Release-As)(\(\w+\))?!?:\s.*'; then
-            echo 'The title does not conform to the Conventional Commit.'
-            echo 'Please refer to "https://www.conventionalcommits.org/"'
-            exit 1
-          fi
 name: Lint pull request title
+
 on:
   pull_request:
     types:
@@ -20,3 +7,29 @@ on:
       - synchronize
       - reopened
       - edited
+
+concurrency:
+  cancel-in-progress: true
+  group: ${{ github.workflow }}-${{ github.ref }}
+
+jobs:
+  lint-title:
+    runs-on: ubuntu-latest
+    steps:
+      # This step is necessary because the lint title uses the .commitlintrc.js file in the project root directory.
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v3
+        with:
+          node-version: '16'
+
+      - name: Install commitlint
+        run: npm install --save-dev @commitlint/{config-conventional,cli}
+
+      - name: Validate PR Title with commitlint
+        env:
+          BODY: ${{ github.event.pull_request.title }}
+        run: |
+          echo "$BODY" | npx commitlint --config .commitlintrc.js

.readthedocs.yaml

@@ -0,0 +1,26 @@
+# .readthedocs.yml
+# Read the Docs configuration file
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+# Required
+version: 2
+
+# Set the version of Python and other tools you might need
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.10"
+
+# Build documentation in the docs/ directory with Sphinx
+sphinx:
+  configuration: docs/conf.py
+
+# Build all formats
+formats: all
+
+# Optionally set the version of Python and requirements required to build your docs
+python:
+  install:
+    - requirements: requirements/docs.txt
+    - method: pip
+      path: .

.streamlit/config.toml

@@ -0,0 +1,2 @@
+[client]
+showSidebarNavigation = false

CHANGELOG.md

@@ -1,5 +1,157 @@
 # Changelog
 
+## [0.3.0](https://github.com/microsoft/RD-Agent/compare/v0.2.1...v0.3.0) (2024-10-21)
+
+
+### Features
+
+* add a new template for kaggle ([#289](https://github.com/microsoft/RD-Agent/issues/289)) ([eee3ab5](https://github.com/microsoft/RD-Agent/commit/eee3ab5b25198224826cb7a8a17eab28bd5d1f7d))
+* add download submission.csv button for kaggle scenario ([#317](https://github.com/microsoft/RD-Agent/issues/317)) ([dcdcbe4](https://github.com/microsoft/RD-Agent/commit/dcdcbe46b4858bfb133ae3cca056e7f602d5cf63))
+* add kaggle command ([#271](https://github.com/microsoft/RD-Agent/issues/271)) ([0938394](https://github.com/microsoft/RD-Agent/commit/0938394b7084ffbf3294d8c23d2d34bf7322ca0b))
+* add kaggle tpl: feedback-prize ([#331](https://github.com/microsoft/RD-Agent/issues/331)) ([a288e39](https://github.com/microsoft/RD-Agent/commit/a288e399e6b0beec62729bd7d46b98a55de5ab79))
+* add more templates for kaggle ([#291](https://github.com/microsoft/RD-Agent/issues/291)) ([da752ec](https://github.com/microsoft/RD-Agent/commit/da752ec806e6f5f5679bc27ac1c072ed9a319251))
+* add normal rag into framework ([#360](https://github.com/microsoft/RD-Agent/issues/360)) ([91b0b1f](https://github.com/microsoft/RD-Agent/commit/91b0b1f66c3c1bf757cb64c4cfbdcaafe59eab74))
+* add qlib_factor_strategy ([#307](https://github.com/microsoft/RD-Agent/issues/307)) ([f8f59ff](https://github.com/microsoft/RD-Agent/commit/f8f59ff0a1be4428a68c8c27f220aabad0b6c9f0))
+* Add ranking in kaggle scenario ([#401](https://github.com/microsoft/RD-Agent/issues/401)) ([b16b4be](https://github.com/microsoft/RD-Agent/commit/b16b4beb402e0c27dfb39ee9d2a120f1b56d447c))
+* Add runtime measurement for each step and loop in RDLoop. ([#281](https://github.com/microsoft/RD-Agent/issues/281)) ([83058c8](https://github.com/microsoft/RD-Agent/commit/83058c864ceeec413dd29bf501030d5a7bd34679))
+* add s3e11 kaggle template ([#324](https://github.com/microsoft/RD-Agent/issues/324)) ([8c57524](https://github.com/microsoft/RD-Agent/commit/8c57524bead1c8f655a08763d608eb7a6dd5975e))
+* Added RepoAnalyzer to empower auto-summary of a workspace ([#264](https://github.com/microsoft/RD-Agent/issues/264)) ([0bd349a](https://github.com/microsoft/RD-Agent/commit/0bd349af50b9b881ba1774bdeb4d723529ef2aa9))
+* Added support for loading and storing RAG in Kaggle scenarios. ([#269](https://github.com/microsoft/RD-Agent/issues/269)) ([c4895de](https://github.com/microsoft/RD-Agent/commit/c4895de83f1ed000e563d42b3468a6bd9e5a4965))
+* announce Discord and WeChat ([#367](https://github.com/microsoft/RD-Agent/issues/367)) ([acac507](https://github.com/microsoft/RD-Agent/commit/acac5078a103b71afa6bd6c053b0766a6a7e609d))
+* auto submit result after one kaggle RDLoop ([#345](https://github.com/microsoft/RD-Agent/issues/345)) ([ab55d70](https://github.com/microsoft/RD-Agent/commit/ab55d7052b53a928b84dc5d5d0d2999d90ca9056))
+* better feedback & evaluation ([#346](https://github.com/microsoft/RD-Agent/issues/346)) ([cc9a8c1](https://github.com/microsoft/RD-Agent/commit/cc9a8c1eab3ca89f8c1e5de4a2bb4e7fcc0cc615))
+* Dynamic scenario based on task ([#392](https://github.com/microsoft/RD-Agent/issues/392)) ([665a037](https://github.com/microsoft/RD-Agent/commit/665a037e4fd7326c450e3fa0d0605eea26fd9ef3))
+* Factor Implement Search Enhancement ([#294](https://github.com/microsoft/RD-Agent/issues/294)) ([4ecf25f](https://github.com/microsoft/RD-Agent/commit/4ecf25f0acf2389a172b14d3dab20895daf2ab89))
+* Feature selection v3 to support all actions  ([#280](https://github.com/microsoft/RD-Agent/issues/280)) ([0047641](https://github.com/microsoft/RD-Agent/commit/00476413fbf00e36e71ab3ccb48d4e766b6ccf4d))
+* fix some bugs and add original features' description ([#259](https://github.com/microsoft/RD-Agent/issues/259)) ([1a5f45a](https://github.com/microsoft/RD-Agent/commit/1a5f45a40d821c017bdba14af8c93710707c5ea5))
+* get kaggle notebooks & disscussion text for RAG ([#371](https://github.com/microsoft/RD-Agent/issues/371)) ([cead345](https://github.com/microsoft/RD-Agent/commit/cead3450a14bf4b142ac988c27fa098c7656a95c))
+* Iceberge competition ([#372](https://github.com/microsoft/RD-Agent/issues/372)) ([c10ea4f](https://github.com/microsoft/RD-Agent/commit/c10ea4f5d4cc56a75b47cf23c7084ee189ba1a25))
+* implement isolated model feature selection loop ([#370](https://github.com/microsoft/RD-Agent/issues/370)) ([cf1292d](https://github.com/microsoft/RD-Agent/commit/cf1292de1a0153ca14ea64971e73a1c93f7d89e3))
+* Initial version if Graph RAG in KAGGLE scenario ([#301](https://github.com/microsoft/RD-Agent/issues/301)) ([fd3c0fd](https://github.com/microsoft/RD-Agent/commit/fd3c0fd26eff7d3be72fa4f2a234e33b9f796627))
+* Integrate RAG into the Kaggle scenarios. ([#262](https://github.com/microsoft/RD-Agent/issues/262)) ([be0e48a](https://github.com/microsoft/RD-Agent/commit/be0e48a7dfbee2b5d2947d09115db5db2e5266f1))
+* Kaggle loop update (Feature & Model) ([#241](https://github.com/microsoft/RD-Agent/issues/241)) ([4cf22a6](https://github.com/microsoft/RD-Agent/commit/4cf22a65c964123b4267569ee02c0c7094c54ca4))
+* kaggle templates related ([#287](https://github.com/microsoft/RD-Agent/issues/287)) ([785fdc1](https://github.com/microsoft/RD-Agent/commit/785fdc144d16fa8454b7c9d2e53e78fe7f22a29a))
+* Model context for tuning and selection ([#284](https://github.com/microsoft/RD-Agent/issues/284)) ([f2831e7](https://github.com/microsoft/RD-Agent/commit/f2831e7442510668b0ca75953b3359894803ef3c))
+* Modify FactorRowCountEvaluator and FactorIndexEvaluator to return the ratio ([#328](https://github.com/microsoft/RD-Agent/issues/328)) ([8f43f8e](https://github.com/microsoft/RD-Agent/commit/8f43f8e87a92e05b541e925910608606ec8f6c4b))
+* New competition - Optiver ([#356](https://github.com/microsoft/RD-Agent/issues/356)) ([3705efe](https://github.com/microsoft/RD-Agent/commit/3705efe3b923748655a57d76b7a236e54d361831))
+* random forest for s3e11 ([#347](https://github.com/microsoft/RD-Agent/issues/347)) ([b57846d](https://github.com/microsoft/RD-Agent/commit/b57846d29314e9a5967945d1b4895f0f48c0f5ce))
+* refine the code in model description and fix some bugs in feedback.py ([#288](https://github.com/microsoft/RD-Agent/issues/288)) ([5b124d7](https://github.com/microsoft/RD-Agent/commit/5b124d7372137e4c613eb2749ddcc773922cc7b6))
+* refine the template in several Kaggle competitions ([#343](https://github.com/microsoft/RD-Agent/issues/343)) ([034f238](https://github.com/microsoft/RD-Agent/commit/034f238ed5ec351486b21250eabc75114961936c))
+* Revise to support better hypothesis proposal  ([#390](https://github.com/microsoft/RD-Agent/issues/390)) ([c55ec0a](https://github.com/microsoft/RD-Agent/commit/c55ec0a0f577bbf7fc6228f7b87d2089ded83b31))
+* show workspace in demo ([#348](https://github.com/microsoft/RD-Agent/issues/348)) ([ddf567c](https://github.com/microsoft/RD-Agent/commit/ddf567c551b553788be022e9312c209ef6137d64))
+* support Multi output ([#330](https://github.com/microsoft/RD-Agent/issues/330)) ([3d36c45](https://github.com/microsoft/RD-Agent/commit/3d36c452ff0983800e5343834cc69f24a508ea70))
+* Supporting COVID-19 competition ([#374](https://github.com/microsoft/RD-Agent/issues/374)) ([a1b63db](https://github.com/microsoft/RD-Agent/commit/a1b63db79600edc9a74ba713c9d0be290214a592))
+* supporting Mnist competition ([#375](https://github.com/microsoft/RD-Agent/issues/375)) ([e958a34](https://github.com/microsoft/RD-Agent/commit/e958a34f5632a46ac43bff8e0d07d6ed020fdfc2))
+* Supporting Model Specifications ([#319](https://github.com/microsoft/RD-Agent/issues/319)) ([e126471](https://github.com/microsoft/RD-Agent/commit/e1264719e10b76158a91cd0ef331848e7c2de7c7))
+* supporting various Kaggle competitions & scenarios for RD-Agent ([#409](https://github.com/microsoft/RD-Agent/issues/409)) ([75eea22](https://github.com/microsoft/RD-Agent/commit/75eea22cc3d4e6f5a94c88cce915e27c507f8c50))
+* template for kaggle ([#308](https://github.com/microsoft/RD-Agent/issues/308)) ([ff97cf0](https://github.com/microsoft/RD-Agent/commit/ff97cf0155ab6941e4b5cf7d103575f934b70dc9))
+* use auto gen seed when using LLM cache ([#441](https://github.com/microsoft/RD-Agent/issues/441)) ([ca15365](https://github.com/microsoft/RD-Agent/commit/ca15365d23eeb094f42cf3dc8f5269b2f1c42bd3))
+* use unified pickle cacher & move llm config into a isolated config ([#424](https://github.com/microsoft/RD-Agent/issues/424)) ([2879ecf](https://github.com/microsoft/RD-Agent/commit/2879ecff816d97688b60909a79c7e568d42608a1))
+* xgboost gpu accelerate ([#359](https://github.com/microsoft/RD-Agent/issues/359)) ([56a5b8f](https://github.com/microsoft/RD-Agent/commit/56a5b8f9b2c6726cc64ec5b04b4ce7935d59b572))
+
+
+### Bug Fixes
+
+* a bug of developer& edit s4e8 template ([#338](https://github.com/microsoft/RD-Agent/issues/338)) ([f12ce72](https://github.com/microsoft/RD-Agent/commit/f12ce726e7de96d478a232a3c27f92439820f8b4))
+* actively raised errors aer also considered as negative feedback. ([#268](https://github.com/microsoft/RD-Agent/issues/268)) ([46ec908](https://github.com/microsoft/RD-Agent/commit/46ec908e3594ac5e4cdc4057268e2f8800f5ed1f))
+* bug of saving preprocess cache files ([#310](https://github.com/microsoft/RD-Agent/issues/310)) ([5fb0608](https://github.com/microsoft/RD-Agent/commit/5fb0608f39f113cc9807fb1f381284a0bd4da318))
+* cache ([#383](https://github.com/microsoft/RD-Agent/issues/383)) ([f2a6e75](https://github.com/microsoft/RD-Agent/commit/f2a6e75b36ca96f7733b9c2a7154ac67bd2d7c6f))
+* change css tag of kaggle competition info crawler ([#306](https://github.com/microsoft/RD-Agent/issues/306)) ([1e3d38b](https://github.com/microsoft/RD-Agent/commit/1e3d38bf1ca3654f3a90ff392ecba1dbb4e80224))
+* debug dsagent ([#387](https://github.com/microsoft/RD-Agent/issues/387)) ([8fe9511](https://github.com/microsoft/RD-Agent/commit/8fe9511e606ba148c66f384add6ab94857079541))
+* eval_method cannot catch run factor error ([#260](https://github.com/microsoft/RD-Agent/issues/260)) ([2aaab31](https://github.com/microsoft/RD-Agent/commit/2aaab317ccb7a0121063bcd85fc36c21c7b8a391))
+* fix a bug in competition metric evaluation ([#407](https://github.com/microsoft/RD-Agent/issues/407)) ([94c47d6](https://github.com/microsoft/RD-Agent/commit/94c47d6fd5c3e38fc786a83e6d0d05e8d04498f3))
+* fix a bug in mini case ([#389](https://github.com/microsoft/RD-Agent/issues/389)) ([e75bb57](https://github.com/microsoft/RD-Agent/commit/e75bb5746f63933b750406bbd34ee63c5ba76b9f))
+* fix a bug in model tuning feedback ([#316](https://github.com/microsoft/RD-Agent/issues/316)) ([8aa088d](https://github.com/microsoft/RD-Agent/commit/8aa088da2dc7525a3970c01d01987246f47d6238))
+* fix a bug in scenario.py ([#388](https://github.com/microsoft/RD-Agent/issues/388)) ([999a1eb](https://github.com/microsoft/RD-Agent/commit/999a1eb0eff9088e1b02419db741db4acf8d9ff7))
+* fix a bug in the format of the model input ([#327](https://github.com/microsoft/RD-Agent/issues/327)) ([8f0574e](https://github.com/microsoft/RD-Agent/commit/8f0574eaaadb245b8c38e09ad4821306996d926f))
+* fix a small bug in cache using module name and function name as unique folder name ([#429](https://github.com/microsoft/RD-Agent/issues/429)) ([4f8134a](https://github.com/microsoft/RD-Agent/commit/4f8134a697d952f7ac824d7ebeec64bbc4545ab3))
+* fix a typo ([#362](https://github.com/microsoft/RD-Agent/issues/362)) ([9fafabd](https://github.com/microsoft/RD-Agent/commit/9fafabdf321b818bdd2211a2324d50cd0ebe1c1f))
+* fix cache result logic ([#430](https://github.com/microsoft/RD-Agent/issues/430)) ([5e34263](https://github.com/microsoft/RD-Agent/commit/5e342637dcc862679fd0642c6ba9ef048c984845))
+* fix command injection ([#421](https://github.com/microsoft/RD-Agent/issues/421)) ([52f30a6](https://github.com/microsoft/RD-Agent/commit/52f30a6184af1295be15e855a80b84bc424fc75d))
+* fix json load error ([#386](https://github.com/microsoft/RD-Agent/issues/386)) ([bba55fb](https://github.com/microsoft/RD-Agent/commit/bba55fb48fe105f4847c1b9c476eedc80835f523))
+* fix some bugs in feedback.py and refine the prompt ([#292](https://github.com/microsoft/RD-Agent/issues/292)) ([d834052](https://github.com/microsoft/RD-Agent/commit/d8340527f133dcc649d599d90d6402eddd37859e))
+* fix some bugs in knowledge base ([#378](https://github.com/microsoft/RD-Agent/issues/378)) ([fa6ff8e](https://github.com/microsoft/RD-Agent/commit/fa6ff8e591cf1847df77d73116649c5623161573))
+* fix some bugs in rag ([#399](https://github.com/microsoft/RD-Agent/issues/399)) ([194215c](https://github.com/microsoft/RD-Agent/commit/194215c4559aee5b6ece18d65c95fb30968e2db6))
+* fix some bugs in the entire loop ([#274](https://github.com/microsoft/RD-Agent/issues/274)) ([8a564ec](https://github.com/microsoft/RD-Agent/commit/8a564ece1d87b27ee98b76db317935e802468965))
+* fix some errors in scenario.py, proposal.py and runner.py and several complex competition scenarios([#365](https://github.com/microsoft/RD-Agent/issues/365)) ([2e383b1](https://github.com/microsoft/RD-Agent/commit/2e383b175d8448a67cb470f4e3ae8977d8ec6b5b))
+* improve_execution_time_in_kaggle_loop ([#279](https://github.com/microsoft/RD-Agent/issues/279)) ([4c8f998](https://github.com/microsoft/RD-Agent/commit/4c8f998c76f1e983a5687d2c65d3251750f2a9a0))
+* kaggle data mount problem ([#297](https://github.com/microsoft/RD-Agent/issues/297)) ([795df31](https://github.com/microsoft/RD-Agent/commit/795df311e3f93cd2f3fb51ba5698adaf10f6bd62))
+* Optiver fixes ([#357](https://github.com/microsoft/RD-Agent/issues/357)) ([b054017](https://github.com/microsoft/RD-Agent/commit/b054017463af0d1784407030f2477d212118f341))
+* partial bug in bench ([#368](https://github.com/microsoft/RD-Agent/issues/368)) ([af9808f](https://github.com/microsoft/RD-Agent/commit/af9808f98736a2df07e121c2f6d7bfeb7b7d3581))
+* preprocess output format & some mistake in spelling ([#358](https://github.com/microsoft/RD-Agent/issues/358)) ([b8b2cd6](https://github.com/microsoft/RD-Agent/commit/b8b2cd6ccd3b27aa73de847e50899a8a53b71b8f))
+* rag save file ([#385](https://github.com/microsoft/RD-Agent/issues/385)) ([1cb01dd](https://github.com/microsoft/RD-Agent/commit/1cb01dd6fe595f2f5fb86487601326611dd1a57a))
+* raise error in demo when no Metric in a Loop ([#313](https://github.com/microsoft/RD-Agent/issues/313)) ([e46a78e](https://github.com/microsoft/RD-Agent/commit/e46a78eb69271cb19978aab2f3b976c2870ca082))
+* refactor Bench ([#302](https://github.com/microsoft/RD-Agent/issues/302)) ([78a87f6](https://github.com/microsoft/RD-Agent/commit/78a87f624780ff67c0fa995ae4692678a120f99c))
+* refine some codes ([#353](https://github.com/microsoft/RD-Agent/issues/353)) ([866c2e6](https://github.com/microsoft/RD-Agent/commit/866c2e63ffa3876a3d16ad37f96da41d0558b714))
+* refine the prompt ([#286](https://github.com/microsoft/RD-Agent/issues/286)) ([77966c4](https://github.com/microsoft/RD-Agent/commit/77966c4f5e9f492c437c5b4b78d89c0f875ef0d8))
+* refine the ucb algorithm ([#406](https://github.com/microsoft/RD-Agent/issues/406)) ([14f7d97](https://github.com/microsoft/RD-Agent/commit/14f7d976e03c92d6e727524e0cdad8a03b585016))
+* revert model and make SOTA model available to COSTEER ([#351](https://github.com/microsoft/RD-Agent/issues/351)) ([3b7437b](https://github.com/microsoft/RD-Agent/commit/3b7437b87e685188259779cd85a78a0b592de9de))
+* stop using markup in docker env print ([#336](https://github.com/microsoft/RD-Agent/issues/336)) ([3009889](https://github.com/microsoft/RD-Agent/commit/3009889b5e2605b5427c76f3084e0e58026bb5ae))
+* support seed and fix absolute path ([#278](https://github.com/microsoft/RD-Agent/issues/278)) ([26352e1](https://github.com/microsoft/RD-Agent/commit/26352e13121cad5be95c0de78bb9f5dda4330614))
+* template for kaggle foreset & s4e9 ([#334](https://github.com/microsoft/RD-Agent/issues/334)) ([2393a41](https://github.com/microsoft/RD-Agent/commit/2393a41e7237615ced2c3fdd5c49308236b9f276))
+* test kaggle method ([#296](https://github.com/microsoft/RD-Agent/issues/296)) ([91a6196](https://github.com/microsoft/RD-Agent/commit/91a619618be1d7db660ea2b413a78dfaba9417a1))
+* update code to fix a small bug in model cache md5 hash ([#303](https://github.com/microsoft/RD-Agent/issues/303)) ([b00e4dc](https://github.com/microsoft/RD-Agent/commit/b00e4dc2eff5b16029a2a12a6589eadac5cfd148))
+* update new feature engineering code format ([#272](https://github.com/microsoft/RD-Agent/issues/272)) ([7850b80](https://github.com/microsoft/RD-Agent/commit/7850b8006a7c89d22629b345b4f361b0f35bc60d))
+* Update prompts.yaml to constrain only one model type   ([#341](https://github.com/microsoft/RD-Agent/issues/341)) ([5b5dfee](https://github.com/microsoft/RD-Agent/commit/5b5dfeefbc7eb9dcbd9923544005c5d281262c03))
+* Update runner.py to fix a small bug ([#282](https://github.com/microsoft/RD-Agent/issues/282)) ([8aef3ab](https://github.com/microsoft/RD-Agent/commit/8aef3abcecd6002bd4bfeedcbe2c786d8bbfe2be))
+* Use fixed file name in model costeer & fixing cache ([#311](https://github.com/microsoft/RD-Agent/issues/311)) ([1f910a5](https://github.com/microsoft/RD-Agent/commit/1f910a5248bc576895ed66c2f7b2c3e046a2bc28))
+
+
+### Performance Improvements
+
+* some small upgrade to factor costeer to improve the performance ([#420](https://github.com/microsoft/RD-Agent/issues/420)) ([9eb931f](https://github.com/microsoft/RD-Agent/commit/9eb931ffd971f252380dbd33ad1db259a4f229fd))
+
+
+### Reverts
+
+* Revert feat: Factor Implement Search Enhancement ([#294](https://github.com/microsoft/RD-Agent/issues/294)) ([#305](https://github.com/microsoft/RD-Agent/issues/305)) ([f663cf4](https://github.com/microsoft/RD-Agent/commit/f663cf42a2f75cd52aef1c6b18be7c27f0641fed))
+
+## [0.2.1](https://github.com/microsoft/RD-Agent/compare/v0.2.0...v0.2.1) (2024-09-10)
+
+
+### Bug Fixes
+
+* default model value in config ([#256](https://github.com/microsoft/RD-Agent/issues/256)) ([c097585](https://github.com/microsoft/RD-Agent/commit/c097585f631f401c2c0966f6ad4c17286924f011))
+* fix_dotenv_error ([#257](https://github.com/microsoft/RD-Agent/issues/257)) ([923063c](https://github.com/microsoft/RD-Agent/commit/923063c1fd957c4ed42e97272c72b5e9545451dc))
+* readme ([#248](https://github.com/microsoft/RD-Agent/issues/248)) ([8cede22](https://github.com/microsoft/RD-Agent/commit/8cede2209922876490148459e1134da828e1fda0))
+
+## [0.2.0](https://github.com/microsoft/RD-Agent/compare/v0.1.0...v0.2.0) (2024-09-07)
+
+
+### Features
+
+* add collect info ([#233](https://github.com/microsoft/RD-Agent/issues/233)) ([89f4af9](https://github.com/microsoft/RD-Agent/commit/89f4af90fb4d95a0689bf9efc8ffd9326469c0aa))
+* add cross validation for kaggle scenario ([#236](https://github.com/microsoft/RD-Agent/issues/236)) ([e0b03ba](https://github.com/microsoft/RD-Agent/commit/e0b03ba6b5c3d9aa552b99d470e106d4e348e64d))
+* add progress status for docker env ([#215](https://github.com/microsoft/RD-Agent/issues/215)) ([538d4ef](https://github.com/microsoft/RD-Agent/commit/538d4ef2e52de795b90d3f75b2e1e877ab85c18d))
+* Added loop code for Kaggle scene. ([#211](https://github.com/microsoft/RD-Agent/issues/211)) ([975c327](https://github.com/microsoft/RD-Agent/commit/975c32715e51aec6b49537401f5fc59115e04a01))
+* Demo display effect and usage ([#162](https://github.com/microsoft/RD-Agent/issues/162)) ([8cf122a](https://github.com/microsoft/RD-Agent/commit/8cf122a0155f434fa4477ae7a6d616b5caecd3e0))
+* piloting of the framework ([#227](https://github.com/microsoft/RD-Agent/issues/227)) ([e9b103e](https://github.com/microsoft/RD-Agent/commit/e9b103e684fdd2b98cd1a89971a3fce2d6e884a1))
+* support more models for kaggle scenario ([#223](https://github.com/microsoft/RD-Agent/issues/223)) ([e3a9659](https://github.com/microsoft/RD-Agent/commit/e3a96598c0720fe092ec86d7ca8c195c7d6bcc72))
+* update model_experiment.py to support basic EDA ([#220](https://github.com/microsoft/RD-Agent/issues/220)) ([bf2684c](https://github.com/microsoft/RD-Agent/commit/bf2684c4d55ab8e1048ac0291695475ad53b0cd6))
+
+
+### Bug Fixes
+
+* fix some bugs in llm calling ([#217](https://github.com/microsoft/RD-Agent/issues/217)) ([7b010f8](https://github.com/microsoft/RD-Agent/commit/7b010f8b5940aba65a58f1d78192aa80bcd0e654))
+* package dependency. ([#234](https://github.com/microsoft/RD-Agent/issues/234)) ([46be295](https://github.com/microsoft/RD-Agent/commit/46be2952952af534fd8d98a656c704c688d7cbdd))
+* remove useless line ([#177](https://github.com/microsoft/RD-Agent/issues/177)) ([64e9a8e](https://github.com/microsoft/RD-Agent/commit/64e9a8e39a2072a962111db18f5b9565df5b0176))
+
+## [0.1.0](https://github.com/microsoft/RD-Agent/compare/v0.0.1...v0.1.0) (2024-08-09)
+
+
+### Features
+
+* add entry for rdagent. ([#187](https://github.com/microsoft/RD-Agent/issues/187)) ([121b6d9](https://github.com/microsoft/RD-Agent/commit/121b6d98de38cd03be30cbee47b40baf39a2b60b))
+* change ui entry ([#197](https://github.com/microsoft/RD-Agent/issues/197)) ([fa5d335](https://github.com/microsoft/RD-Agent/commit/fa5d3354d22240888f4fc4007d9834f7424632aa))
+* remove pdfs and enable online pdf readings ([#183](https://github.com/microsoft/RD-Agent/issues/183)) ([18c0501](https://github.com/microsoft/RD-Agent/commit/18c05016a23d694c7b12759cf1322562dcffc56a))
+
+
+### Bug Fixes
+
+* Fix a fail href in readme ([#189](https://github.com/microsoft/RD-Agent/issues/189)) ([1b89218](https://github.com/microsoft/RD-Agent/commit/1b89218f6bc697494f4a1b8a42ad18963002714f))
+* fix quick start problem ([#191](https://github.com/microsoft/RD-Agent/issues/191)) ([44f61bf](https://github.com/microsoft/RD-Agent/commit/44f61bfa1058a8efb59ca48b7f1417765aeea33e))
+* update command line in readme.md ([#192](https://github.com/microsoft/RD-Agent/issues/192)) ([9c45d24](https://github.com/microsoft/RD-Agent/commit/9c45d24a192da02f7d9765cb001097da1bc36c61))
+
 ## 0.0.1 (2024-08-08)
 
 

Makefile

@@ -97,7 +97,7 @@ mypy:
 # First deal with the core folder, and then gradually increase the scope of detection,
 # and eventually realize the detection of the complete project.
 ruff:
-	$(PIPRUN) ruff check rdagent/core --ignore FBT001,FBT002   # --exclude rdagent/scripts,git_ignore_folder
+	$(PIPRUN) ruff check rdagent/core --ignore FBT001,FBT002,I001   # --exclude rdagent/scripts,git_ignore_folder
 
 # Check lint with toml-sort.
 toml-sort:
@@ -141,10 +141,21 @@ test-run:
 	$(PIPRUN) python -m coverage run --concurrency=multiprocessing -m pytest --ignore test/scripts
 	$(PIPRUN) python -m coverage combine
 
+test-run-offline:
+	# some test that does not require api calling
+	$(PIPRUN) python -m coverage erase
+	$(PIPRUN) python -m coverage run --concurrency=multiprocessing -m pytest -m "offline" --ignore test/scripts
+	$(PIPRUN) python -m coverage combine
+
 # Generate coverage report for terminal and xml.
+# TODO: we may have higher coverage rate if we have more test
 test: test-run
-	$(PIPRUN) python -m coverage report --fail-under 80
-	$(PIPRUN) python -m coverage xml --fail-under 80
+	$(PIPRUN) python -m coverage report --fail-under 20  # 80
+	$(PIPRUN) python -m coverage xml --fail-under 20  # 80
+
+test-offline: test-run-offline
+	$(PIPRUN) python -m coverage report --fail-under 20  # 80
+	$(PIPRUN) python -m coverage xml --fail-under 20  # 80
 
 ########################################################################################
 # Package
@@ -189,7 +200,7 @@ release-notes:
 
 # Build documentation only from rdagent.
 docs-gen:
-	$(PIPRUN) python -m sphinx.cmd.build docs $(PUBLIC_DIR)
+	$(PIPRUN) python -m sphinx.cmd.build -W docs $(PUBLIC_DIR)
 
 # Generate mypy reports.
 docs-mypy: docs-gen

README.md

@@ -1,48 +1,71 @@
+<h4 align="center">
+  <img src="docs/_static/logo.png" alt="RA-Agent logo" style="width:70%; ">
+  
+  <a href="https://rdagent.azurewebsites.net" target="_blank">🖥️ Live Demo</a> | <a href="https://rdagent.azurewebsites.net/factor_loop" target="_blank">🎥 Demo Video</a> <a href="https://www.youtube.com/watch?v=JJ4JYO3HscM&list=PLALmKB0_N3_i52fhUmPQiL4jsO354uopR" target="_blank">▶️YouTube</a>   | <a href="https://rdagent.readthedocs.io/en/latest/index.html" target="_blank">📖 Documentation</a> | <a href="#-paperwork-list"> 📃 Papers </a>
+</h3>
+
+
 [![CI](https://github.com/microsoft/RD-Agent/actions/workflows/ci.yml/badge.svg)](https://github.com/microsoft/RD-Agent/actions/workflows/ci.yml)
 [![CodeQL](https://github.com/microsoft/RD-Agent/actions/workflows/github-code-scanning/codeql/badge.svg)](https://github.com/microsoft/RD-Agent/actions/workflows/github-code-scanning/codeql)
 [![Dependabot Updates](https://github.com/microsoft/RD-Agent/actions/workflows/dependabot/dependabot-updates/badge.svg)](https://github.com/microsoft/RD-Agent/actions/workflows/dependabot/dependabot-updates)
 [![Lint PR Title](https://github.com/microsoft/RD-Agent/actions/workflows/pr.yml/badge.svg)](https://github.com/microsoft/RD-Agent/actions/workflows/pr.yml)
-[![Readthedocs Preview](https://github.com/microsoft/RD-Agent/actions/workflows/readthedocs-preview.yml/badge.svg)](https://github.com/microsoft/RD-Agent/actions/workflows/readthedocs-preview.yml)
 [![Release.yml](https://github.com/microsoft/RD-Agent/actions/workflows/release.yml/badge.svg)](https://github.com/microsoft/RD-Agent/actions/workflows/release.yml)
+[![Platform](https://img.shields.io/badge/platform-Linux-blue)](https://pypi.org/project/rdagent/#files)
+[![PyPI](https://img.shields.io/pypi/v/rdagent)](https://pypi.org/project/rdagent/)
+[![PyPI - Python Version](https://img.shields.io/pypi/pyversions/rdagent)](https://pypi.org/project/rdagent/)
 [![Release](https://img.shields.io/github/v/release/microsoft/RD-Agent)](https://github.com/microsoft/RD-Agent/releases)
+[![GitHub](https://img.shields.io/github/license/microsoft/RD-Agent)](https://github.com/microsoft/RD-Agent/blob/main/LICENSE)
 [![pre-commit](https://img.shields.io/badge/pre--commit-enabled-brightgreen?logo=pre-commit)](https://github.com/pre-commit/pre-commit)
 [![Checked with mypy](https://www.mypy-lang.org/static/mypy_badge.svg)](http://mypy-lang.org/)
 [![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)
-<!-- TODO: License / pypi / PyPI - Python Version  --> 
+[![Chat](https://img.shields.io/badge/chat-discord-blue)](https://discord.gg/ybQ97B6Jjy)
+[![Readthedocs Preview](https://github.com/microsoft/RD-Agent/actions/workflows/readthedocs-preview.yml/badge.svg)](https://github.com/microsoft/RD-Agent/actions/workflows/readthedocs-preview.yml) <!-- this badge is too long, please place it in the last one to make it pretty --> 
 
 # 📰 News
-| 🗞️News        | 📝Description                 |
+| 🗞️ News        | 📝 Description                 |
 | --            | ------                        |
-| First release | RDAgent are release on Github |
+| Official WeChat group release  | We created a WeChat group, welcome to join! (🗪[QR Code](docs/WeChat_QR_code.jpg)) |
+| Official Discord release  | We launch our first chatting channel in Discord (🗪[![Chat](https://img.shields.io/badge/chat-discord-blue)](https://discord.gg/ybQ97B6Jjy)) |
+| First release | **RDAgent** is released on GitHub |
 
 
 # 🌟 Introduction
+<div align="center">
+      <img src="docs/_static/scen.png" alt="Our focused scenario" style="width:80%; ">
+</div>
 
-![Our focused scenario](docs/_static/scen.jpg)
-
-RDAgent aims to automate the most critical and valuable aspects of the industrial R&D process, and we begins with focusing on the data-driven scenarios to streamline the development of models and data. 
+RDAgent aims to automate the most critical and valuable aspects of the industrial R&D process, and we begin with focusing on the data-driven scenarios to streamline the development of models and data. 
 Methodologically, we have identified a framework with two key components: 'R' for proposing new ideas and 'D' for implementing them.
 We believe that the automatic evolution of R&D will lead to solutions of significant industrial value.
 
 
 <!-- Tag Cloud -->
 R&D is a very general scenario. The advent of RDAgent can be your
-- [🎥Automatic Quant Factory]()
-- 🤖Data mining agent: iteratively proposing [🎥data]() & [models]() and implementing them by gaining knowledge from data.
-- 🦾Research copilot: Auto read [🎥research papers]()/[🎥reports]() and implement model structures or building datasets.
+- 💰 **Automatic Quant Factory** ([🎥Demo Video](https://rdagent.azurewebsites.net/factor_loop)|[▶️YouTube](https://www.youtube.com/watch?v=X4DK2QZKaKY&t=6s))
+- 🤖 **Data Mining Agent:** Iteratively proposing data & models ([🎥Demo Video 1](https://rdagent.azurewebsites.net/model_loop)|[▶️YouTube](https://www.youtube.com/watch?v=dm0dWL49Bc0&t=104s)) ([🎥Demo Video 2](https://rdagent.azurewebsites.net/dmm)|[▶️YouTube](https://www.youtube.com/watch?v=VIaSTZuoZg4))  and implementing them by gaining knowledge from data.
+- 🦾 **Research Copilot:** Auto read research papers ([🎥Demo Video](https://rdagent.azurewebsites.net/report_model)|[▶️YouTube](https://www.youtube.com/watch?v=BiA2SfdKQ7o)) / financial reports ([🎥Demo Video](https://rdagent.azurewebsites.net/report_factor)|[▶️YouTube](https://www.youtube.com/watch?v=ECLTXVcSx-c)) and implement model structures or building datasets.
 - ...
 
-You can click the [🎥link]() above to view the demo. More methods and scenarios are being added to the project to empower your R&D processes and boost productivity.
+You can click the links above to view the demo. We're continuously adding more methods and scenarios to the project to enhance your R&D processes and boost productivity. 
 
-We have a quick 🎥demo for one use case of RDAgent.
-- TODO: Demo
+Additionally, you can take a closer look at the examples in our **[🖥️ Live Demo](https://rdagent.azurewebsites.net/)**.
+
+<div align="center">
+    <a href="https://rdagent.azurewebsites.net/" target="_blank">
+        <img src="docs/_static/demo.png" alt="Watch the demo" width="80%">
+    </a>
+</div>
 
 
-# ⚡Quick start
-You can try our demo by running the following command:
+# ⚡ Quick start
+
+You can try above demos by running the following command:
+
+### 🐳 Docker installation.
+Users must ensure Docker is installed before attempting most scenarios. Please refer to the [official 🐳Docker page](https://docs.docker.com/engine/install/) for installation instructions.
 
 ### 🐍 Create a Conda Environment
-- Create a new conda environment with Python (3.10 and 3.11 are well tested in our CI):
+- Create a new conda environment with Python (3.10 and 3.11 are well-tested in our CI):
   ```sh
   conda create -n rdagent python=3.10
   ```
@@ -51,89 +74,119 @@ You can try our demo by running the following command:
   conda activate rdagent
   ```
 
-### 🛠️ Run Make Files
-TODO: `pip install rdagent` in the future.
-
-- **Navigate to the directory containing the MakeFile** and set up the development environment:
+### 🛠️ Install the RDAgent
+- You can directly install the RDAgent package from PyPI:
   ```sh
-  make dev
+  pip install rdagent
   ```
 
-### 📦 Install Pytorch
-TODO: use docker in quick start intead.
-
-- Install Pytorch and related libraries:
-  ```sh
-  pip3 install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu
-  pip3 install torch_geometric
+### ⚙️ Configuration
+- You have to config your GPT model in the `.env`
+  ```bash
+  cat << EOF  > .env
+  OPENAI_API_KEY=<your_api_key>
+  # EMBEDDING_MODEL=text-embedding-3-small
+  CHAT_MODEL=gpt-4-turbo
+  EOF
   ```
 
-### ⚙️ Environment Configuration
-- Place the `.env` file in the same directory as the `.env.example` file.
-  - The `.env.example` file contains the environment variables required for users using the OpenAI API (Please note that `.env.example` is an example file. `.env` is the one that will be finally used.)
-  - please refer to [Configuration](docs/build/html/installation.html#azure-openai) for the detailed explanation of the `.env`
-- Export each variable in the `.env` file:
-  ```sh
-  export $(grep -v '^#' .env | xargs)
-  ```
 ### 🚀 Run the Application
-TODO: run the front-page demo.
 
-The [🎥demo]() is implemented by the above commands.
+The **[🖥️ Live Demo](https://rdagent.azurewebsites.net/)** is implemented by the following commands(each item represents one demo, you can select the one you prefer):
 
-- Run the factor extraction and implementation application based on financial reports:
+- Run the **Automated Quantitative Trading & Iterative Factors Evolution**:  [Qlib](http://github.com/microsoft/qlib) self-loop factor proposal and implementation application
   ```sh
-  python rdagent/app/qlib_rd_loop/factor_from_report_sh.py
+  rdagent fin_factor
   ```
 
-- Run the self-loop factor extraction and implementation application:
+- Run the **Automated Quantitative Trading & Iterative Model Evolution**: [Qlib](http://github.com/microsoft/qlib) self-loop model proposal and implementation application
   ```sh
-  python rdagent/app/qlib_rd_loop/factor.py
+  rdagent fin_model
   ```
 
-- Run the self-loop model extraction and implementation application:
+- Run the **Automated Medical Prediction Model Evolution**: Medical self-loop model proposal and implementation application
+  >(1) Apply for an account at [PhysioNet](https://physionet.org/). <br /> (2) Request access to FIDDLE preprocessed data: [FIDDLE Dataset](https://physionet.org/content/mimic-eicu-fiddle-feature/1.0.0/). <br />
+  (3) Place your username and password in `.env`.
+  ```bash
+  cat << EOF  >> .env
+  DM_USERNAME=<your_username>
+  DM_PASSWORD=<your_password>
+  EOF
+  ```
   ```sh
-  python rdagent/app/qlib_rd_loop/model.py
+  rdagent med_model
   ```
 
+- Run the **Automated Quantitative Trading & Factors Extraction from Financial Reports**:  Run the [Qlib](http://github.com/microsoft/qlib) factor extraction and implementation application based on financial reports
+  ```sh
+  # 1. Generally, you can run this scenario using the following command:
+  rdagent fin_factor_report --report_folder=<Your financial reports folder path>
 
-# Scenarios
+  # 2. Specifically, you need to prepare some financial reports first. You can follow this concrete example:
+  wget https://github.com/SunsetWolf/rdagent_resource/releases/download/reports/all_reports.zip
+  unzip all_reports.zip -d git_ignore_folder/reports
+  rdagent fin_factor_report --report_folder=git_ignore_folder/reports
+  ```
 
-We have applied RD-Agent to multiple valuable data-driven industrial scenarios..
+- Run the **Automated Model Research & Development Copilot**: model extraction and implementation application
+  ```sh
+  # 1. Generally, you can run your own papers/reports with the following command:
+  rdagent general_model <Your paper URL>
+
+  # 2. Specifically, you can do it like this. For more details and additional paper examples, use `rdagent general_model -h`:
+  rdagent general_model  "https://arxiv.org/pdf/2210.09789"
+  ```
+
+### 🖥️ Monitor the Application Results
+- You can serve our demo app to monitor the RD loop by running the following command:
+  ```sh
+  rdagent ui --port 80 --log_dir <your log folder like "log/">
+  ```
+
+# 🏭 Scenarios
+
+We have applied RD-Agent to multiple valuable data-driven industrial scenarios.
 
 
 ## 🎯 Goal: Agent for Data-driven R&D
 
-In this project, we are aiming to build a Agent to automate Data-Driven R\&D that can
-+ 📄Read real-world material (reports, papers, etc.) and **extract** key formulas, descriptions of interested **features** and **models**, which are the key components of data-driven R&D .
-+ 🛠️**Implement** the extracted formulas (e.g., features, factors, and models) in runnable codes.
-   + Due to the limited ability of LLM in implementing at once, evolve the agent to be able to extend abilities by learning from feedback and knowledge and improve the agent's ability to implement more complex models.
-+ 💡Propose **new ideas** based on current knowledge and observations.
+In this project, we are aiming to build an Agent to automate Data-Driven R\&D that can
++ 📄 Read real-world material (reports, papers, etc.) and **extract** key formulas, descriptions of interested **features** and **models**, which are the key components of data-driven R&D .
++ 🛠️ **Implement** the extracted formulas (e.g., features, factors, and models) in runnable codes.
+   + Due to the limited ability of LLM in implementing at once, build an evolving process for the agent to improve performance by learning from feedback and knowledge.
++ 💡 Propose **new ideas** based on current knowledge and observations.
 
 <!-- ![Data-Centric R&D Overview](docs/_static/overview.png) -->
 
 ## 📈 Scenarios/Demos
 
-In the two key areas of data-driven scenarios, model implementation and data building, our system aims to serve two main roles: 🦾copilot and 🤖agent. 
-- The 🦾copilot follows human instructions to automate repetitive tasks. 
-- The 🤖agent, being more autonomous, actively proposes ideas for better results in the future.
+In the two key areas of data-driven scenarios, model implementation and data building, our system aims to serve two main roles: 🦾Copilot and 🤖Agent. 
+- The 🦾Copilot follows human instructions to automate repetitive tasks. 
+- The 🤖Agent, being more autonomous, actively proposes ideas for better results in the future.
 
 The supported scenarios are listed below:
 
 | Scenario/Target | Model Implementation                   | Data Building                                                                      |
 | --              | --                                     | --                                                                                 |
-| 💹 Finance      | 🤖Iteratively Proposing Ideas & Evolving | - 🦾Auto reports reading & implementation <br/> - 🤖Iteratively Proposing Ideas & Evolving |
-| 🩺 Medical      | 🤖Iteratively Proposing Ideas & Evolving | -                                                                                  |
-| 🏭 General      | 🦾Auto paper reading & implementation    | -                                                                                  |
+| **💹 Finance**      | 🤖 [Iteratively Proposing Ideas & Evolving](https://rdagent.azurewebsites.net/model_loop)[▶️YouTube](https://www.youtube.com/watch?v=dm0dWL49Bc0&t=104s) |  🤖 [Iteratively Proposing Ideas & Evolving](https://rdagent.azurewebsites.net/factor_loop) [▶️YouTube](https://www.youtube.com/watch?v=X4DK2QZKaKY&t=6s) <br/>   🦾 [Auto reports reading & implementation](https://rdagent.azurewebsites.net/report_factor)[▶️YouTube](https://www.youtube.com/watch?v=ECLTXVcSx-c)  |
+| **🩺 Medical**      | 🤖 [Iteratively Proposing Ideas & Evolving](https://rdagent.azurewebsites.net/dmm)[▶️YouTube](https://www.youtube.com/watch?v=VIaSTZuoZg4) | -                                                                                  |
+| **🏭 General**      | 🦾 [Auto paper reading & implementation](https://rdagent.azurewebsites.net/report_model)[▶️YouTube](https://www.youtube.com/watch?v=BiA2SfdKQ7o)    | -                                                                                  |
 
 Different scenarios vary in entrance and configuration. Please check the detailed setup tutorial in the scenarios documents.
 
-TODO: Scenario Gallary
-- map(scenario) => knowledge list;
+Here is a gallery of [successful explorations](https://github.com/SunsetWolf/rdagent_resource/releases/download/demo_traces/demo_traces.zip) (5 traces showed in **[🖥️ Live Demo](https://rdagent.azurewebsites.net/)**). You can download and view the execution trace using the command below:
 
-# ⚙️Framework
+```bash
+rdagent ui --port 80 --log_dir ./demo_traces
+```
 
-![image](https://github.com/user-attachments/assets/98fce923-77ab-4982-93c8-a7a01aece766)
+Please refer to **[📖readthedocs_scen](https://rdagent.readthedocs.io/en/latest/scens/catalog.html)** for more details of the scenarios.
+
+# ⚙️ Framework
+
+<div align="center">
+    <img src="docs/_static/Framework-RDAgent.png" alt="Framework-RDAgent" width="85%">
+</div>
 
 
 Automating the R&D process in data science is a highly valuable yet underexplored area in industry. We propose a framework to push the boundaries of this important research field.
@@ -141,17 +194,18 @@ Automating the R&D process in data science is a highly valuable yet underexplore
 The research questions within this framework can be divided into three main categories:
 | Research Area | Paper/Work List |
 |--------------------|-----------------|
-| Benchmark the R&D abilities | [Benchmark](#benchmark) |
-| Idea proposal: Explore new ideas or refine existing ones | [Research](#research) |
-| Ability to realize ideas: Implement and execute ideas | [Development](#development) |
+| **Benchmark the R&D abilities** | [Benchmark](#benchmark) |
+| **Idea proposal:** Explore new ideas or refine existing ones | [Research](#research) |
+| **Ability to realize ideas:** Implement and execute ideas | [Development](#development) |
 
 We believe that the key to delivering high-quality solutions lies in the ability to evolve R&D capabilities. Agents should learn like human experts, continuously improving their R&D skills.
 
+More documents can be found in the **[📖 readthedocs](https://rdagent.readthedocs.io/)**.
 
-# 📃Paper/Work list
+# 📃 Paper/Work list
 
-## Benchmark
-- [Towards Data-Centric Automatic R&D](https://arxiv.org/abs/2404.11276);
+## 📊 Benchmark
+- [Towards Data-Centric Automatic R&D](https://arxiv.org/abs/2404.11276)
 ```BibTeX
 @misc{chen2024datacentric,
     title={Towards Data-Centric Automatic R&D},
@@ -164,15 +218,15 @@ We believe that the key to delivering high-quality solutions lies in the ability
 ```
 ![image](https://github.com/user-attachments/assets/494f55d3-de9e-4e73-ba3d-a787e8f9e841)
 
-## Research
+## 🔍 Research
 
 In a data mining expert's daily research and development process, they propose a hypothesis (e.g., a model structure like RNN can capture patterns in time-series data), design experiments (e.g., finance data contains time-series and we can verify the hypothesis in this scenario), implement the experiment as code (e.g., Pytorch model structure), and then execute the code to get feedback (e.g., metrics, loss curve, etc.). The experts learn from the feedback and improve in the next iteration.
 
 Based on the principles above, we have established a basic method framework that continuously proposes hypotheses, verifies them, and gets feedback from the real-world practice. This is the first scientific research automation framework that supports linking with real-world verification.
 
-[Demos](#📈 Scenarios/Demos) are released.
+For more detail, please refer to our **[🖥️ Live Demo page](https://rdagent.azurewebsites.net)**.
 
-## Development
+## 🛠️ Development
 
 - [Collaborative Evolving Strategy for Automatic Data-Centric Development](https://arxiv.org/abs/2407.18690)
 ```BibTeX
@@ -188,18 +242,21 @@ Based on the principles above, we have established a basic method framework that
 ![image](https://github.com/user-attachments/assets/75d9769b-0edd-4caf-9d45-57d1e577054b)
 
 
-# Contributing
+# 🤝 Contributing
 
-More documents can be found in the [📚readthedocs](). TODO: add link
-
-## Guidance
+## 📝 Guidelines
 This project welcomes contributions and suggestions.
-You can find issues in the issues list or simply running `grep -r "TODO:"`.
+Contributing to this project is straightforward and rewarding. Whether it's solving an issue, addressing a bug, enhancing documentation, or even correcting a typo, every contribution is valuable and helps improve RDAgent.
+
+To get started, you can explore the issues list, or search for `TODO:` comments in the codebase by running the command `grep -r "TODO:"`.
 
-Making contributions is not a hard thing. Solving an issue(maybe just answering a question raised in issues list ), fixing/issuing a bug, improving the documents and even fixing a typo are important contributions to RDAgent.
 <img src="https://img.shields.io/github/contributors-anon/microsoft/RD-Agent"/>
 
-<a href="https://github.com/microsoft/RD-Agent/graphs/contributors"><img src="https://contrib.rocks/image?repo=microsoft/RD-Agent&max=240&columns=18" /></a>
+<a href="https://github.com/microsoft/RD-Agent/graphs/contributors">
+  <img src="https://contrib.rocks/image?repo=microsoft/RD-Agent&max=100&columns=15" />
+</a>
 
-# Disclaimer
-**The RD-agent is provided “as is”, without warranty of any kind, express or implied, including but not limited to the warranties of merchantability, fitness for a particular purpose and noninfringement. The RD-agent is aimed to facilitate research and development process in the financial industry and not ready-to-use for any financial investment or advice. Users shall independently assess and test the risks of the RD-agent in a specific use scenario, ensure the responsible use of AI technology, including but not limited to developing and integrating risk mitigation measures, and comply with all applicable laws and regulations in all applicable jurisdictions. The RD-agent does not provide financial opinions or reflect the opinions of Microsoft, nor is it designed to replace the role of qualified financial professionals in formulating, assessing, and approving finance products. The inputs and outputs of the RD-agent belong to the users and users shall assume all liability under any theory of liability, whether in contract, torts, regulatory, negligence, products liability, or otherwise, associated with use of the RD-agent and any inputs and outputs thereof.**
+Before we released RD-Agent as an open-source project on GitHub, it was an internal project within our group. Unfortunately, the internal commit history was not preserved when we removed some confidential code. As a result, some contributions from our group members, including Haotian Chen, Wenjun Feng, Haoxue Wang, Zeqi Ye, Xinjie Shen, and Jinhui Li, were not included in the public commits.
+
+# ⚖️ Legal disclaimer
+<p style="line-height: 1; font-style: italic;">The RD-agent is provided “as is”, without warranty of any kind, express or implied, including but not limited to the warranties of merchantability, fitness for a particular purpose and noninfringement. The RD-agent is aimed to facilitate research and development process in the financial industry and not ready-to-use for any financial investment or advice. Users shall independently assess and test the risks of the RD-agent in a specific use scenario, ensure the responsible use of AI technology, including but not limited to developing and integrating risk mitigation measures, and comply with all applicable laws and regulations in all applicable jurisdictions. The RD-agent does not provide financial opinions or reflect the opinions of Microsoft, nor is it designed to replace the role of qualified financial professionals in formulating, assessing, and approving finance products. The inputs and outputs of the RD-agent belong to the users and users shall assume all liability under any theory of liability, whether in contract, torts, regulatory, negligence, products liability, or otherwise, associated with use of the RD-agent and any inputs and outputs thereof.</p>

constraints/3.10.txt

@@ -71,7 +71,6 @@ isort==5.13.2
 jaraco.classes==3.3.0
 jedi==0.19.1
 jeepney==0.8.0
-Jinja2==3.1.2
 joblib==1.4.2
 json5==0.9.25
 jsonpatch==1.33
@@ -102,7 +101,6 @@ loguru==0.7.2
 loguru-mypy==0.0.4
 lxml==5.0.0
 markdown-it-py==3.0.0
-MarkupSafe==2.1.3
 marshmallow==3.20.1
 matplotlib==3.9.1
 matplotlib-inline==0.1.7
@@ -110,7 +108,6 @@ mdit-py-plugins==0.4.0
 mdurl==0.1.2
 mistune==3.0.2
 more-itertools==10.1.0
-mpmath==1.3.0
 msal==1.30.0
 msal-extensions==1.2.0
 msgpack==1.0.8
@@ -124,24 +121,11 @@ nbconvert==7.16.4
 nbformat==5.10.4
 ndindex==1.8
 nest-asyncio==1.6.0
-networkx==3.2.1
 nh3==0.2.15
 notebook==7.2.1
 notebook_shim==0.2.4
 numexpr==2.10.1
 numpy==1.26.2
-nvidia-cublas-cu12==12.1.3.1
-nvidia-cuda-cupti-cu12==12.1.105
-nvidia-cuda-nvrtc-cu12==12.1.105
-nvidia-cuda-runtime-cu12==12.1.105
-nvidia-cudnn-cu12==8.9.2.26
-nvidia-cufft-cu12==11.0.2.54
-nvidia-curand-cu12==10.3.2.106
-nvidia-cusolver-cu12==11.4.5.107
-nvidia-cusparse-cu12==12.1.0.106
-nvidia-nccl-cu12==2.18.1
-nvidia-nvjitlink-cu12==12.3.101
-nvidia-nvtx-cu12==12.1.105
 oauthlib==3.2.2
 openai==1.6.1
 overrides==7.7.0
@@ -153,7 +137,6 @@ parso==0.8.4
 pathspec==0.12.1
 patsy==0.5.6
 pexpect==4.9.0
-pillow==10.4.0
 pkginfo==1.9.6
 platformdirs==4.1.0
 pluggy==1.3.0
@@ -201,7 +184,6 @@ ruamel.yaml==0.18.5
 ruamel.yaml.clib==0.2.8
 ruff==0.4.5
 scikit-learn==1.5.1
-scipy==1.11.4
 SecretStorage==3.3.3
 semver==3.0.2
 Send2Trash==1.8.3
@@ -226,7 +208,6 @@ sphinxcontrib-serializinghtml==1.1.9
 SQLAlchemy==2.0.24
 stack-data==0.6.3
 statsmodels==0.14.2
-sympy==1.12
 tables==3.9.2
 tabulate==0.9.0
 tenacity==8.2.3
@@ -238,14 +219,11 @@ tinycss2==1.3.0
 toml-sort==0.23.1
 tomli==2.0.1
 tomlkit==0.12.3
-torch==2.1.2
-torch_geometric==2.5.3
 tornado==6.4
 tqdm==4.66.1
 traitlets==5.14.3
 tree-sitter==0.22.3
 tree-sitter-python==0.21.0
-triton==2.1.0
 twine==4.0.2
 typer==0.9.0
 types-psutil==6.0.0.20240621
@@ -253,7 +231,6 @@ types-python-dateutil==2.9.0.20240316
 types-PyYAML==6.0.12.20240724
 types-tqdm==4.66.0.20240417
 typing-inspect==0.9.0
-typing_extensions==4.9.0
 tzdata==2023.4
 uri-template==1.3.0
 urllib3==2.1.0

constraints/3.11.txt

@@ -69,7 +69,6 @@ isort==5.13.2
 jaraco.classes==3.3.0
 jedi==0.19.1
 jeepney==0.8.0
-Jinja2==3.1.2
 joblib==1.4.2
 json5==0.9.25
 jsonpatch==1.33
@@ -100,7 +99,6 @@ loguru==0.7.2
 loguru-mypy==0.0.4
 lxml==5.0.0
 markdown-it-py==3.0.0
-MarkupSafe==2.1.3
 marshmallow==3.20.1
 matplotlib==3.9.1
 matplotlib-inline==0.1.7
@@ -108,7 +106,6 @@ mdit-py-plugins==0.4.0
 mdurl==0.1.2
 mistune==3.0.2
 more-itertools==10.1.0
-mpmath==1.3.0
 msal==1.30.0
 msal-extensions==1.2.0
 msgpack==1.0.8
@@ -122,24 +119,11 @@ nbconvert==7.16.4
 nbformat==5.10.4
 ndindex==1.8
 nest-asyncio==1.6.0
-networkx==3.2.1
 nh3==0.2.15
 notebook==7.2.1
 notebook_shim==0.2.4
 numexpr==2.10.1
 numpy==1.26.2
-nvidia-cublas-cu12==12.1.3.1
-nvidia-cuda-cupti-cu12==12.1.105
-nvidia-cuda-nvrtc-cu12==12.1.105
-nvidia-cuda-runtime-cu12==12.1.105
-nvidia-cudnn-cu12==8.9.2.26
-nvidia-cufft-cu12==11.0.2.54
-nvidia-curand-cu12==10.3.2.106
-nvidia-cusolver-cu12==11.4.5.107
-nvidia-cusparse-cu12==12.1.0.106
-nvidia-nccl-cu12==2.18.1
-nvidia-nvjitlink-cu12==12.3.101
-nvidia-nvtx-cu12==12.1.105
 oauthlib==3.2.2
 openai==1.6.1
 overrides==7.7.0
@@ -151,7 +135,6 @@ parso==0.8.4
 pathspec==0.12.1
 patsy==0.5.6
 pexpect==4.9.0
-pillow==10.4.0
 pkginfo==1.9.6
 platformdirs==4.1.0
 pluggy==1.3.0
@@ -199,7 +182,6 @@ ruamel.yaml==0.18.5
 ruamel.yaml.clib==0.2.8
 ruff==0.4.5
 scikit-learn==1.5.1
-scipy==1.11.4
 SecretStorage==3.3.3
 semver==3.0.2
 Send2Trash==1.8.3
@@ -224,7 +206,6 @@ sphinxcontrib-serializinghtml==1.1.9
 SQLAlchemy==2.0.24
 stack-data==0.6.3
 statsmodels==0.14.2
-sympy==1.12
 tables==3.9.2
 tabulate==0.9.0
 tenacity==8.2.3
@@ -235,14 +216,11 @@ tiktoken==0.7.0
 tinycss2==1.3.0
 toml-sort==0.23.1
 tomlkit==0.12.3
-torch==2.1.2
-torch_geometric==2.5.3
 tornado==6.4
 tqdm==4.66.1
 traitlets==5.14.3
 tree-sitter==0.22.3
 tree-sitter-python==0.21.0
-triton==2.1.0
 twine==4.0.2
 typer==0.9.0
 types-psutil==6.0.0.20240621
@@ -250,7 +228,6 @@ types-python-dateutil==2.9.0.20240316
 types-PyYAML==6.0.12.20240724
 types-tqdm==4.66.0.20240417
 typing-inspect==0.9.0
-typing_extensions==4.9.0
 tzdata==2023.4
 uri-template==1.3.0
 urllib3==2.1.0

docs/conf.py

@@ -6,7 +6,9 @@
 # -- Project information -----------------------------------------------------
 # https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information
 
-import importlib.metadata
+import subprocess
+
+latest_tag = subprocess.check_output(["git", "describe", "--tags", "--abbrev=0"], text=True).strip()
 
 project = "RDAgent"
 copyright = "2024, Microsoft"
@@ -20,7 +22,7 @@ extensions = ["sphinx.ext.autodoc", "sphinxcontrib.autodoc_pydantic"]
 autodoc_member_order = "bysource"
 
 # The suffix of source filenames.
-source_suffix = ".rst"
+source_suffix = {".rst": "restructuredtext"}
 
 # The encoding of source files.
 source_encoding = "utf-8"
@@ -33,8 +35,8 @@ master_doc = "index"
 # built documents.
 #
 # The short X.Y version.
-version = importlib.metadata.version("rdagent")
-release = importlib.metadata.version("rdagent")
+version = latest_tag
+release = latest_tag
 
 # The language for content autogenerated by Sphinx. Refer to documentation for
 # a list of supported languages.
@@ -59,4 +61,12 @@ try:
 except ImportError:
     html_theme = "default"
 
+html_logo = "_static/logo.png"
 html_static_path = ["_static"]
+html_favicon = "_static/favicon.ico"
+
+html_theme_options = {
+    "source_repository": "https://github.com/microsoft/RD-Agent",
+    "source_branch": "main",
+    "source_directory": "docs/",
+}

docs/development.rst

@@ -2,26 +2,35 @@
 For Development
 =========================
 
+If you want to try the latest version or contribute to RD-Agent. You can install it from the source and follow the commands in this page.
+
+   .. code-block:: bash
+
+      git clone https://github.com/microsoft/RD-Agent
+
+
 🔧Prepare for development
 =========================
 
 - Set up the development environment.
 
-   ```bash
-   make dev
-   ```
+   .. code-block:: bash
+
+      make dev
 
 - Run linting and checking.
 
-   ```bash
-   make lint
-   ```
+   .. code-block:: bash
+
+      make lint
+
 
 - Some linting issues can be fixed automatically. We have added a command in the Makefile for easy use.
 
-  ```bash
-  make auto-lint
-  ```
+   .. code-block:: bash
+
+      make auto-lint
+
 
 
 Code Structure
@@ -73,4 +82,4 @@ File Naming Convention
    * - `conf.py`
      - The configuration for the module, app, and project.
 
-<!-- TODO: renaming files -->
+.. <!-- TODO: renaming files -->

docs/index.rst

@@ -6,6 +6,9 @@
 Welcome to RDAgent's documentation!
 ===================================
 
+.. image:: _static/logo.png
+   :alt: RD-Agent Logo
+
 .. toctree::
    :maxdepth: 3
    :caption: Doctree:
@@ -20,6 +23,8 @@ Welcome to RDAgent's documentation!
    api_reference
    policy
 
+   GitHub <https://github.com/microsoft/RD-Agent>
+
 
 Indices and tables
 ==================

docs/installation_and_configuration.rst

@@ -138,8 +138,28 @@ Configuration List
 +------------------------------+--------------------------------------------------+-------------------------+
 | prompt_cache_path            | Path to prompt cache                             | ./prompt_cache.db       |
 +------------------------------+--------------------------------------------------+-------------------------+
-| session_cache_folder_location| Path to session cache                            | ./session_cache_folder  |
-+------------------------------+--------------------------------------------------+-------------------------+
 | max_past_message_include     | Maximum number of past messages to include       | 10                      |
 +------------------------------+--------------------------------------------------+-------------------------+
 
+
+
+
+Loading Configuration
+---------------------
+
+For users' convenience, we provide a CLI interface called `rdagent`, which automatically runs `load_dotenv()` to load environment variables from the `.env` file.
+However, this feature is not enabled by default for other scripts. We recommend users load the environment with the following steps:
+
+
+- ⚙️ Environment Configuration
+    - Place the `.env` file in the same directory as the `.env.example` file.
+        - The `.env.example` file contains the environment variables required for users using the OpenAI API (Please note that `.env.example` is an example file. `.env` is the one that will be finally used.)
+
+    - Export each variable in the .env file:
+
+      .. code-block:: sh
+
+          export $(grep -v '^#' .env | xargs)
+    
+    - If you want to change the default environment variables, you can refer to the above configuration and edith the `.env` file.
+

docs/introduction.rst

@@ -6,7 +6,7 @@ Introduction
 
 In modern industry, research and development (R&D) is crucial for the enhancement of industrial productivity, especially in the AI era, where the core aspects of R&D are mainly focused on data and models. We are committed to automate these high-value generic R&D processes through our open source R&D automation tool RDAgent, which let AI drive data-driven AI.
 
-.. image:: _static/scen.jpg
+.. image:: _static/scen.png
    :alt: Our focused scenario
 
 

docs/project_framework_introduction.rst

@@ -7,7 +7,7 @@ Framework & Components
 
 .. NOTE: This depends on the correctness of `c-v` of github.
 
-.. image:: https://github.com/user-attachments/assets/98fce923-77ab-4982-93c8-a7a01aece766
+.. image:: _static/Framework-RDAgent.png
     :alt: Components & Feature Level
 
 The image above shows the overall framework of RDAgent.
@@ -23,19 +23,5 @@ We have established a basic method framework that continuously proposes hypothes
 The figure above shows the main classes and how they fit into the workflow for those interested in the detailed code.
 
 
-Detailed Design
-=========================
-
-
-Configuration
--------------
-
-You can manually source the `.env` file in your shell before running the Python script:
-Most of the workflow are controlled by the environment variables.
-```sh
-# Export each variable in the .env file; Please note that it is different from `source .env` without export
-export $(grep -v '^#' .env | xargs)
-# Run the Python script
-python your_script.py
-```
-
+.. Detailed Design
+.. ===============

docs/scens/data_agent_fin.rst

@@ -10,17 +10,24 @@ Finance Data Agent
 
 📖 Background
 ~~~~~~~~~~~~~~
-In the dynamic world of quantitative trading, **factors** are the secret weapons that traders use to harness market inefficiencies. 
+In the dynamic world of quantitative trading, **factors** serve as the strategic tools that enable traders to exploit market inefficiencies. 
+These factors—ranging from simple metrics like price-to-earnings ratios to complex models like discounted cash flows—are the key to predicting stock prices with a high degree of accuracy.
 
-These powerful tools—ranging from straightforward metrics like price-to-earnings ratios to intricate discounted cash flow models—unlock the potential to predict stock prices with remarkable precision. 
-By tapping into this rich vein of data, quantitative traders craft sophisticated strategies that not only capitalize on market patterns but also drastically enhance trading efficiency and accuracy. 
+By leveraging these factors, quantitative traders can develop sophisticated strategies that not only identify market patterns but also significantly enhance trading efficiency and precision. 
+The ability to systematically analyze and apply these factors is what separates ordinary trading from truly strategic market outmaneuvering.
+And this is where the **Finance Model Agent** comes into play.
 
-Embrace the power of factors, and you're not just trading; you're strategically outsmarting the market.
+🎥 `Demo <https://rdagent.azurewebsites.net/factor_loop>`_
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
+.. raw:: html
 
-🎥 Demo
-~~~~~~~~~~
-TODO: Here should put a video of the demo.
+    <div style="display: flex; justify-content: center; align-items: center;">
+      <video width="600" controls>
+        <source src="https://rdagent.azurewebsites.net/media/65bb598f1372c1857ccbf09b2acf5d55830911625048c03102291098.mp4" type="video/mp4">
+        Your browser does not support the video tag.
+      </video>
+    </div>
 
 
 🌟 Introduction
@@ -76,52 +83,39 @@ Here's an enhanced outline of the steps:
 ⚡ Quick Start
 ~~~~~~~~~~~~~~~~~
 
+Please refer to the installation part in :doc:`../installation_and_configuration` to prepare your system dependency.
+
 You can try our demo by running the following command:
 
 - 🐍 Create a Conda Environment
-    - Create a new conda environment with Python (3.10 and 3.11 are well tested in our CI):
-    
-      .. code-block:: sh
-      
-          conda create -n rdagent python=3.10
 
-    - Activate the environment:
+  - Create a new conda environment with Python (3.10 and 3.11 are well tested in our CI):
 
-      .. code-block:: sh
-
-          conda activate rdagent
-
-- 🛠️ Run Make Files
-    - Navigate to the directory containing the MakeFile and set up the development environment:
-
-      .. code-block:: sh
-
-          make dev
-
-- 📦 Install Pytorch
-    - Install Pytorch and related libraries:
-
-      .. code-block:: sh
-
-          pip3 install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu
-          pip3 install torch_geometric
-
-- ⚙️ Environment Configuration
-    - Place the `.env` file in the same directory as the `.env.example` file.
-        - The `.env.example` file contains the environment variables required for users using the OpenAI API (Please note that `.env.example` is an example file. `.env` is the one that will be finally used.)
-
-    - Export each variable in the .env file:
-
-      .. code-block:: sh
-
-          export $(grep -v '^#' .env | xargs)
-    
-    - If you want to change the default environment variables, you can refer to `Env Config`_ below
-
-- 🚀 Run the Application
     .. code-block:: sh
 
-        python rdagent/app/qlib_rd_loop/factor_w_sc.py
+          conda create -n rdagent python=3.10
+
+  - Activate the environment:
+
+    .. code-block:: sh
+
+        conda activate rdagent
+
+- 📦 Install the RDAgent
+  
+  - You can install the RDAgent package from PyPI:
+
+    .. code-block:: sh
+
+        pip install rdagent
+
+- 🚀 Run the Application
+    
+  - You can directly run the application by using the following command:
+    
+    .. code-block:: sh
+
+        rdagent fin_factor
 
 
 🛠️ Usage of modules
@@ -132,33 +126,13 @@ You can try our demo by running the following command:
 - **Env Config**
 
 The following environment variables can be set in the `.env` file to customize the application's behavior:
-    - **Path to the folder containing private data (default fundamental data in Qlib):**
 
-        .. code-block:: sh
-
-          FACTOR_CODER_DATA_FOLDER=/path/to/data/factor_implementation_source_data_all
-
-    - **Path to the folder containing partial private data (for debugging):**
-
-      .. code-block:: sh
-
-          FACTOR_CODER_DATA_FOLDER_DEBUG=/path/to/data/factor_implementation_source_data_debug
-
-    - **Maximum time (in seconds) for writing factor code:**
-
-      .. code-block:: sh
-
-          FACTOR_CODER_FILE_BASED_EXECUTION_TIMEOUT=300
-
-    - **Maximum number of factors to write in one experiment:**
-
-      .. code-block:: sh
-
-          FACTOR_CODER_SELECT_THRESHOLD=5
-
-    - **Number of developing loops for writing factors:**
-
-      .. code-block:: sh
-
-          FACTOR_CODER_MAX_LOOP=10
+.. autopydantic_settings:: rdagent.app.qlib_rd_loop.conf.FactorBasePropSetting
+    :settings-show-field-summary: False
+    :exclude-members: Config
 
+.. autopydantic_settings:: rdagent.components.coder.factor_coder.config.FactorImplementSettings
+    :settings-show-field-summary: False
+    :members: coder_use_cache, data_folder, data_folder_debug, file_based_execution_timeout, select_method, select_threshold, max_loop, knowledge_base_path, new_knowledge_base_path
+    :exclude-members: Config, fail_task_trial_limit, v1_query_former_trace_limit, v1_query_similar_success_limit, v2_query_component_limit, v2_query_error_limit, v2_query_former_trace_limit, v2_error_summary, v2_knowledge_sampler
+    :no-index:

docs/scens/data_copilot_fin.rst

@@ -17,12 +17,20 @@ Furthermore, rather than hastily replicating factors from a report, it's essenti
 Does the factor capture the essential market dynamics? How unique is it compared to the factors already in your library?
 
 Therefore, there is an urgent need for a systematic approach to design a framework that can effectively manage this process. 
-This is where our RDAgent comes into play.
+And this is where the **Finance Data Copilot** steps in.
 
 
-🎥 Demo
-~~~~~~~~~~
-TODO: Here should put a video of the demo.
+🎥 `Demo <https://rdagent.azurewebsites.net/report_factor>`_
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. raw:: html
+
+    <div style="display: flex; justify-content: center; align-items: center;">
+      <video width="600" controls>
+        <source src="https://rdagent.azurewebsites.net/media/7b14b2bd3d8771da9cf7eb799b6d96729cec3d35c8d4f68060f3e2fd.mp4" type="video/mp4">
+        Your browser does not support the video tag.
+      </video>
+    </div>
 
 
 🌟 Introduction
@@ -76,54 +84,64 @@ Here's an enhanced outline of the steps:
 ⚡ Quick Start
 ~~~~~~~~~~~~~~~~~
 
+Please refer to the installation part in :doc:`../installation_and_configuration` to prepare your system dependency.
+
 You can try our demo by running the following command:
 
 - 🐍 Create a Conda Environment
-    - Create a new conda environment with Python (3.10 and 3.11 are well tested in our CI):
     
-      .. code-block:: sh
-      
-          conda create -n rdagent python=3.10
-
-    - Activate the environment:
-
-      .. code-block:: sh
-
-          conda activate rdagent
-
-- 🛠️ Run Make Files
-    - Navigate to the directory containing the MakeFile and set up the development environment:
-
-      .. code-block:: sh
-
-          make dev
-
-- 📦 Install Pytorch
-    - Install Pytorch and related libraries:
-
-      .. code-block:: sh
-
-          pip3 install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu
-          pip3 install torch_geometric
-
-- ⚙️ Environment Configuration
-    - Place the `.env` file in the same directory as the `.env.example` file.
-        - The `.env.example` file contains the environment variables required for users using the OpenAI API (Please note that `.env.example` is an example file. `.env` is the one that will be finally used.)
-
-    - Export each variable in the .env file:
-
-      .. code-block:: sh
-
-          export $(grep -v '^#' .env | xargs)
+  - Create a new conda environment with Python (3.10 and 3.11 are well tested in our CI):
     
-    - If you want to change the default environment variables, you can refer to `Env Config`_ below
+    .. code-block:: sh
+    
+        conda create -n rdagent python=3.10
+
+  - Activate the environment:
 
-- 🚀 Run the Application
     .. code-block:: sh
 
-        python rdagent/app/qlib_rd_loop/factor_from_report_w_sc.py
+        conda activate rdagent
 
+- 📦 Install the RDAgent
+  
+  - You can install the RDAgent package from PyPI:
 
+    .. code-block:: sh
+
+        pip install rdagent
+
+- 🚀 Run the Application
+    
+  - Download the financial reports you wish to extract factors from and store them in your preferred folder.
+
+  - Specifically, you can follow this example, or use your own method:
+
+    .. code-block:: sh
+
+        wget https://github.com/SunsetWolf/rdagent_resource/releases/download/reports/all_reports.zip
+        unzip all_reports.zip -d git_ignore_folder/reports
+
+  - Run the application with the following command:
+
+    .. code-block:: sh
+
+        rdagent fin_factor_report --report_folder=git_ignore_folder/reports
+
+  - Alternatively, you can store the paths of the reports in `report_result_json_file_path`. The format should be:
+
+    .. code-block:: json
+
+        [
+            "git_ignore_folder/report/fin_report1.pdf",
+            "git_ignore_folder/report/fin_report2.pdf",
+            "git_ignore_folder/report/fin_report3.pdf"
+        ]
+
+  - Then, run the application using the following command:
+
+    .. code-block:: sh
+
+        rdagent fin_factor_report
 
 🛠️ Usage of modules
 ~~~~~~~~~~~~~~~~~~~~~
@@ -133,32 +151,14 @@ You can try our demo by running the following command:
 - **Env Config**
 
 The following environment variables can be set in the `.env` file to customize the application's behavior:
-    - **Path to the folder containing research reports:**
 
-      .. code-block:: sh
+.. autopydantic_settings:: rdagent.app.qlib_rd_loop.conf.FactorFromReportPropSetting
+    :settings-show-field-summary: False
+    :show-inheritance:
+    :exclude-members: Config
 
-          QLIB_FACTOR_LOCAL_REPORT_PATH=/path/to/research/reports
-
-    - **Path to the JSON file listing research reports for factor extraction:**
-
-      .. code-block:: sh
-
-          QLIB_FACTOR_REPORT_RESULT_JSON_FILE_PATH=/path/to/reports/list.json
-
-    - **Maximum time (in seconds) for writing factor code:**
-
-      .. code-block:: sh
-
-          FACTOR_CODER_FILE_BASED_EXECUTION_TIMEOUT=300
-
-    - **Maximum number of factors to write in one experiment:**
-
-      .. code-block:: sh
-
-          FACTOR_CODER_SELECT_THRESHOLD=5
-
-    - **Number of developing loops for writing factors:**
-
-      .. code-block:: sh
-
-          FACTOR_CODER_MAX_LOOP=10
+.. autopydantic_settings:: rdagent.components.coder.factor_coder.config.FactorImplementSettings
+    :settings-show-field-summary: False
+    :members: coder_use_cache, data_folder, data_folder_debug, file_based_execution_timeout, select_method, select_threshold, max_loop, knowledge_base_path, new_knowledge_base_path
+    :exclude-members: Config, python_bin, fail_task_trial_limit, v1_query_former_trace_limit, v1_query_similar_success_limit, v2_query_component_limit, v2_query_error_limit, v2_query_former_trace_limit, v2_error_summary, v2_knowledge_sampler
+    :no-index:

docs/scens/model_agent_fin.rst

@@ -9,19 +9,33 @@ Finance Model Agent
 
 📖 Background
 ~~~~~~~~~~~~~~
-TODO
+In the realm of quantitative finance, both factor discovery and model development play crucial roles in driving performance. 
+While much attention is often given to the discovery of new financial factors, the **models** that leverage these factors are equally important. 
+The effectiveness of a quantitative strategy depends not only on the factors used but also on how well these factors are integrated into robust, predictive models.
 
-🎥 Demo
-~~~~~~~~~~
-TODO: Here should put a video of the demo.
+However, the process of developing and optimizing these models can be labor-intensive and complex, requiring continuous refinement and adaptation to ever-changing market conditions. 
+And this is where the **Finance Model Agent** steps in.
+
+
+🎥 `Demo <https://rdagent.azurewebsites.net/model_loop>`_
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. raw:: html
+
+    <div style="display: flex; justify-content: center; align-items: center;">
+      <video width="600" controls>
+        <source src="https://rdagent.azurewebsites.net/media/d85e8cab1da1cd3501d69ce837452f53a971a24911eae7bfa9237137.mp4" type="video/mp4">
+        Your browser does not support the video tag.
+      </video>
+    </div>
 
 
 🌟 Introduction
 ~~~~~~~~~~~~~~~~
 
-In this scenario, our automated system proposes hypothesis, constructs model, implements code, receives back-testing, and uses feedbacks. 
-Hypothesis is iterated in this continuous process. 
-The system aims to automatically optimise performance metrics from Qlib library thereby finding the optimised code through autonomous research and development.
+In this scenario, our automated system proposes hypothesis, constructs model, implements code, conducts back-testing, and utilizes feedback in a continuous, iterative process.
+
+The goal is to automatically optimize performance metrics within the Qlib library, ultimately discovering the most efficient code through autonomous research and development.
 
 Here's an enhanced outline of the steps:
 
@@ -69,51 +83,68 @@ Here's an enhanced outline of the steps:
 ⚡ Quick Start
 ~~~~~~~~~~~~~~~~~
 
+Please refer to the installation part in :doc:`../installation_and_configuration` to prepare your system dependency.
+
 You can try our demo by running the following command:
 
 - 🐍 Create a Conda Environment
-    - Create a new conda environment with Python (3.10 and 3.11 are well tested in our CI):
+
+  - Create a new conda environment with Python (3.10 and 3.11 are well tested in our CI):
+
+    .. code-block:: sh
     
-      .. code-block:: sh
-      
-          conda create -n rdagent python=3.10
+        conda create -n rdagent python=3.10
 
-    - Activate the environment:
+  - Activate the environment:
 
-      .. code-block:: sh
-
-          conda activate rdagent
-
-- 🛠️ Run Make Files
-    - Navigate to the directory containing the MakeFile and set up the development environment:
-
-      .. code-block:: sh
-
-          make dev
-
-- 📦 Install Pytorch
-    - Install Pytorch and related libraries:
-
-      .. code-block:: sh
-
-          pip3 install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu
-          pip3 install torch_geometric
-
-- ⚙️ Environment Configuration
-    - Place the `.env` file in the same directory as the `.env.example` file.
-        - The `.env.example` file contains the environment variables required for users using the OpenAI API (Please note that `.env.example` is an example file. `.env` is the one that will be finally used.)
-
-    - Export each variable in the .env file:
-
-      .. code-block:: sh
-
-          export $(grep -v '^#' .env | xargs)
-
-- 🚀 Run the Application
     .. code-block:: sh
 
-        python rdagent/app/qlib_rd_loop/model_w_sc.py
+        conda activate rdagent
+
+- 📦 Install the RDAgent
+    
+  - You can install the RDAgent package from PyPI:
+
+    .. code-block:: sh
+
+        pip install rdagent
+
+- 🚀 Run the Application
+    
+  - You can directly run the application by using the following command:
+    
+    .. code-block:: sh
+
+        rdagent fin_model
 
 🛠️ Usage of modules
 ~~~~~~~~~~~~~~~~~~~~~
-TODO: Show some examples:
+
+.. _Env Config: 
+
+- **Env Config**
+
+The following environment variables can be set in the `.env` file to customize the application's behavior:
+
+.. autopydantic_settings:: rdagent.app.qlib_rd_loop.conf.ModelBasePropSetting
+    :settings-show-field-summary: False
+    :exclude-members: Config
+
+- **Qlib Config**
+    - The `config.yaml` file located in the `model_template` folder contains the relevant configurations for running the developed model in Qlib. The default settings include key information such as:
+        - **market**: Specifies the market, which is set to `csi300`.
+        - **fields_group**: Defines the fields group, with the value `feature`.
+        - **col_list**: A list of columns used, including various indicators such as `RESI5`, `WVMA5`, `RSQR5`, and others.
+        - **start_time**: The start date for the data, set to `2008-01-01`.
+        - **end_time**: The end date for the data, set to `2020-08-01`.
+        - **fit_start_time**: The start date for fitting the model, set to `2008-01-01`.
+        - **fit_end_time**: The end date for fitting the model, set to `2014-12-31`.
+
+    - The default hyperparameters used in the configuration are as follows:
+        - **n_epochs**: The number of epochs, set to `100`.
+        - **lr**: The learning rate, set to `1e-3`.
+        - **early_stop**: The early stopping criterion, set to `10`.
+        - **batch_size**: The batch size, set to `2000`.
+        - **metric**: The evaluation metric, set to `loss`.
+        - **loss**: The loss function, set to `mse`.
+        - **n_jobs**: The number of parallel jobs, set to `20`.

docs/scens/model_agent_med.rst

@@ -1,5 +1,128 @@
 .. _model_agent_med:
 
-===================
+=======================
 Medical Model Agent
-===================
+=======================
+
+**🤖 Automated Medical Predtion Model Evolution**
+------------------------------------------------------------------------------------------
+
+📖 Background
+~~~~~~~~~~~~~~
+In this scenario, we consider the problem of risk prediction from patients' ICU monitoring data. We use the a public EHR dataset - MIMIC-III and extract a binary classification task for evaluating the framework.
+In this task, we aim at predicting the whether the patients will suffer from Acute Respiratory Failure (ARF) based their first 12 hours ICU monitoring data. 
+
+🎥 `Demo <https://rdagent.azurewebsites.net/dmm>`_
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. raw:: html
+
+    <div style="display: flex; justify-content: center; align-items: center;">
+      <video width="600" controls>
+        <source src="https://rdagent.azurewebsites.net/media/1653542fc1b9fa14a306c35c1b1fc48288f980793f38abe82b023af9.mp4" type="video/mp4">
+        Your browser does not support the video tag.
+      </video>
+    </div>
+
+
+🌟 Introduction
+~~~~~~~~~~~~~~~~
+
+In this scenario, our automated system proposes hypothesis, constructs model, implements code, receives back-testing, and uses feedbacks. 
+Hypothesis is iterated in this continuous process. 
+The system aims to automatically optimise performance metrics of medical prediction thereby finding the optimised code through autonomous research and development.
+
+Here's an enhanced outline of the steps:
+
+**Step 1 : Hypothesis Generation 🔍**
+
+- Generate and propose initial hypotheses based on previous experiment analysis and domain expertise, with thorough reasoning and justification.
+
+**Step 2 : Model Creation ✨**
+
+- Transform the hypothesis into a model.
+- Develop, define, and implement a machine learning model, including its name, description, and formulation.
+
+**Step 3 : Model Implementation 👨‍💻**
+
+- Implement the model code based on the detailed description.
+- Evolve the model iteratively as a developer would, ensuring accuracy and efficiency.
+
+**Step 4 : Backtesting with MIMIC-III 📉**
+
+- Conduct backtesting using the newly developed model on the extracted task from MIMIC-III.
+- Evaluate the model's effectiveness and performance in terms of AUROC score.
+
+**Step 5 : Feedback Analysis 🔍**
+
+- Analyze backtest results to assess performance.
+- Incorporate feedback to refine hypotheses and improve the model.
+
+**Step 6 :Hypothesis Refinement ♻️**
+
+- Refine hypotheses based on feedback from backtesting.
+- Repeat the process to continuously improve the model.
+
+⚡ Quick Start
+~~~~~~~~~~~~~~~~~
+
+Please refer to the installation part in :doc:`../installation_and_configuration` to prepare your system dependency.
+
+You can try our demo by running the following command:
+
+- 🐍 Create a Conda Environment
+  
+  - Create a new conda environment with Python (3.10 and 3.11 are well tested in our CI):
+
+    .. code-block:: sh
+    
+        conda create -n rdagent python=3.10
+
+  - Activate the environment:
+
+    .. code-block:: sh
+
+        conda activate rdagent
+
+- 📦 Install the RDAgent
+    
+  - You can install the RDAgent package from PyPI:
+
+    .. code-block:: sh
+
+        pip install rdagent
+
+- 📦 Request PhysioNet Account
+    
+  - Apply for an account at `PhysioNet <https://physionet.org/>`_.
+  - Request access to FIDDLE preprocessed data: `FIDDLE Dataset <https://physionet.org/content/mimic-eicu-fiddle-feature/1.0.0/>`_.
+  - Place your username and password in `.env`.
+
+    .. code-block:: bash
+
+        cat << EOF  >> .env
+        DM_USERNAME=<your_username>
+        DM_PASSWORD=<your_password>
+        EOF
+
+
+- 🚀 Run the Application
+    
+  - You can directly run the application by using the following command:
+    
+    .. code-block:: sh
+
+        rdagent med_model
+
+🛠️ Usage of modules
+~~~~~~~~~~~~~~~~~~~~~
+
+.. _Env Config: 
+
+- **Env Config**
+
+The following environment variables can be set in the `.env` file to customize the application's behavior:
+
+.. autopydantic_settings:: rdagent.app.data_mining.conf.MedBasePropSetting
+    :settings-show-field-summary: False
+    :exclude-members: Config

docs/scens/model_copilot_general.rst

@@ -9,11 +9,23 @@ General Model Copilot
 
 📖 Background
 ~~~~~~~~~~~~~~
-TODO:
+In the fast-paced field of artificial intelligence, the number of academic papers published each year is skyrocketing. 
+These papers introduce new models, techniques, and approaches that can significantly advance the state of the art. 
+However, reproducing and implementing these models can be a daunting task, requiring substantial time and expertise. 
+Researchers often face challenges in extracting the essential details from these papers and converting them into functional code.
+And this is where the **General Model Copilot** steps in.
 
-🎥 Demo
-~~~~~~~~~~
-TODO: Here should put a video of the demo.
+🎥 `Demo <https://rdagent.azurewebsites.net/report_model>`_
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. raw:: html
+
+    <div style="display: flex; justify-content: center; align-items: center;">
+      <video width="600" controls>
+        <source src="https://rdagent.azurewebsites.net/media/b35f904765b05099b0fcddbebe041a04f4d7bde239657e5fc24bf0cc.mp4" type="video/mp4">
+        Your browser does not support the video tag.
+      </video>
+    </div>
 
 🌟 Introduction
 ~~~~~~~~~~~~~~~~
@@ -45,79 +57,43 @@ This demo automates the extraction and iterative development of models from acad
 ⚡ Quick Start
 ~~~~~~~~~~~~~~~~~
 
+Please refer to the installation part in :doc:`../installation_and_configuration` to prepare your system dependency.
+
 You can try our demo by running the following command:
 
 - 🐍 Create a Conda Environment
-    - Create a new conda environment with Python (3.10 and 3.11 are well tested in our CI):
+  
+  - Create a new conda environment with Python (3.10 and 3.11 are well tested in our CI):
+
+    .. code-block:: sh
     
-      .. code-block:: sh
-      
-          conda create -n rdagent python=3.10
+        conda create -n rdagent python=3.10
 
-    - Activate the environment:
+  - Activate the environment:
 
-      .. code-block:: sh
+    .. code-block:: sh
 
-          conda activate rdagent
+        conda activate rdagent
 
-- 🛠️ Run Make Files
-    - Navigate to the directory containing the MakeFile and set up the development environment:
+- 📦 Install the RDAgent
+    
+  - You can install the RDAgent package from PyPI:
 
-      .. code-block:: sh
+    .. code-block:: sh
 
-          make dev
+        pip install rdagent
 
-- 📦 Install Pytorch
-    - Install Pytorch and related libraries:
-
-      .. code-block:: sh
-
-          pip3 install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu
-          pip3 install torch_geometric
-
-- ⚙️ Environment Configuration
-    - Place the `.env` file in the same directory as the `.env.example` file.
-        - The `.env.example` file contains the environment variables required for users using the OpenAI API (Please note that `.env.example` is an example file. `.env` is the one that will be finally used.)
-
-    - Export each variable in the .env file:
-
-      .. code-block:: sh
-
-          export $(grep -v '^#' .env | xargs)
 
 - 🚀 Run the Application
-    - Prepare relevant files (in pdf format) by uploading papers to the directory below and copy the path as report_file_path.
-      
-      .. code-block:: sh
-
-          rdagent/scenarios/general_model
     
-    - Run the following command in your terminal within the same virtual environment:
+  - Prepare relevant files (in pdf format) by uploading papers to the directory below and copy the path as report_file_path.
+      
+    .. code-block:: sh
 
-      .. code-block:: sh
+        rdagent/scenarios/general_model
+    
+  - Run the following command in your terminal within the same virtual environment:
+  
+    .. code-block:: sh
 
-          python rdagent/app/general_model/general_model.py report_file_path 
-
-🛠️ Usage of modules
-~~~~~~~~~~~~~~~~~~~~~
-There are mainly two modules in this scenario: one that reads the paper and returns a model card & one that reads the model card and returns functional code. The moduldes can also be used separately as components for developers to build up new scenarios.
-
-
-- Configurations
-    - The `config.yaml` file located in the `model_template` folder contains the relevant configurations for running the developed model in Qlib. The default settings include key information such as:
-        - **market**: Specifies the market, which is set to `csi300`.
-        - **fields_group**: Defines the fields group, with the value `feature`.
-        - **col_list**: A list of columns used, including various indicators such as `RESI5`, `WVMA5`, `RSQR5`, and others.
-        - **start_time**: The start date for the data, set to `2008-01-01`.
-        - **end_time**: The end date for the data, set to `2020-08-01`.
-        - **fit_start_time**: The start date for fitting the model, set to `2008-01-01`.
-        - **fit_end_time**: The end date for fitting the model, set to `2014-12-31`.
-
-    - The default hyperparameters used in the configuration are as follows:
-        - **n_epochs**: The number of epochs, set to `100`.
-        - **lr**: The learning rate, set to `1e-3`.
-        - **early_stop**: The early stopping criterion, set to `10`.
-        - **batch_size**: The batch size, set to `2000`.
-        - **metric**: The evaluation metric, set to `loss`.
-        - **loss**: The loss function, set to `mse`.
-        - **n_jobs**: The number of parallel jobs, set to `20`.
+        rdagent general_model --report_file_path=<path_to_pdf_file>

docs/ui.rst

@@ -18,18 +18,21 @@ In `RD-Agent/` folder, run:
 
 .. code-block:: bash
 
-    streamlit run rdagent/log/ui/app.py --server.port <port> -- --log_dir <log_dir>
+    rdagent ui --port <port> --log_dir <log_dir like "log/"> [--debug]
 
 This will start a web app on `http://localhost:<port>`.
 
 **NOTE**: The log_dir parameter is not required. You can manually enter the log_path in the web app. If you set the log_dir parameter, you can easily select a different log_path in the web app.
 
+--debug is optional, it will show a "Single Step Run" button in sidebar and saved objects info in the web app.
+
 Use Web App
 -----------
 
 1. Open the sidebar.
 
 .. TODO: update these
+
 2. Select the scenario you want to show. There are some pre-defined scenarios:
     - Qlib Model
     - Qlib Factor

pyproject.toml

@@ -31,6 +31,9 @@ name = "rdagent"
 readme = "README.md"
 requires-python = ">=3.10"
 
+[project.scripts]
+rdagent = "rdagent.app.cli:app"
+
 [project.urls]
 homepage = "https://github.com/microsoft/RD-Agent/"
 issue = "https://github.com/microsoft/RD-Agent/issues"

rdagent/app/benchmark/factor/analysis.py

@@ -2,7 +2,9 @@ import json
 import pickle
 from pathlib import Path
 
+import fire
 import matplotlib.pyplot as plt
+import numpy as np
 import pandas as pd
 import seaborn as sns
 
@@ -42,7 +44,24 @@ class BenchmarkAnalyzer:
             final_res[experiment] = processed_data.iloc[-1, :]
         return final_res
 
-    def reformat_succ_rate(self, display_df):
+    def reformat_index(self, display_df):
+        """
+        reform the results from
+
+        .. code-block:: python
+
+                              success rate
+            High_Beta_Factor           0.2
+
+        to
+
+        .. code-block:: python
+
+                                                    success rate
+            Category Difficulty Factor
+            量价       Hard       High_Beta_Factor           0.2
+
+        """
         new_idx = []
         display_df = display_df[display_df.index.isin(self.index_map.keys())]
         for idx in display_df.index:
@@ -76,11 +95,9 @@ class BenchmarkAnalyzer:
     def analyze_data(self, sum_df):
         index = [
             "FactorSingleColumnEvaluator",
-            "FactorOutputFormatEvaluator",
             "FactorRowCountEvaluator",
             "FactorIndexEvaluator",
-            "FactorMissingValuesEvaluator",
-            "FactorEqualValueCountEvaluator",
+            "FactorEqualValueRatioEvaluator",
             "FactorCorrelationEvaluator",
             "run factor error",
         ]
@@ -91,33 +108,35 @@ class BenchmarkAnalyzer:
         succ_rate = ~run_error
         succ_rate = succ_rate.mean(axis=0).to_frame("success rate")
 
-        succ_rate_f = self.reformat_succ_rate(succ_rate)
-        succ_rate_f
+        succ_rate_f = self.reformat_index(succ_rate)
 
-        sum_df_clean["FactorRowCountEvaluator"]
+        # if it rasis Error when running the evaluator, we will get NaN
+        # Running failures are reguarded to zero score.
+        format_issue = sum_df_clean[["FactorRowCountEvaluator", "FactorIndexEvaluator"]].apply(
+            lambda x: np.mean(x.fillna(0.0)), axis=1
+        )
+        format_succ_rate = format_issue.unstack().T.mean(axis=0).to_frame("success rate")
+        format_succ_rate_f = self.reformat_index(format_succ_rate)
 
-        format_issue = sum_df_clean["FactorRowCountEvaluator"] & sum_df_clean["FactorIndexEvaluator"]
-        eval_series = format_issue.unstack()
-        succ_rate = eval_series.T.fillna(False).astype(bool)  # false indicate failure
-        format_succ_rate = succ_rate.mean(axis=0).to_frame("success rate")
-        format_succ_rate_f = self.reformat_succ_rate(format_succ_rate)
-
-        corr = sum_df_clean["FactorCorrelationEvaluator"] * format_issue
+        corr = sum_df_clean["FactorCorrelationEvaluator"].fillna(0.0)
         corr = corr.unstack().T.mean(axis=0).to_frame("corr(only success)")
-        corr_res = self.reformat_succ_rate(corr)
-        corr_max = sum_df_clean["FactorCorrelationEvaluator"] * format_issue
+        corr_res = self.reformat_index(corr)
+        corr_max = sum_df_clean["FactorCorrelationEvaluator"]
 
         corr_max = corr_max.unstack().T.max(axis=0).to_frame("corr(only success)")
-        corr_max_res = self.reformat_succ_rate(corr_max)
+        corr_max_res = self.reformat_index(corr_max)
 
-        value_max = sum_df_clean["FactorMissingValuesEvaluator"] * format_issue
+        value_max = sum_df_clean["FactorEqualValueRatioEvaluator"]
         value_max = value_max.unstack().T.max(axis=0).to_frame("max_value")
-        value_max_res = self.reformat_succ_rate(value_max)
+        value_max_res = self.reformat_index(value_max)
 
         value_avg = (
-            (sum_df_clean["FactorMissingValuesEvaluator"] * format_issue).unstack().T.mean(axis=0).to_frame("avg_value")
+            (sum_df_clean["FactorEqualValueRatioEvaluator"] * format_issue)
+            .unstack()
+            .T.mean(axis=0)
+            .to_frame("avg_value")
         )
-        value_avg_res = self.reformat_succ_rate(value_avg)
+        value_avg_res = self.reformat_index(value_avg)
 
         result_all = pd.concat(
             {
@@ -159,20 +178,24 @@ class Plotter:
         plt.rc("figure", titlesize=font_size)
 
     @staticmethod
-    def plot_data(data, file_name):
+    def plot_data(data, file_name, title):
         plt.figure(figsize=(10, 6))
         sns.barplot(x="index", y="b", hue="a", data=data)
         plt.xlabel("Method")
         plt.ylabel("Value")
-        plt.title("Comparison of Different Methods")
+        plt.title(title)
         plt.savefig(file_name)
 
 
-if __name__ == "__main__":
+def main(
+    path="git_ignore_folder/eval_results/res_promptV220240724-060037.pkl",
+    round=1,
+    title="Comparison of Different Methods",
+):
     settings = BenchmarkSettings()
     benchmark = BenchmarkAnalyzer(settings)
     results = {
-        "1 round experiment": "git_ignore_folder/eval_results/res_promptV220240724-060037.pkl",
+        f"{round} round experiment": path,
     }
     final_results = benchmark.process_results(results)
     final_results_df = pd.DataFrame(final_results)
@@ -180,4 +203,8 @@ if __name__ == "__main__":
     Plotter.change_fs(20)
     plot_data = final_results_df.drop(["max. accuracy", "avg. accuracy"], axis=0).T
     plot_data = plot_data.reset_index().melt("index", var_name="a", value_name="b")
-    Plotter.plot_data(plot_data, "rdagent/app/quant_factor_benchmark/comparison_plot.png")
+    Plotter.plot_data(plot_data, "./comparison_plot.png", title)
+
+
+if __name__ == "__main__":
+    fire.Fire(main)

rdagent/app/benchmark/factor/eval.py

@@ -15,27 +15,28 @@ from rdagent.scenarios.qlib.factor_experiment_loader.json_loader import (
     FactorTestCaseLoaderFromJsonFile,
 )
 
-# 1.read the settings
-bs = BenchmarkSettings()
+if __name__ == "__main__":
+    # 1.read the settings
+    bs = BenchmarkSettings()
 
-# 2.read and prepare the eval_data
-test_cases = FactorTestCaseLoaderFromJsonFile().load(bs.bench_data_path)
+    # 2.read and prepare the eval_data
+    test_cases = FactorTestCaseLoaderFromJsonFile().load(bs.bench_data_path)
 
-# 3.declare the method to be tested and pass the arguments.
+    # 3.declare the method to be tested and pass the arguments.
 
-scen: Scenario = import_class(FACTOR_PROP_SETTING.scen)()
-generate_method = import_class(bs.bench_method_cls)(scen=scen)
-# 4.declare the eval method and pass the arguments.
-eval_method = FactorImplementEval(
-    method=generate_method,
-    test_cases=test_cases,
-    scen=scen,
-    catch_eval_except=True,
-    test_round=bs.bench_test_round,
-)
+    scen: Scenario = import_class(FACTOR_PROP_SETTING.scen)()
+    generate_method = import_class(bs.bench_method_cls)(scen=scen)
+    # 4.declare the eval method and pass the arguments.
+    eval_method = FactorImplementEval(
+        method=generate_method,
+        test_cases=test_cases,
+        scen=scen,
+        catch_eval_except=True,
+        test_round=bs.bench_test_round,
+    )
 
-# 5.run the eval
-res = eval_method.eval()
+    # 5.run the eval
+    res = eval_method.eval()
 
-# 6.save the result
-logger.log_object(res)
+    # 6.save the result
+    logger.log_object(res)

rdagent/app/benchmark/model/README.md

@@ -1,17 +1,3 @@
-
-# Preparation
-
-## Install Pytorch
-CPU CUDA will be enough for verify the implementation
-
-Please install pytorch based on your system.
-Here is an example on my system
-```bash
-pip3 install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu
-pip3 install torch_geometric
-
-```
-
 # Tasks
 
 ## Task Extraction

rdagent/app/benchmark/model/eval.py

@@ -7,35 +7,36 @@ from rdagent.scenarios.qlib.experiment.model_experiment import (
     QlibModelScenario,
 )
 
-DIRNAME = Path(__file__).absolute().resolve().parent
+if __name__ == "__main__":
+    DIRNAME = Path(__file__).absolute().resolve().parent
 
-from rdagent.components.coder.model_coder.benchmark.eval import ModelImpValEval
-from rdagent.components.coder.model_coder.one_shot import ModelCodeWriter
+    from rdagent.components.coder.model_coder.benchmark.eval import ModelImpValEval
+    from rdagent.components.coder.model_coder.one_shot import ModelCodeWriter
 
-bench_folder = DIRNAME.parent.parent / "components" / "coder" / "model_coder" / "benchmark"
-mtl = ModelTaskLoaderJson(str(bench_folder / "model_dict.json"))
+    bench_folder = DIRNAME.parent.parent / "components" / "coder" / "model_coder" / "benchmark"
+    mtl = ModelTaskLoaderJson(str(bench_folder / "model_dict.json"))
 
-task_l = mtl.load()
+    task_l = mtl.load()
 
-task_l = [t for t in task_l if t.name == "A-DGN"]  # FIXME: other models does not work well
+    task_l = [t for t in task_l if t.name == "A-DGN"]  # FIXME: other models does not work well
 
-model_experiment = QlibModelExperiment(sub_tasks=task_l)
-# mtg = ModelCodeWriter(scen=QlibModelScenario())
-mtg = ModelCoSTEER(scen=QlibModelScenario())
+    model_experiment = QlibModelExperiment(sub_tasks=task_l)
+    # mtg = ModelCodeWriter(scen=QlibModelScenario())
+    mtg = ModelCoSTEER(scen=QlibModelScenario())
 
-model_experiment = mtg.develop(model_experiment)
+    model_experiment = mtg.develop(model_experiment)
 
-# TODO: Align it with the benchmark framework after @wenjun's refine the evaluation part.
-# Currently, we just handcraft a workflow for fast evaluation.
+    # TODO: Align it with the benchmark framework after @wenjun's refine the evaluation part.
+    # Currently, we just handcraft a workflow for fast evaluation.
 
-mil = ModelWsLoader(bench_folder / "gt_code")
+    mil = ModelWsLoader(bench_folder / "gt_code")
 
-mie = ModelImpValEval()
-# Evaluation:
-eval_l = []
-for impl in model_experiment.sub_workspace_list:
-    print(impl.target_task)
-    gt_impl = mil.load(impl.target_task)
-    eval_l.append(mie.evaluate(gt_impl, impl))
+    mie = ModelImpValEval()
+    # Evaluation:
+    eval_l = []
+    for impl in model_experiment.sub_workspace_list:
+        print(impl.target_task)
+        gt_impl = mil.load(impl.target_task)
+        eval_l.append(mie.evaluate(gt_impl, impl))
 
-print(eval_l)
+    print(eval_l)

rdagent/app/cli.py

@@ -0,0 +1,57 @@
+"""
+CLI entrance for all rdagent application.
+
+This will 
+- make rdagent a nice entry and
+- autoamtically load dotenv
+"""
+from dotenv import load_dotenv
+
+load_dotenv(".env")
+# 1) Make sure it is at the beginning of the script so that it will load dotenv before initializing BaseSettings.
+# 2) The ".env" argument is necessary to make sure it loads `.env` from the current directory.
+
+import subprocess
+from importlib.resources import path as rpath
+
+import fire
+
+from rdagent.app.data_mining.model import main as med_model
+from rdagent.app.general_model.general_model import (
+    extract_models_and_implement as general_model,
+)
+from rdagent.app.kaggle.loop import main as kaggle_main
+from rdagent.app.qlib_rd_loop.factor import main as fin_factor
+from rdagent.app.qlib_rd_loop.factor_from_report import main as fin_factor_report
+from rdagent.app.qlib_rd_loop.model import main as fin_model
+from rdagent.app.utils.info import collect_info
+
+
+def ui(port=80, log_dir="", debug=False):
+    """
+    start web app to show the log traces.
+    """
+    with rpath("rdagent.log.ui", "app.py") as app_path:
+        cmds = ["streamlit", "run", app_path, f"--server.port={port}"]
+        if log_dir or debug:
+            cmds.append("--")
+        if log_dir:
+            cmds.append(f"--log_dir={log_dir}")
+        if debug:
+            cmds.append("--debug")
+        subprocess.run(cmds)
+
+
+def app():
+    fire.Fire(
+        {
+            "fin_factor": fin_factor,
+            "fin_factor_report": fin_factor_report,
+            "fin_model": fin_model,
+            "med_model": med_model,
+            "general_model": general_model,
+            "ui": ui,
+            "collect_info": collect_info,
+            "kaggle": kaggle_main,
+        }
+    )

rdagent/app/data_mining/conf.py

@@ -1,20 +1,38 @@
 from pathlib import Path
 
+from pydantic_settings import BaseSettings
+
 from rdagent.components.workflow.conf import BasePropSetting
 
 
-class PropSetting(BasePropSetting):
+class MedBasePropSetting(BasePropSetting):
     class Config:
-        env_prefix = "DM_"  # Use MODEL_CODER_ as prefix for environment variables
-        protected_namespaces = ()  # Add 'model_' to the protected namespaces
+        env_prefix = "DM_"
+        """Use `DM_` as prefix for environment variables"""
+        protected_namespaces = ()
+        """Add 'model_' to the protected namespaces"""
 
     # 1) overriding the default
     scen: str = "rdagent.scenarios.data_mining.experiment.model_experiment.DMModelScenario"
+    """Scenario class for data mining model"""
+
     hypothesis_gen: str = "rdagent.scenarios.data_mining.proposal.model_proposal.DMModelHypothesisGen"
+    """Hypothesis generation class"""
+
     hypothesis2experiment: str = "rdagent.scenarios.data_mining.proposal.model_proposal.DMModelHypothesis2Experiment"
+    """Hypothesis to experiment class"""
+
     coder: str = "rdagent.scenarios.data_mining.developer.model_coder.DMModelCoSTEER"
+    """Coder class"""
+
     runner: str = "rdagent.scenarios.data_mining.developer.model_runner.DMModelRunner"
+    """Runner class"""
+
     summarizer: str = "rdagent.scenarios.data_mining.developer.feedback.DMModelHypothesisExperiment2Feedback"
+    """Summarizer class"""
+
+    evolving_n: int = 10
+    """Number of evolutions"""
 
     evolving_n: int = 10
 
@@ -22,7 +40,10 @@ class PropSetting(BasePropSetting):
     # physionet account
     # NOTE: You should apply the account in https://physionet.org/
     username: str = ""
+    """Physionet account username"""
+
     password: str = ""
+    """Physionet account password"""
 
 
-PROP_SETTING = PropSetting()
+MED_PROP_SETTING = MedBasePropSetting()

rdagent/app/data_mining/model.py

@@ -1,6 +1,6 @@
 import fire
 
-from rdagent.app.data_mining.conf import PROP_SETTING
+from rdagent.app.data_mining.conf import MED_PROP_SETTING
 from rdagent.components.workflow.rd_loop import RDLoop
 from rdagent.core.exception import ModelEmptyError
 
@@ -11,6 +11,8 @@ class ModelRDLoop(RDLoop):
 
 def main(path=None, step_n=None):
     """
+    Auto R&D Evolving loop for models in a medical scenario.
+
     You can continue running session by
 
     .. code-block:: python
@@ -19,7 +21,7 @@ def main(path=None, step_n=None):
 
     """
     if path is None:
-        model_loop = ModelRDLoop(PROP_SETTING)
+        model_loop = ModelRDLoop(MED_PROP_SETTING)
     else:
         model_loop = ModelRDLoop.load(path)
     model_loop.run(step_n=step_n)

rdagent/app/general_model/general_model.py

@@ -1,10 +1,3 @@
-# %%
-from dotenv import load_dotenv
-
-from rdagent.scenarios.general_model.scenario import GeneralModelScenario
-
-load_dotenv(override=True)
-
 import fire
 
 from rdagent.components.coder.model_coder.task_loader import (
@@ -14,12 +7,29 @@ from rdagent.components.document_reader.document_reader import (
     extract_first_page_screenshot_from_pdf,
 )
 from rdagent.log import rdagent_logger as logger
+from rdagent.scenarios.general_model.scenario import GeneralModelScenario
 from rdagent.scenarios.qlib.developer.model_coder import QlibModelCoSTEER
 
 
-def extract_models_and_implement(
-    report_file_path: str,
-) -> None:
+def extract_models_and_implement(report_file_path: str) -> None:
+    """
+    This is a research copilot to automatically implement models from a report file or paper.
+
+    It extracts models from a given PDF report file and implements the necessary operations.
+
+    Parameters:
+    report_file_path (str): The path to the report file. The file must be a PDF file.
+
+    Example URLs of PDF reports:
+    - https://arxiv.org/pdf/2210.09789
+    - https://arxiv.org/pdf/2305.10498
+    - https://arxiv.org/pdf/2110.14446
+    - https://arxiv.org/pdf/2205.12454
+    - https://arxiv.org/pdf/2210.16518
+
+    Returns:
+    None
+    """
     with logger.tag("init"):
         scenario = GeneralModelScenario()
         logger.log_object(scenario, tag="scenario")

rdagent/app/kaggle/conf.py

@@ -0,0 +1,72 @@
+from pathlib import Path
+
+from pydantic_settings import BaseSettings
+
+from rdagent.components.workflow.conf import BasePropSetting
+
+
+class KaggleBasePropSetting(BasePropSetting):
+    class Config:
+        env_prefix = "KG_"
+        """Use `KG_` as prefix for environment variables"""
+        protected_namespaces = ()
+        """Do not allow overriding of these namespaces"""
+
+    # 1) overriding the default
+    scen: str = "rdagent.scenarios.kaggle.experiment.scenario.KGScenario"
+    """Scenario class for data mining model"""
+
+    knowledge_base: str = ""  # TODO enable this line to use the knowledge base
+    # knowledge_base: str = "rdagent.scenarios.kaggle.knowledge_management.graph.KGKnowledgeGraph"
+    """Knowledge base class"""
+
+    knowledge_base_path: str = "kg_graph.pkl"
+    """Knowledge base path"""
+
+    hypothesis_gen: str = "rdagent.scenarios.kaggle.proposal.proposal.KGHypothesisGen"
+    """Hypothesis generation class"""
+
+    hypothesis2experiment: str = "rdagent.scenarios.kaggle.proposal.proposal.KGHypothesis2Experiment"
+    """Hypothesis to experiment class"""
+
+    feature_coder: str = "rdagent.scenarios.kaggle.developer.coder.KGFactorCoSTEER"
+    """Feature Coder class"""
+
+    model_feature_selection_coder: str = "rdagent.scenarios.kaggle.developer.coder.KGModelFeatureSelectionCoder"
+    """Model Feature Selection Coder class"""
+
+    model_coder: str = "rdagent.scenarios.kaggle.developer.coder.KGModelCoSTEER"
+    """Model Coder class"""
+
+    feature_runner: str = "rdagent.scenarios.kaggle.developer.runner.KGFactorRunner"
+    """Feature Runner class"""
+
+    model_runner: str = "rdagent.scenarios.kaggle.developer.runner.KGModelRunner"
+    """Model Runner class"""
+
+    summarizer: str = "rdagent.scenarios.kaggle.developer.feedback.KGHypothesisExperiment2Feedback"
+    """Summarizer class"""
+
+    evolving_n: int = 10
+    """Number of evolutions"""
+
+    competition: str = ""
+
+    local_data_path: str = "/data/userdata/share/kaggle"
+
+    domain_knowledge_path: str = "/data/userdata/share/kaggle/domain_knowledge"
+
+    rag_path: str = "git_ignore_folder/rag"
+
+    if_action_choosing_based_on_UCB: bool = False
+
+    if_using_graph_rag: bool = False
+
+    if_using_vector_rag: bool = False
+
+    auto_submit: bool = True
+
+    mini_case: bool = False
+
+
+KAGGLE_IMPLEMENT_SETTING = KaggleBasePropSetting()

rdagent/app/kaggle/loop.py

@@ -0,0 +1,142 @@
+import subprocess
+from collections import defaultdict
+from typing import Any
+
+import fire
+
+from rdagent.app.kaggle.conf import KAGGLE_IMPLEMENT_SETTING
+from rdagent.components.workflow.conf import BasePropSetting
+from rdagent.components.workflow.rd_loop import RDLoop
+from rdagent.core.developer import Developer
+from rdagent.core.exception import FactorEmptyError, ModelEmptyError
+from rdagent.core.proposal import (
+    Hypothesis2Experiment,
+    HypothesisExperiment2Feedback,
+    HypothesisGen,
+    Trace,
+)
+from rdagent.core.scenario import Scenario
+from rdagent.core.utils import import_class
+from rdagent.log import rdagent_logger as logger
+from rdagent.log.time import measure_time
+from rdagent.scenarios.kaggle.experiment.scenario import (
+    KG_ACTION_FEATURE_ENGINEERING,
+    KG_ACTION_FEATURE_PROCESSING,
+    KG_ACTION_MODEL_FEATURE_SELECTION,
+)
+from rdagent.scenarios.kaggle.experiment.utils import python_files_to_notebook
+from rdagent.scenarios.kaggle.kaggle_crawler import download_data
+from rdagent.scenarios.kaggle.proposal.proposal import KGTrace
+
+
+class KaggleRDLoop(RDLoop):
+    @measure_time
+    def __init__(self, PROP_SETTING: BasePropSetting):
+        with logger.tag("init"):
+            scen: Scenario = import_class(PROP_SETTING.scen)(PROP_SETTING.competition)
+            logger.log_object(scen, tag="scenario")
+            knowledge_base = (
+                import_class(PROP_SETTING.knowledge_base)(PROP_SETTING.knowledge_base_path, scen)
+                if PROP_SETTING.knowledge_base != ""
+                else None
+            )
+            logger.log_object(knowledge_base, tag="knowledge_base")
+            self.hypothesis_gen: HypothesisGen = import_class(PROP_SETTING.hypothesis_gen)(scen)
+            logger.log_object(self.hypothesis_gen, tag="hypothesis generator")
+            self.hypothesis2experiment: Hypothesis2Experiment = import_class(PROP_SETTING.hypothesis2experiment)()
+            logger.log_object(self.hypothesis2experiment, tag="hypothesis2experiment")
+            self.feature_coder: Developer = import_class(PROP_SETTING.feature_coder)(scen)
+            logger.log_object(self.feature_coder, tag="feature coder")
+            self.model_feature_selection_coder: Developer = import_class(PROP_SETTING.model_feature_selection_coder)(
+                scen
+            )
+            logger.log_object(self.model_feature_selection_coder, tag="model feature selection coder")
+            self.model_coder: Developer = import_class(PROP_SETTING.model_coder)(scen)
+            logger.log_object(self.model_coder, tag="model coder")
+            self.feature_runner: Developer = import_class(PROP_SETTING.feature_runner)(scen)
+            logger.log_object(self.feature_runner, tag="feature runner")
+            self.model_runner: Developer = import_class(PROP_SETTING.model_runner)(scen)
+            logger.log_object(self.model_runner, tag="model runner")
+            self.summarizer: HypothesisExperiment2Feedback = import_class(PROP_SETTING.summarizer)(scen)
+            logger.log_object(self.summarizer, tag="summarizer")
+            self.trace = KGTrace(scen=scen, knowledge_base=knowledge_base)
+            super(RDLoop, self).__init__()
+
+    @measure_time
+    def coding(self, prev_out: dict[str, Any]):
+        with logger.tag("d"):  # develop
+            if prev_out["propose"].action in [KG_ACTION_FEATURE_ENGINEERING, KG_ACTION_FEATURE_PROCESSING]:
+                exp = self.feature_coder.develop(prev_out["exp_gen"])
+            elif prev_out["propose"].action == KG_ACTION_MODEL_FEATURE_SELECTION:
+                exp = self.model_feature_selection_coder.develop(prev_out["exp_gen"])
+            else:
+                exp = self.model_coder.develop(prev_out["exp_gen"])
+            logger.log_object(exp.sub_workspace_list, tag="coder result")
+        return exp
+
+    @measure_time
+    def running(self, prev_out: dict[str, Any]):
+        with logger.tag("ef"):  # evaluate and feedback
+            if prev_out["propose"].action in [KG_ACTION_FEATURE_ENGINEERING, KG_ACTION_FEATURE_PROCESSING]:
+                exp = self.feature_runner.develop(prev_out["coding"])
+            else:
+                exp = self.model_runner.develop(prev_out["coding"])
+            logger.log_object(exp, tag="runner result")
+            if KAGGLE_IMPLEMENT_SETTING.competition in [
+                "optiver-realized-volatility-prediction",
+                "covid19-global-forecasting-week-1",
+            ]:
+                try:
+                    python_files_to_notebook(
+                        KAGGLE_IMPLEMENT_SETTING.competition, exp.experiment_workspace.workspace_path
+                    )
+                except Exception as e:
+                    logger.error(f"Merge python files to one file failed: {e}")
+            if KAGGLE_IMPLEMENT_SETTING.auto_submit:
+                csv_path = exp.experiment_workspace.workspace_path / "submission.csv"
+                try:
+                    subprocess.run(
+                        [
+                            "kaggle",
+                            "competitions",
+                            "submit",
+                            "-f",
+                            str(csv_path.absolute()),
+                            "-m",
+                            str(csv_path.parent.absolute()),
+                            KAGGLE_IMPLEMENT_SETTING.competition,
+                        ],
+                        check=True,
+                    )
+                except subprocess.CalledProcessError as e:
+                    logger.error(f"Auto submission failed: \n{e}")
+                except Exception as e:
+                    logger.error(f"Other exception when use kaggle api:\n{e}")
+
+        return exp
+
+    skip_loop_error = (ModelEmptyError, FactorEmptyError)
+
+
+def main(path=None, step_n=None, competition=None):
+    """
+    Auto R&D Evolving loop for models in a kaggle{} scenario.
+    You can continue running session by
+    .. code-block:: bash
+        dotenv run -- python rdagent/app/kaggle/loop.py [--competition titanic] $LOG_PATH/__session__/1/0_propose  --step_n 1   # `step_n` is a optional parameter
+        rdagent kaggle --competition playground-series-s4e8  # You are encouraged to use this one.
+    """
+    if competition:
+        KAGGLE_IMPLEMENT_SETTING.competition = competition
+        download_data(competition=competition, local_path=KAGGLE_IMPLEMENT_SETTING.local_data_path)
+    else:
+        logger.error("Please specify competition name.")
+    if path is None:
+        kaggle_loop = KaggleRDLoop(KAGGLE_IMPLEMENT_SETTING)
+    else:
+        kaggle_loop = KaggleRDLoop.load(path)
+    kaggle_loop.run(step_n=step_n)
+
+
+if __name__ == "__main__":
+    fire.Fire(main)

rdagent/app/qlib_rd_loop/RDAgent.py

@@ -1,112 +0,0 @@
-import pickle
-
-from rdagent.app.qlib_rd_loop.conf import PROP_SETTING
-from rdagent.core.developer import Developer
-from rdagent.core.exception import ModelEmptyError
-from rdagent.core.proposal import (
-    Hypothesis2Experiment,
-    HypothesisExperiment2Feedback,
-    HypothesisGen,
-    Trace,
-)
-from rdagent.core.scenario import Scenario
-from rdagent.core.utils import import_class
-from rdagent.log import rdagent_logger as logger
-
-
-# TODO: we can design a workflow that can automatically save session and traceback in the future
-class Model_RD_Agent:
-    def __init__(self):
-        self.scen: Scenario = import_class(PROP_SETTING.model_scen)()
-        self.hypothesis_gen: HypothesisGen = import_class(PROP_SETTING.model_hypothesis_gen)(self.scen)
-        self.hypothesis2experiment: Hypothesis2Experiment = import_class(PROP_SETTING.model_hypothesis2experiment)()
-        self.qlib_model_coder: Developer = import_class(PROP_SETTING.model_coder)(self.scen)
-        self.qlib_model_runner: Developer = import_class(PROP_SETTING.model_runner)(self.scen)
-        self.qlib_model_summarizer: HypothesisExperiment2Feedback = import_class(PROP_SETTING.model_summarizer)(
-            self.scen
-        )
-        self.trace = Trace(scen=self.scen)
-
-    def generate_hypothesis(self):
-        hypothesis = self.hypothesis_gen.gen(self.trace)
-        self.dump_objects(hypothesis=hypothesis, trace=self.trace, filename="step_hypothesis.pkl")
-        return hypothesis
-
-    def convert_hypothesis(self, hypothesis):
-        exp = self.hypothesis2experiment.convert(hypothesis, self.trace)
-        self.dump_objects(exp=exp, hypothesis=hypothesis, trace=self.trace, filename="step_experiment.pkl")
-        return exp
-
-    def generate_code(self, exp):
-        exp = self.qlib_model_coder.develop(exp)
-        self.dump_objects(exp=exp, trace=self.trace, filename="step_code.pkl")
-        return exp
-
-    def run_experiment(self, exp):
-        exp = self.qlib_model_runner.develop(exp)
-        self.dump_objects(exp=exp, trace=self.trace, filename="step_run.pkl")
-        return exp
-
-    def generate_feedback(self, exp, hypothesis):
-        feedback = self.qlib_model_summarizer.generate_feedback(exp, hypothesis, self.trace)
-        self.dump_objects(
-            exp=exp, hypothesis=hypothesis, feedback=feedback, trace=self.trace, filename="step_feedback.pkl"
-        )
-        return feedback
-
-    def append_to_trace(self, hypothesis, exp, feedback):
-        self.trace.hist.append((hypothesis, exp, feedback))
-        self.dump_objects(trace=self.trace, filename="step_trace.pkl")
-
-    def dump_objects(self, exp=None, hypothesis=None, feedback=None, trace=None, filename="dumped_objects.pkl"):
-        with open(filename, "wb") as f:
-            pickle.dump((exp, hypothesis, feedback, trace or self.trace), f)
-
-    def load_objects(self, filename):
-        with open(filename, "rb") as f:
-            return pickle.load(f)
-
-
-def process_steps(agent):
-    # Load trace if available
-    try:
-        _, _, _, trace = agent.load_objects("step_trace.pkl")
-        agent.trace = trace
-        print(trace.get_sota_hypothesis_and_experiment())
-    except FileNotFoundError:
-        pass
-
-    # # # Step 1: Generate hypothesis
-    # try:
-    #     _, hypothesis, _, _ = agent.load_objects('step_hypothesis.pkl')
-    # except FileNotFoundError:
-    hypothesis = agent.generate_hypothesis()
-
-    # # # Step 2: Convert hypothesis
-    # try:
-    #     exp, _, _, _ = agent.load_objects('step_experiment.pkl')
-    # except FileNotFoundError:
-    #     exp = agent.convert_hypothesis(hypothesis)
-
-    # # # Step 3: Generate code
-    # try:
-    #     exp, _, _, _ = agent.load_objects('step_code.pkl')
-    # except FileNotFoundError:
-    #     exp = agent.generate_code(exp)
-
-    # # # Step 4: Run experiment
-    # try:
-    #     exp, _, _, _ = agent.load_objects('step_run.pkl')
-    # except FileNotFoundError:
-    #     exp = agent.run_experiment(exp)
-
-    # # Step 5: Generate feedback
-    # feedback = agent.generate_feedback(exp, hypothesis)
-
-    # # Step 6: Append to trace
-    # agent.append_to_trace(hypothesis, exp, feedback)
-
-
-if __name__ == "__main__":
-    agent = Model_RD_Agent()
-    process_steps(agent)

rdagent/app/qlib_rd_loop/conf.py

@@ -1,45 +1,79 @@
+from pydantic_settings import BaseSettings
+
 from rdagent.components.workflow.conf import BasePropSetting
 
 
 class ModelBasePropSetting(BasePropSetting):
     class Config:
-        env_prefix = "QLIB_MODEL_"  # Use MODEL_CODER_ as prefix for environment variables
-        protected_namespaces = ()  # Add 'model_' to the protected namespaces
+        env_prefix = "QLIB_MODEL_"
+        """Use `QLIB_MODEL_` as prefix for environment variables"""
+        protected_namespaces = ()
+        """Add 'model_' to the protected namespaces"""
 
+    # 1) override base settings
     scen: str = "rdagent.scenarios.qlib.experiment.model_experiment.QlibModelScenario"
+    """Scenario class for Qlib Model"""
+
     hypothesis_gen: str = "rdagent.scenarios.qlib.proposal.model_proposal.QlibModelHypothesisGen"
+    """Hypothesis generation class"""
+
     hypothesis2experiment: str = "rdagent.scenarios.qlib.proposal.model_proposal.QlibModelHypothesis2Experiment"
+    """Hypothesis to experiment class"""
+
     coder: str = "rdagent.scenarios.qlib.developer.model_coder.QlibModelCoSTEER"
+    """Coder class"""
+
     runner: str = "rdagent.scenarios.qlib.developer.model_runner.QlibModelRunner"
+    """Runner class"""
+
     summarizer: str = "rdagent.scenarios.qlib.developer.feedback.QlibModelHypothesisExperiment2Feedback"
+    """Summarizer class"""
 
     evolving_n: int = 10
+    """Number of evolutions"""
 
 
 class FactorBasePropSetting(BasePropSetting):
     class Config:
-        env_prefix = "QLIB_FACTOR_"  # Use MODEL_CODER_ as prefix for environment variables
-        protected_namespaces = ()  # Add 'model_' to the protected namespaces
+        env_prefix = "QLIB_FACTOR_"
+        """Use `QLIB_FACTOR_` as prefix for environment variables"""
+        protected_namespaces = ()
+        """Add 'factor_' to the protected namespaces"""
 
     # 1) override base settings
-    # TODO: model part is not finished yet
     scen: str = "rdagent.scenarios.qlib.experiment.factor_experiment.QlibFactorScenario"
+    """Scenario class for Qlib Factor"""
+
     hypothesis_gen: str = "rdagent.scenarios.qlib.proposal.factor_proposal.QlibFactorHypothesisGen"
+    """Hypothesis generation class"""
+
     hypothesis2experiment: str = "rdagent.scenarios.qlib.proposal.factor_proposal.QlibFactorHypothesis2Experiment"
+    """Hypothesis to experiment class"""
+
     coder: str = "rdagent.scenarios.qlib.developer.factor_coder.QlibFactorCoSTEER"
+    """Coder class"""
+
     runner: str = "rdagent.scenarios.qlib.developer.factor_runner.QlibFactorRunner"
+    """Runner class"""
+
     summarizer: str = "rdagent.scenarios.qlib.developer.feedback.QlibFactorHypothesisExperiment2Feedback"
+    """Summarizer class"""
 
     evolving_n: int = 10
-
-    # 2) sub task specific:
-    report_result_json_file_path: str = "git_ignore_folder/report_list.json"
-    max_factors_per_exp: int = 10000
+    """Number of evolutions"""
 
 
 class FactorFromReportPropSetting(FactorBasePropSetting):
-    # Override the scen attribute
+    # 1) override the scen attribute
     scen: str = "rdagent.scenarios.qlib.experiment.factor_from_report_experiment.QlibFactorFromReportScenario"
+    """Scenario class for Qlib Factor from Report"""
+
+    # 2) sub task specific:
+    report_result_json_file_path: str = "git_ignore_folder/report_list.json"
+    """Path to the JSON file listing research reports for factor extraction"""
+
+    max_factors_per_exp: int = 10000
+    """Maximum number of factors implemented per experiment"""
 
 
 FACTOR_PROP_SETTING = FactorBasePropSetting()

rdagent/app/qlib_rd_loop/factor.py

@@ -10,11 +10,13 @@ from rdagent.app.qlib_rd_loop.conf import FACTOR_PROP_SETTING
 from rdagent.components.workflow.rd_loop import RDLoop
 from rdagent.core.exception import FactorEmptyError
 from rdagent.log import rdagent_logger as logger
+from rdagent.log.time import measure_time
 
 
 class FactorRDLoop(RDLoop):
     skip_loop_error = (FactorEmptyError,)
 
+    @measure_time
     def running(self, prev_out: dict[str, Any]):
         with logger.tag("ef"):  # evaluate and feedback
             exp = self.runner.develop(prev_out["coding"])
@@ -27,11 +29,13 @@ class FactorRDLoop(RDLoop):
 
 def main(path=None, step_n=None):
     """
+    Auto R&D Evolving loop for fintech factors.
+
     You can continue running session by
 
     .. code-block:: python
 
-        dotenv run -- python rdagent/app/qlib_rd_loop/factor_w_sc.py $LOG_PATH/__session__/1/0_propose  --step_n 1   # `step_n` is a optional paramter
+        dotenv run -- python rdagent/app/qlib_rd_loop/factor.py $LOG_PATH/__session__/1/0_propose  --step_n 1   # `step_n` is a optional paramter
 
     """
     if path is None:

rdagent/app/qlib_rd_loop/factor_from_report.py

@@ -6,16 +6,15 @@ import fire
 from jinja2 import Environment, StrictUndefined
 
 from rdagent.app.qlib_rd_loop.conf import FACTOR_FROM_REPORT_PROP_SETTING
-from rdagent.app.qlib_rd_loop.factor_w_sc import FactorRDLoop
+from rdagent.app.qlib_rd_loop.factor import FactorRDLoop
 from rdagent.components.document_reader.document_reader import (
     extract_first_page_screenshot_from_pdf,
     load_and_process_pdfs_by_langchain,
 )
-from rdagent.components.workflow.rd_loop import RDLoop
-from rdagent.core.exception import FactorEmptyError
 from rdagent.core.prompts import Prompts
 from rdagent.core.proposal import Hypothesis
 from rdagent.log import rdagent_logger as logger
+from rdagent.log.time import measure_time
 from rdagent.oai.llm_utils import APIBackend
 from rdagent.scenarios.qlib.experiment.factor_experiment import QlibFactorExperiment
 from rdagent.scenarios.qlib.factor_experiment_loader.pdf_loader import (
@@ -85,7 +84,7 @@ def extract_hypothesis_and_exp_from_reports(report_file_path: str) -> Tuple[Qlib
             pdf_screenshot = extract_first_page_screenshot_from_pdf(report_file_path)
             logger.log_object(pdf_screenshot)
 
-    docs_dict = load_and_process_pdfs_by_langchain(Path(report_file_path))
+    docs_dict = load_and_process_pdfs_by_langchain(report_file_path)
 
     factor_result = {
         task.factor_name: {
@@ -103,15 +102,23 @@ def extract_hypothesis_and_exp_from_reports(report_file_path: str) -> Tuple[Qlib
 
 
 class FactorReportLoop(FactorRDLoop, metaclass=LoopMeta):
-    def __init__(self, PROP_SETTING: FACTOR_FROM_REPORT_PROP_SETTING):
-        super().__init__(PROP_SETTING=PROP_SETTING)
-        self.judge_pdf_data_items = json.load(open(PROP_SETTING.report_result_json_file_path, "r"))
+    @measure_time
+    def __init__(self, report_folder: str = None):
+        super().__init__(PROP_SETTING=FACTOR_FROM_REPORT_PROP_SETTING)
+        if report_folder is None:
+            self.judge_pdf_data_items = json.load(
+                open(FACTOR_FROM_REPORT_PROP_SETTING.report_result_json_file_path, "r")
+            )
+        else:
+            self.judge_pdf_data_items = [i for i in Path(report_folder).rglob("*.pdf")]
+
         self.pdf_file_index = 0
         self.valid_pdf_file_count = 0
         self.current_loop_hypothesis = None
         self.current_loop_exp = None
         self.steps = ["propose_hypo_exp", "propose", "exp_gen", "coding", "running", "feedback"]
 
+    @measure_time
     def propose_hypo_exp(self, prev_out: dict[str, Any]):
         with logger.tag("r"):
             while True:
@@ -133,26 +140,31 @@ class FactorReportLoop(FactorRDLoop, metaclass=LoopMeta):
                 self.current_loop_exp = exp
                 return None
 
+    @measure_time
     def propose(self, prev_out: dict[str, Any]):
         return self.current_loop_hypothesis
 
+    @measure_time
     def exp_gen(self, prev_out: dict[str, Any]):
         return self.current_loop_exp
 
 
-def main(path=None, step_n=None):
+def main(report_folder=None, path=None, step_n=None):
     """
-    You can continue running session by
-
-    .. code-block:: python
-
-        dotenv run -- python rdagent/app/qlib_rd_loop/factor_from_report_w_sc.py $LOG_PATH/__session__/1/0_propose  --step_n 1   # `step_n` is a optional parameter
+    Auto R&D Evolving loop for fintech factors (the factors are extracted from finance reports).
 
+    Args:
+        report_folder (str, optional): The folder contains the report PDF files. Reports will be loaded from this folder.
+        path (str, optional): The path for loading a session. If provided, the session will be loaded.
+        step_n (int, optional): Step number to continue running a session.
     """
-    if path is None:
-        model_loop = FactorReportLoop(FACTOR_FROM_REPORT_PROP_SETTING)
-    else:
+    if path is None and report_folder is None:
+        model_loop = FactorReportLoop()
+    elif path is not None:
         model_loop = FactorReportLoop.load(path)
+    else:
+        model_loop = FactorReportLoop(report_folder=report_folder)
+
     model_loop.run(step_n=step_n)
 
 

rdagent/app/qlib_rd_loop/model.py

@@ -15,11 +15,13 @@ class ModelRDLoop(RDLoop):
 
 def main(path=None, step_n=None):
     """
+    Auto R&D Evolving loop for fintech models
+
     You can continue running session by
 
     .. code-block:: python
 
-        dotenv run -- python rdagent/app/qlib_rd_loop/model_w_sc.py $LOG_PATH/__session__/1/0_propose  --step_n 1   # `step_n` is a optional paramter
+        dotenv run -- python rdagent/app/qlib_rd_loop/model.py $LOG_PATH/__session__/1/0_propose  --step_n 1   # `step_n` is a optional paramter
 
     """
     if path is None:

rdagent/app/utils/info.py

@@ -0,0 +1,86 @@
+import importlib.metadata
+import platform
+import sys
+from pathlib import Path
+
+import docker
+import requests
+from setuptools_scm import get_version
+
+from rdagent.log import rdagent_logger as logger
+
+
+def sys_info():
+    """collect system related info"""
+    method_list = [
+        ["Name of current operating system: ", "system"],
+        ["Processor architecture: ", "machine"],
+        ["System, version, and hardware information: ", "platform"],
+        ["Version number of the system: ", "version"],
+    ]
+    for method in method_list:
+        logger.info(f"{method[0]}{getattr(platform, method[1])()}")
+    return None
+
+
+def python_info():
+    """collect Python related info"""
+    python_version = sys.version.replace("\n", " ")
+    logger.info(f"Python version: {python_version}")
+    return None
+
+
+def docker_info():
+    client = docker.from_env()
+    containers = client.containers.list(all=True)
+    if containers:
+        containers.sort(key=lambda c: c.attrs["Created"])
+        last_container = containers[-1]
+        logger.info(f"Container ID: {last_container.id}")
+        logger.info(f"Container Name: {last_container.name}")
+        logger.info(f"Container Status: {last_container.status}")
+        logger.info(f"Image ID used by the container: {last_container.image.id}")
+        logger.info(f"Image tag used by the container: {last_container.image.tags}")
+        logger.info(f"Container port mapping: {last_container.ports}")
+        logger.info(f"Container Label: {last_container.labels}")
+        logger.info(f"Startup Commands: {' '.join(client.containers.get(last_container.id).attrs['Config']['Cmd'])}")
+    else:
+        logger.info(f"No run containers.")
+
+
+def rdagent_info():
+    """collect rdagent related info"""
+    current_version = importlib.metadata.version("rdagent")
+    logger.info(f"RD-Agent version: {current_version}")
+    api_url = f"https://api.github.com/repos/microsoft/RD-Agent/contents/requirements.txt?ref=main"
+    response = requests.get(api_url)
+    if response.status_code == 200:
+        files = response.json()
+        file_url = files["download_url"]
+        file_response = requests.get(file_url)
+        if file_response.status_code == 200:
+            all_file_contents = file_response.text.split("\n")
+        else:
+            logger.warning(f"Failed to retrieve {files['name']}, status code: {file_response.status_code}")
+    else:
+        logger.warning(f"Failed to retrieve files in folder, status code: {response.status_code}")
+    package_list = [
+        item.split("#")[0].strip() for item in all_file_contents if item.strip() and not item.startswith("#")
+    ]
+    package_version_list = []
+    for package in package_list:
+        if package == "typer[all]":
+            package = "typer"
+        version = importlib.metadata.version(package)
+        package_version_list.append(f"{package}=={version}")
+    logger.info(f"Package version: {package_version_list}")
+    return None
+
+
+def collect_info():
+    """Prints information about the system and the installed packages."""
+    sys_info()
+    python_info()
+    docker_info()
+    rdagent_info()
+    return None

rdagent/components/benchmark/conf.py

@@ -2,13 +2,8 @@ from dataclasses import field
 from pathlib import Path
 from typing import Optional
 
-from dotenv import load_dotenv
 from pydantic_settings import BaseSettings
 
-# Load environment variables
-load_dotenv(verbose=True, override=True)
-
-
 DIRNAME = Path("./")
 
 

rdagent/components/benchmark/eval_method.py

@@ -8,11 +8,9 @@ from tqdm import tqdm
 from rdagent.components.coder.factor_coder.config import FACTOR_IMPLEMENT_SETTINGS
 from rdagent.components.coder.factor_coder.CoSTEER.evaluators import (
     FactorCorrelationEvaluator,
-    FactorEqualValueCountEvaluator,
+    FactorEqualValueRatioEvaluator,
     FactorEvaluator,
     FactorIndexEvaluator,
-    FactorMissingValuesEvaluator,
-    FactorOutputFormatEvaluator,
     FactorRowCountEvaluator,
     FactorSingleColumnEvaluator,
 )
@@ -20,7 +18,7 @@ from rdagent.components.coder.factor_coder.factor import FactorFBWorkspace
 from rdagent.core.conf import RD_AGENT_SETTINGS
 from rdagent.core.developer import Developer
 from rdagent.core.exception import CoderError
-from rdagent.core.experiment import Task, Workspace
+from rdagent.core.experiment import Experiment, Task, Workspace
 from rdagent.core.scenario import Scenario
 from rdagent.core.utils import multiprocessing_wrapper
 
@@ -33,11 +31,34 @@ EVAL_RES = Dict[
 class TestCase:
     def __init__(
         self,
-        target_task: list[Task] = [],
-        ground_truth: list[Workspace] = [],
+        target_task: Task,
+        ground_truth: Workspace,
     ):
-        self.ground_truth = ground_truth
         self.target_task = target_task
+        self.ground_truth = ground_truth
+
+
+class TestCases:
+    def __init__(self, test_case_l: list[TestCase] = []):
+        # self.test_case_l = [TestCase(task, gt) for task, gt in zip(target_task, ground_truth)]
+        self.test_case_l = test_case_l
+
+    def __getitem__(self, item):
+        return self.test_case_l[item]
+
+    def __len__(self):
+        return len(self.test_case_l)
+
+    def get_exp(self):
+        return Experiment([case.target_task for case in self.test_case_l])
+
+    @property
+    def target_task(self):
+        return [case.target_task for case in self.test_case_l]
+
+    @property
+    def ground_truth(self):
+        return [case.ground_truth for case in self.test_case_l]
 
 
 class BaseEval:
@@ -48,13 +69,13 @@ class BaseEval:
     def __init__(
         self,
         evaluator_l: List[FactorEvaluator],
-        test_cases: List[TestCase],
+        test_cases: TestCases,
         generate_method: Developer,
         catch_eval_except: bool = True,
     ):
         """Parameters
         ----------
-        test_cases : List[TestCase]
+        test_cases : TestCases
             cases to be evaluated, ground truth are included in the test cases.
         evaluator_l : List[FactorEvaluator]
             A list of evaluators to evaluate the generated code.
@@ -102,6 +123,7 @@ class BaseEval:
         eval_res = []
         for ev in self.evaluator_l:
             try:
+                case_gen.raise_exception = True
                 eval_res.append((ev, ev.evaluate(implementation=case_gen, gt_implementation=case_gt)))
                 # if the corr ev is successfully evaluated and achieve the best performance, then break
             except CoderError as e:
@@ -118,7 +140,7 @@ class BaseEval:
 class FactorImplementEval(BaseEval):
     def __init__(
         self,
-        test_cases: TestCase,
+        test_cases: TestCases,
         method: Developer,
         *args,
         scen: Scenario,
@@ -127,26 +149,22 @@ class FactorImplementEval(BaseEval):
     ):
         online_evaluator_l = [
             FactorSingleColumnEvaluator(scen),
-            FactorOutputFormatEvaluator(scen),
             FactorRowCountEvaluator(scen),
             FactorIndexEvaluator(scen),
-            FactorMissingValuesEvaluator(scen),
-            FactorEqualValueCountEvaluator(scen),
+            FactorEqualValueRatioEvaluator(scen),
             FactorCorrelationEvaluator(hard_check=False, scen=scen),
         ]
         super().__init__(online_evaluator_l, test_cases, method, *args, **kwargs)
         self.test_round = test_round
 
-    def eval(self):
+    def develop(self):
         gen_factor_l_all_rounds = []
-        test_cases_all_rounds = []
-        res = defaultdict(list)
         for _ in tqdm(range(self.test_round), desc="Rounds of Eval"):
             print("\n========================================================")
             print(f"Eval {_}-th times...")
             print("========================================================\n")
             try:
-                gen_factor_l = self.generate_method.develop(self.test_cases.target_task)
+                gen_factor_l = self.generate_method.develop(self.test_cases.get_exp())
             except KeyboardInterrupt:
                 # TODO: Why still need to save result after KeyboardInterrupt?
                 print("Manually interrupted the evaluation. Saving existing results")
@@ -157,8 +175,14 @@ class FactorImplementEval(BaseEval):
                     "The number of cases to eval should be equal to the number of test cases.",
                 )
             gen_factor_l_all_rounds.extend(gen_factor_l.sub_workspace_list)
-            test_cases_all_rounds.extend(self.test_cases.ground_truth)
 
+        return gen_factor_l_all_rounds
+
+    def eval(self, gen_factor_l_all_rounds):
+        test_cases_all_rounds = []
+        res = defaultdict(list)
+        for _ in range(self.test_round):
+            test_cases_all_rounds.extend(self.test_cases.ground_truth)
         eval_res_list = multiprocessing_wrapper(
             [
                 (self.eval_case, (gt_case, gen_factor))

rdagent/components/coder/factor_coder/CoSTEER/__init__.py

@@ -88,7 +88,7 @@ class FactorCoSTEER(Developer[FactorExperiment]):
         self.rag = FactorGraphRAGStrategy(factor_knowledge_base)
 
         # init intermediate items
-        factor_experiment = FactorEvolvingItem(sub_tasks=exp.sub_tasks)
+        factor_experiment = FactorEvolvingItem.from_experiment(exp)
 
         self.evolve_agent = FactorRAGEvoAgent(
             max_loop=self.max_loop,

rdagent/components/coder/factor_coder/CoSTEER/evaluators.py

@@ -19,6 +19,7 @@ from rdagent.core.experiment import Task, Workspace
 from rdagent.core.prompts import Prompts
 from rdagent.core.utils import multiprocessing_wrapper
 from rdagent.log import rdagent_logger as logger
+from rdagent.oai.llm_conf import LLM_SETTINGS
 from rdagent.oai.llm_utils import APIBackend
 
 evaluate_prompts = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")
@@ -89,7 +90,13 @@ class FactorCodeEvaluator(FactorEvaluator):
         system_prompt = (
             Environment(undefined=StrictUndefined)
             .from_string(evaluate_prompts["evaluator_code_feedback_v1_system"])
-            .render(scenario=self.scen.get_scenario_all_desc() if self.scen is not None else "No scenario description.")
+            .render(
+                scenario=(
+                    self.scen.get_scenario_all_desc(target_task)
+                    if self.scen is not None
+                    else "No scenario description."
+                )
+            )
         )
 
         execution_feedback_to_render = execution_feedback
@@ -112,7 +119,7 @@ class FactorCodeEvaluator(FactorEvaluator):
                     user_prompt=user_prompt,
                     system_prompt=system_prompt,
                 )
-                > RD_AGENT_SETTINGS.chat_token_limit
+                > LLM_SETTINGS.chat_token_limit
             ):
                 execution_feedback_to_render = execution_feedback_to_render[len(execution_feedback_to_render) // 2 :]
             else:
@@ -126,6 +133,28 @@ class FactorCodeEvaluator(FactorEvaluator):
         return critic_response, None
 
 
+class FactorInfEvaluator(FactorEvaluator):
+    def evaluate(
+        self,
+        implementation: Workspace,
+        gt_implementation: Workspace,
+    ) -> Tuple[str, object]:
+        _, gen_df = self._get_df(gt_implementation, implementation)
+        if gen_df is None:
+            return (
+                "The source dataframe is None. Please check the implementation.",
+                False,
+            )
+        INF_count = gen_df.isin([float("inf"), -float("inf")]).sum().sum()
+        if INF_count == 0:
+            return "The source dataframe does not have any infinite values.", True
+        else:
+            return (
+                f"The source dataframe has {INF_count} infinite values. Please check the implementation.",
+                False,
+            )
+
+
 class FactorSingleColumnEvaluator(FactorEvaluator):
     def evaluate(
         self,
@@ -133,7 +162,11 @@ class FactorSingleColumnEvaluator(FactorEvaluator):
         gt_implementation: Workspace,
     ) -> Tuple[str, object]:
         _, gen_df = self._get_df(gt_implementation, implementation)
-
+        if gen_df is None:
+            return (
+                "The source dataframe is None. Please check the implementation.",
+                False,
+            )
         if len(gen_df.columns) == 1:
             return "The source dataframe has only one column which is correct.", True
         else:
@@ -157,13 +190,19 @@ class FactorOutputFormatEvaluator(FactorEvaluator):
             )
         buffer = io.StringIO()
         gen_df.info(buf=buffer)
-        gen_df_info_str = buffer.getvalue()
+        gen_df_info_str = f"The use is currently working on a feature related task.\nThe output dataframe info is:\n{buffer.getvalue()}"
         system_prompt = (
             Environment(undefined=StrictUndefined)
             .from_string(
                 evaluate_prompts["evaluator_output_format_system"],
             )
-            .render(scenario=self.scen.get_scenario_all_desc() if self.scen is not None else "No scenario description.")
+            .render(
+                scenario=(
+                    self.scen.get_scenario_all_desc(implementation.target_task)
+                    if self.scen is not None
+                    else "No scenario description."
+                )
+            )
         )
 
         # TODO: with retry_context(retry_n=3, except_list=[KeyError]):
@@ -173,18 +212,12 @@ class FactorOutputFormatEvaluator(FactorEvaluator):
 
         while attempts < max_attempts:
             try:
-                resp = APIBackend().build_messages_and_create_chat_completion(
+                api = APIBackend() if attempts == 0 else APIBackend(use_chat_cache=False)
+                resp = api.build_messages_and_create_chat_completion(
                     user_prompt=gen_df_info_str, system_prompt=system_prompt, json_mode=True
                 )
                 resp_dict = json.loads(resp)
-
-                if isinstance(resp_dict["output_format_decision"], str) and resp_dict[
-                    "output_format_decision"
-                ].lower() in (
-                    "true",
-                    "false",
-                ):
-                    resp_dict["output_format_decision"] = bool(resp_dict["output_format_decision"])
+                resp_dict["output_format_decision"] = str(resp_dict["output_format_decision"]).lower() in ["true", "1"]
 
                 return (
                     resp_dict["output_format_feedback"],
@@ -221,11 +254,11 @@ class FactorDatetimeDailyEvaluator(FactorEvaluator):
             pd.to_datetime(gen_df.index.get_level_values("datetime"))
         except Exception:
             return (
-                "The source dataframe has a datetime index but it is not in the correct format (maybe a regular string or other objects). Please check the implementation.",
+                f"The source dataframe has a datetime index but it is not in the correct format (maybe a regular string or other objects). Please check the implementation.\n The head of the output dataframe is: \n{gen_df.head()}",
                 False,
             )
 
-        time_diff = gen_df.index.get_level_values("datetime").to_series().diff().dropna().unique()
+        time_diff = pd.to_datetime(gen_df.index.get_level_values("datetime")).to_series().diff().dropna().unique()
         if pd.Timedelta(minutes=1) in time_diff:
             return (
                 "The generated dataframe is not daily. The implementation is definitely wrong. Please check the implementation.",
@@ -241,14 +274,21 @@ class FactorRowCountEvaluator(FactorEvaluator):
         gt_implementation: Workspace,
     ) -> Tuple[str, object]:
         gt_df, gen_df = self._get_df(gt_implementation, implementation)
-
-        if gen_df.shape[0] == gt_df.shape[0]:
-            return "Both dataframes have the same rows count.", True
-        else:
+        if gen_df is None:
             return (
-                f"The source dataframe and the ground truth dataframe have different rows count. The source dataframe has {gen_df.shape[0]} rows, while the ground truth dataframe has {gt_df.shape[0]} rows. Please check the implementation.",
+                "The source dataframe is None. Please check the implementation.",
                 False,
             )
+        ratio = min(len(gen_df), len(gt_df)) / max(len(gen_df), len(gt_df))
+        return (
+            (
+                f"The ratio of rows count in the source dataframe to the ground truth dataframe is {ratio:.2f}. "
+                + "Please verify the implementation. "
+                if ratio <= 0.99
+                else ""
+            ),
+            ratio,
+        )
 
 
 class FactorIndexEvaluator(FactorEvaluator):
@@ -258,14 +298,22 @@ class FactorIndexEvaluator(FactorEvaluator):
         gt_implementation: Workspace,
     ) -> Tuple[str, object]:
         gt_df, gen_df = self._get_df(gt_implementation, implementation)
-
-        if gen_df.index.equals(gt_df.index):
-            return "Both dataframes have the same index.", True
-        else:
+        if gen_df is None:
             return (
-                "The source dataframe and the ground truth dataframe have different index. Please check the implementation.",
+                "The source dataframe is None. Please check the implementation.",
                 False,
             )
+        gen_index_set, gt_index_set = set(gen_df.index), set(gt_df.index)
+        similarity = len(gen_index_set.intersection(gt_index_set)) / len(gen_index_set.union(gt_index_set))
+        return (
+            (
+                f"The source dataframe and the ground truth dataframe have different index with a similarity of {similarity:.2%}. The similarity is calculated by the number of shared indices divided by the union indices. "
+                + "Please check the implementation."
+                if similarity <= 0.99
+                else ""
+            ),
+            similarity,
+        )
 
 
 class FactorMissingValuesEvaluator(FactorEvaluator):
@@ -275,7 +323,11 @@ class FactorMissingValuesEvaluator(FactorEvaluator):
         gt_implementation: Workspace,
     ) -> Tuple[str, object]:
         gt_df, gen_df = self._get_df(gt_implementation, implementation)
-
+        if gen_df is None:
+            return (
+                "The source dataframe is None. Please check the implementation.",
+                False,
+            )
         if gen_df.isna().sum().sum() == gt_df.isna().sum().sum():
             return "Both dataframes have the same missing values.", True
         else:
@@ -285,14 +337,18 @@ class FactorMissingValuesEvaluator(FactorEvaluator):
             )
 
 
-class FactorEqualValueCountEvaluator(FactorEvaluator):
+class FactorEqualValueRatioEvaluator(FactorEvaluator):
     def evaluate(
         self,
         implementation: Workspace,
         gt_implementation: Workspace,
     ) -> Tuple[str, object]:
         gt_df, gen_df = self._get_df(gt_implementation, implementation)
-
+        if gen_df is None:
+            return (
+                "The source dataframe is None. Please check the implementation.",
+                -1,
+            )
         try:
             close_values = gen_df.sub(gt_df).abs().lt(1e-6)
             result_int = close_values.astype(int)
@@ -323,7 +379,11 @@ class FactorCorrelationEvaluator(FactorEvaluator):
         gt_implementation: Workspace,
     ) -> Tuple[str, object]:
         gt_df, gen_df = self._get_df(gt_implementation, implementation)
-
+        if gen_df is None:
+            return (
+                "The source dataframe is None. Please check the implementation.",
+                False,
+            )
         concat_df = pd.concat([gen_df, gt_df], axis=1)
         concat_df.columns = ["source", "gt"]
         ic = concat_df.groupby("datetime").apply(lambda df: df["source"].corr(df["gt"])).dropna().mean()
@@ -354,40 +414,51 @@ class FactorValueEvaluator(FactorEvaluator):
         self,
         implementation: Workspace,
         gt_implementation: Workspace,
+        version: int = 1,  # 1 for qlib factors and 2 for kaggle factors
         **kwargs,
     ) -> Tuple:
         conclusions = []
 
         # Initialize result variables
-        single_column_result = None
-        same_index_result = None
+        row_result = 0
+        index_result = 0
         output_format_result = None
         equal_value_ratio_result = 0
         high_correlation_result = False
+        row_result = None
 
-        # Check if both dataframe has only one columns
-        feedback_str, _ = FactorSingleColumnEvaluator(self.scen).evaluate(implementation, gt_implementation)
+        # Check if both dataframe has only one columns Mute this since factor task might generate more than one columns now
+        if version == 1:
+            feedback_str, _ = FactorSingleColumnEvaluator(self.scen).evaluate(implementation, gt_implementation)
+            conclusions.append(feedback_str)
+        elif version == 2:
+            input_shape = self.scen.input_shape
+            _, gen_df = self._get_df(gt_implementation, implementation)
+            if gen_df.shape[-1] > input_shape[-1]:
+                conclusions.append(
+                    "Output dataframe has more columns than input feature which is not acceptable in feature processing tasks. Please check the implementation to avoid generating too many columns. Consider this implementation as a failure."
+                )
+
+        feedback_str, inf_evaluate_res = FactorInfEvaluator(self.scen).evaluate(implementation, gt_implementation)
         conclusions.append(feedback_str)
 
         # Check if the index of the dataframe is ("datetime", "instrument")
         feedback_str, _ = FactorOutputFormatEvaluator(self.scen).evaluate(implementation, gt_implementation)
         conclusions.append(feedback_str)
-
-        feedback_str, daily_check_result = FactorDatetimeDailyEvaluator(self.scen).evaluate(
-            implementation, gt_implementation
-        )
-        conclusions.append(feedback_str)
-
-        # Check if both dataframe have the same rows count
-        if gt_implementation is not None:
-            feedback_str, single_column_result = FactorRowCountEvaluator(self.scen).evaluate(
+        if version == 1:
+            feedback_str, daily_check_result = FactorDatetimeDailyEvaluator(self.scen).evaluate(
                 implementation, gt_implementation
             )
             conclusions.append(feedback_str)
+        else:
+            daily_check_result = None
 
-            feedback_str, same_index_result = FactorIndexEvaluator(self.scen).evaluate(
-                implementation, gt_implementation
-            )
+        # Check dataframe format
+        if gt_implementation is not None:
+            feedback_str, row_result = FactorRowCountEvaluator(self.scen).evaluate(implementation, gt_implementation)
+            conclusions.append(feedback_str)
+
+            feedback_str, index_result = FactorIndexEvaluator(self.scen).evaluate(implementation, gt_implementation)
             conclusions.append(feedback_str)
 
             feedback_str, output_format_result = FactorMissingValuesEvaluator(self.scen).evaluate(
@@ -395,12 +466,12 @@ class FactorValueEvaluator(FactorEvaluator):
             )
             conclusions.append(feedback_str)
 
-            feedback_str, equal_value_ratio_result = FactorEqualValueCountEvaluator(self.scen).evaluate(
+            feedback_str, equal_value_ratio_result = FactorEqualValueRatioEvaluator(self.scen).evaluate(
                 implementation, gt_implementation
             )
             conclusions.append(feedback_str)
 
-            if same_index_result:
+            if index_result > 0.99:
                 feedback_str, high_correlation_result = FactorCorrelationEvaluator(
                     hard_check=True, scen=self.scen
                 ).evaluate(implementation, gt_implementation)
@@ -414,7 +485,13 @@ class FactorValueEvaluator(FactorEvaluator):
 
         if gt_implementation is not None and (equal_value_ratio_result > 0.99) or high_correlation_result:
             decision_from_value_check = True
-        elif single_column_result is False or output_format_result is False or daily_check_result is False:
+        elif (
+            row_result is not None
+            and row_result <= 0.99
+            or output_format_result is False
+            or daily_check_result is False
+            or inf_evaluate_res is False
+        ):
             decision_from_value_check = False
         else:
             decision_from_value_check = None
@@ -433,7 +510,13 @@ class FactorFinalDecisionEvaluator(Evaluator):
         system_prompt = (
             Environment(undefined=StrictUndefined)
             .from_string(evaluate_prompts["evaluator_final_decision_v1_system"])
-            .render(scenario=self.scen.get_scenario_all_desc() if self.scen is not None else "No scenario description.")
+            .render(
+                scenario=(
+                    self.scen.get_scenario_all_desc(target_task)
+                    if self.scen is not None
+                    else "No scenario description."
+                )
+            )
         )
         execution_feedback_to_render = execution_feedback
 
@@ -459,7 +542,7 @@ class FactorFinalDecisionEvaluator(Evaluator):
                     user_prompt=user_prompt,
                     system_prompt=system_prompt,
                 )
-                > RD_AGENT_SETTINGS.chat_token_limit
+                > LLM_SETTINGS.chat_token_limit
             ):
                 execution_feedback_to_render = execution_feedback_to_render[len(execution_feedback_to_render) // 2 :]
             else:
@@ -472,19 +555,19 @@ class FactorFinalDecisionEvaluator(Evaluator):
 
         while attempts < max_attempts:
             try:
+                api = APIBackend() if attempts == 0 else APIBackend(use_chat_cache=False)
                 final_evaluation_dict = json.loads(
-                    APIBackend().build_messages_and_create_chat_completion(
+                    api.build_messages_and_create_chat_completion(
                         user_prompt=user_prompt,
                         system_prompt=system_prompt,
                         json_mode=True,
+                        seed=attempts,  # in case of useless retrying when cache enabled.
                     ),
                 )
                 final_decision = final_evaluation_dict["final_decision"]
                 final_feedback = final_evaluation_dict["final_feedback"]
 
-                if isinstance(final_decision, str) and final_decision.lower() in ("true", "false"):
-                    final_decision = bool(final_decision)
-
+                final_decision = str(final_decision).lower() in ["true", "1"]
                 return final_decision, final_feedback
 
             except json.JSONDecodeError as e:
@@ -603,7 +686,9 @@ class FactorEvaluatorForCoder(FactorEvaluator):
                 (
                     factor_feedback.factor_value_feedback,
                     decision_from_value_check,
-                ) = self.value_evaluator.evaluate(implementation=implementation, gt_implementation=gt_implementation)
+                ) = self.value_evaluator.evaluate(
+                    implementation=implementation, gt_implementation=gt_implementation, version=target_task.version
+                )
 
             factor_feedback.final_decision_based_on_gt = gt_implementation is not None
 
@@ -613,8 +698,12 @@ class FactorEvaluatorForCoder(FactorEvaluator):
                 factor_feedback.final_decision = decision_from_value_check
                 factor_feedback.final_feedback = "Value evaluation passed, skip final decision evaluation."
             elif decision_from_value_check is not None and decision_from_value_check is False:
-                factor_feedback.code_feedback = (
-                    "Final decision is False because value evaluation gets a confident rejection to the result."
+                factor_feedback.code_feedback, _ = self.code_evaluator.evaluate(
+                    target_task=target_task,
+                    implementation=implementation,
+                    execution_feedback=factor_feedback.execution_feedback,
+                    factor_value_feedback=factor_feedback.factor_value_feedback,
+                    gt_implementation=gt_implementation,
                 )
                 factor_feedback.final_decision = decision_from_value_check
                 factor_feedback.final_feedback = "Value evaluation failed, skip final decision evaluation."
@@ -623,7 +712,7 @@ class FactorEvaluatorForCoder(FactorEvaluator):
                     target_task=target_task,
                     implementation=implementation,
                     execution_feedback=factor_feedback.execution_feedback,
-                    value_feedback=factor_feedback.factor_value_feedback,
+                    factor_value_feedback=factor_feedback.factor_value_feedback,
                     gt_implementation=gt_implementation,
                 )
                 (

rdagent/components/coder/factor_coder/CoSTEER/evolvable_subjects.py

@@ -28,3 +28,10 @@ class FactorEvolvingItem(FactorExperiment, EvolvableSubjects):
             )
         else:
             self.sub_gt_implementations = sub_gt_implementations
+
+    @classmethod
+    def from_experiment(cls, exp: FactorExperiment) -> "FactorExperiment":
+        ei = cls(sub_tasks=exp.sub_tasks)
+        ei.based_experiments = exp.based_experiments
+        ei.experiment_workspace = exp.experiment_workspace
+        return ei

rdagent/components/coder/factor_coder/CoSTEER/evolving_strategy.py

@@ -22,6 +22,7 @@ from rdagent.core.evolving_framework import EvolvingStrategy, QueriedKnowledge
 from rdagent.core.experiment import Workspace
 from rdagent.core.prompts import Prompts
 from rdagent.core.utils import multiprocessing_wrapper
+from rdagent.oai.llm_conf import LLM_SETTINGS
 from rdagent.oai.llm_utils import APIBackend
 
 if TYPE_CHECKING:
@@ -90,6 +91,7 @@ class MultiProcessEvolvingStrategy(EvolvingStrategy):
             ],
             n=RD_AGENT_SETTINGS.multi_proc_n,
         )
+        from rdagent.components.coder.factor_coder.factor import FactorFBWorkspace
 
         for index, target_index in enumerate(to_be_finished_task_index):
             if evo.sub_workspace_list[target_index] is None:
@@ -133,7 +135,7 @@ class FactorEvolvingStrategy(MultiProcessEvolvingStrategy):
                     implement_prompts["evolving_strategy_factor_implementation_v1_system"],
                 )
                 .render(
-                    scenario=self.scen.get_scenario_all_desc(),
+                    scenario=self.scen.get_scenario_all_desc(target_task),
                     queried_former_failed_knowledge=queried_former_failed_knowledge_to_render,
                 )
             )
@@ -159,7 +161,7 @@ class FactorEvolvingStrategy(MultiProcessEvolvingStrategy):
                     session.build_chat_completion_message_and_calculate_token(
                         user_prompt,
                     )
-                    < RD_AGENT_SETTINGS.chat_token_limit
+                    < LLM_SETTINGS.chat_token_limit
                 ):
                     break
                 elif len(queried_former_failed_knowledge_to_render) > 1:
@@ -215,18 +217,24 @@ class FactorEvolvingStrategyWithGraph(MultiProcessEvolvingStrategy):
             )  # A dict, {{error_type:[[error_imp_knowledge, success_imp_knowledge],...]},...}
 
             queried_former_failed_knowledge = (
-                queried_knowledge.former_traces[target_factor_task_information] if queried_knowledge is not None else []
+                queried_knowledge.former_traces[target_factor_task_information][0]
+                if queried_knowledge is not None
+                else []
             )
 
             queried_former_failed_knowledge_to_render = queried_former_failed_knowledge
 
+            latest_attempt_to_latest_successful_execution = queried_knowledge.former_traces[
+                target_factor_task_information
+            ][1]
+
             system_prompt = (
                 Environment(undefined=StrictUndefined)
                 .from_string(
                     implement_prompts["evolving_strategy_factor_implementation_v1_system"],
                 )
                 .render(
-                    scenario=self.scen.get_scenario_all_desc(),
+                    scenario=self.scen.get_scenario_all_desc(target_task),
                     queried_former_failed_knowledge=queried_former_failed_knowledge_to_render,
                 )
             )
@@ -250,7 +258,7 @@ class FactorEvolvingStrategyWithGraph(MultiProcessEvolvingStrategy):
                         Environment(undefined=StrictUndefined)
                         .from_string(implement_prompts["evolving_strategy_error_summary_v2_system"])
                         .render(
-                            scenario=self.scen.get_scenario_all_desc(),
+                            scenario=self.scen.get_scenario_all_desc(target_task),
                             factor_information_str=target_factor_task_information,
                             code_and_feedback=queried_former_failed_knowledge_to_render[
                                 -1
@@ -274,7 +282,7 @@ class FactorEvolvingStrategyWithGraph(MultiProcessEvolvingStrategy):
                         )
                         if (
                             session_summary.build_chat_completion_message_and_calculate_token(error_summary_user_prompt)
-                            < RD_AGENT_SETTINGS.chat_token_limit
+                            < LLM_SETTINGS.chat_token_limit
                         ):
                             break
                         elif len(queried_similar_error_knowledge_to_render) > 0:
@@ -295,6 +303,7 @@ class FactorEvolvingStrategyWithGraph(MultiProcessEvolvingStrategy):
                         queried_similar_error_knowledge=queried_similar_error_knowledge_to_render,
                         error_summary=error_summary,
                         error_summary_critics=error_summary_critics,
+                        latest_attempt_to_latest_successful_execution=latest_attempt_to_latest_successful_execution,
                     )
                     .strip("\n")
                 )
@@ -302,7 +311,7 @@ class FactorEvolvingStrategyWithGraph(MultiProcessEvolvingStrategy):
                     session.build_chat_completion_message_and_calculate_token(
                         user_prompt,
                     )
-                    < RD_AGENT_SETTINGS.chat_token_limit
+                    < LLM_SETTINGS.chat_token_limit
                 ):
                     break
                 elif len(queried_former_failed_knowledge_to_render) > 1:

rdagent/components/coder/factor_coder/CoSTEER/knowledge_management.py

@@ -21,9 +21,9 @@ from rdagent.components.knowledge_management.graph import (
 )
 from rdagent.core.evolving_framework import (
     EvolvableSubjects,
+    EvolvingKnowledgeBase,
     EvoStep,
     Knowledge,
-    KnowledgeBase,
     QueriedKnowledge,
     RAGStrategy,
 )
@@ -70,12 +70,13 @@ class FactorQueriedKnowledge(QueriedKnowledge):
         self.failed_task_info_set = failed_task_info_set
 
 
-class FactorKnowledgeBaseV1(KnowledgeBase):
-    def __init__(self) -> None:
+class FactorKnowledgeBaseV1(EvolvingKnowledgeBase):
+    def __init__(self, path: str | Path = None) -> None:
         self.implementation_trace: dict[str, FactorKnowledge] = dict()
         self.success_task_info_set: set[str] = set()
 
         self.task_to_embedding = dict()
+        super().__init__(path)
 
     def query(self) -> QueriedKnowledge | None:
         """
@@ -295,6 +296,7 @@ class FactorGraphRAGStrategy(RAGStrategy):
             evo,
             factor_implementation_queried_graph_knowledge,
             FACTOR_IMPLEMENT_SETTINGS.v2_query_former_trace_limit,
+            FACTOR_IMPLEMENT_SETTINGS.v2_add_fail_attempt_to_latest_successful_execution,
         )
         factor_implementation_queried_graph_knowledge = self.component_query(
             evo,
@@ -391,6 +393,7 @@ class FactorGraphRAGStrategy(RAGStrategy):
         evo: EvolvableSubjects,
         factor_implementation_queried_graph_knowledge: FactorQueriedGraphKnowledge,
         v2_query_former_trace_limit: int = 5,
+        v2_add_fail_attempt_to_latest_successful_execution: bool = False,
     ) -> Union[QueriedKnowledge, set]:
         """
         Query the former trace knowledge of the working trace, and find all the failed task information which tried more than fail_task_trial_limit times
@@ -428,11 +431,25 @@ class FactorGraphRAGStrategy(RAGStrategy):
                     else:
                         current_index += 1
 
-                factor_implementation_queried_graph_knowledge.former_traces[
-                    target_factor_task_information
-                ] = former_trace_knowledge[-v2_query_former_trace_limit:]
+                latest_attempt = None
+                if v2_add_fail_attempt_to_latest_successful_execution:
+                    # When the last successful execution is not the last one in the working trace, it means we have tried to correct it. We should tell the agent this fail trial to avoid endless loop in the future.
+                    if (
+                        len(former_trace_knowledge) > 0
+                        and len(self.knowledgebase.working_trace_knowledge[target_factor_task_information]) > 1
+                        and self.knowledgebase.working_trace_knowledge[target_factor_task_information].index(
+                            former_trace_knowledge[-1]
+                        )
+                        < len(self.knowledgebase.working_trace_knowledge[target_factor_task_information]) - 1
+                    ):
+                        latest_attempt = self.knowledgebase.working_trace_knowledge[target_factor_task_information][-1]
+
+                factor_implementation_queried_graph_knowledge.former_traces[target_factor_task_information] = (
+                    former_trace_knowledge[-v2_query_former_trace_limit:],
+                    latest_attempt,
+                )
             else:
-                factor_implementation_queried_graph_knowledge.former_traces[target_factor_task_information] = []
+                factor_implementation_queried_graph_knowledge.former_traces[target_factor_task_information] = ([], None)
 
         return factor_implementation_queried_graph_knowledge
 
@@ -564,7 +581,7 @@ class FactorGraphRAGStrategy(RAGStrategy):
                     if knowledge.feedback is not None and knowledge.feedback.final_decision_based_on_gt == False
                 ]
                 queried_from_gt_knowledge_count = max(
-                    min(v2_query_component_limit // 2, len(queried_from_gt_knowledge_list)),
+                    min((v2_query_component_limit // 2 + 1), len(queried_from_gt_knowledge_list)),
                     v2_query_component_limit - len(queried_without_gt_knowledge_list),
                 )
                 factor_implementation_queried_graph_knowledge.component_with_success_task[
@@ -606,7 +623,7 @@ class FactorGraphRAGStrategy(RAGStrategy):
                 ):
                     queried_last_trace = factor_implementation_queried_graph_knowledge.former_traces[
                         target_factor_task_information
-                    ][-1]
+                    ][0][-1]
                     target_index = self.knowledgebase.working_trace_knowledge[target_factor_task_information].index(
                         queried_last_trace,
                     )
@@ -711,12 +728,12 @@ class FactorGraphRAGStrategy(RAGStrategy):
         return factor_implementation_queried_graph_knowledge
 
 
-class FactorGraphKnowledgeBase(KnowledgeBase):
-    def __init__(self, init_component_list=None) -> None:
+class FactorGraphKnowledgeBase(EvolvingKnowledgeBase):
+    def __init__(self, init_component_list=None, path: str | Path = None) -> None:
         """
         Load knowledge, offer brief information of knowledge and common handle interfaces
         """
-        self.graph: UndirectedGraph = UndirectedGraph.load(Path.cwd() / "graph.pkl")
+        self.graph: UndirectedGraph = UndirectedGraph(Path.cwd() / "graph.pkl")
         logger.info(f"Knowledge Graph loaded, size={self.graph.size()}")
 
         if init_component_list:

rdagent/components/coder/factor_coder/CoSTEER/scheduler.py

@@ -7,10 +7,10 @@ from jinja2 import Environment, StrictUndefined
 from rdagent.components.coder.factor_coder.CoSTEER.evolvable_subjects import (
     FactorEvolvingItem,
 )
-from rdagent.core.conf import RD_AGENT_SETTINGS
 from rdagent.core.prompts import Prompts
 from rdagent.core.scenario import Scenario
 from rdagent.log import rdagent_logger as logger
+from rdagent.oai.llm_conf import LLM_SETTINGS
 from rdagent.oai.llm_utils import APIBackend
 
 scheduler_prompts = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")
@@ -40,7 +40,7 @@ def LLMSelect(
         # find corresponding former trace for each task
         target_factor_task_information = evo.sub_tasks[i].get_task_information()
         if target_factor_task_information in former_trace:
-            tasks.append((i, evo.sub_tasks[i], former_trace[target_factor_task_information]))
+            tasks.append((i, evo.sub_tasks[i], former_trace[target_factor_task_information][0]))
 
     system_prompt = (
         Environment(undefined=StrictUndefined)
@@ -68,7 +68,7 @@ def LLMSelect(
                 user_prompt=user_prompt,
                 system_prompt=system_prompt,
             )
-            < RD_AGENT_SETTINGS.chat_token_limit
+            < LLM_SETTINGS.chat_token_limit
         ):
             break
 

rdagent/components/coder/factor_coder/config.py

@@ -8,19 +8,17 @@ SELECT_METHOD = Literal["random", "scheduler"]
 
 class FactorImplementSettings(BaseSettings):
     class Config:
-        env_prefix = "FACTOR_CODER_"  # Use FACTOR_CODER_ as prefix for environment variables
+        env_prefix = "FACTOR_CODER_"
+        """Use `FACTOR_CODER_` as prefix for environment variables"""
 
     coder_use_cache: bool = False
-    data_folder: str = str(
-        (Path().cwd() / "git_ignore_folder" / "factor_implementation_source_data").absolute(),
-    )
-    data_folder_debug: str = str(
-        (Path().cwd() / "git_ignore_folder" / "factor_implementation_source_data_debug").absolute(),
-    )
-    cache_location: str = str(
-        (Path().cwd() / "git_ignore_folder" / "factor_implementation_execution_cache").absolute(),
-    )
-    enable_execution_cache: bool = True  # whether to enable the execution cache
+    """Indicates whether to use cache for the coder"""
+
+    data_folder: str = "git_ignore_folder/factor_implementation_source_data"
+    """Path to the folder containing financial data (default is fundamental data in Qlib)"""
+
+    data_folder_debug: str = "git_ignore_folder/factor_implementation_source_data_debug"
+    """Path to the folder containing partial financial data (for debugging)"""
 
     # TODO: the factor implement specific settings should not appear in this settings
     # Evolving should have a method specific settings
@@ -33,20 +31,30 @@ class FactorImplementSettings(BaseSettings):
     v2_query_component_limit: int = 1
     v2_query_error_limit: int = 1
     v2_query_former_trace_limit: int = 1
+    v2_add_fail_attempt_to_latest_successful_execution: bool = False
     v2_error_summary: bool = False
     v2_knowledge_sampler: float = 1.0
 
-    file_based_execution_timeout: int = 120  # seconds for each factor implementation execution
+    file_based_execution_timeout: int = 120
+    """Timeout in seconds for each factor implementation execution"""
+
+    select_method: str = "random"
+    """Method for the selection of factors implementation"""
 
-    select_method: SELECT_METHOD = "random"
     select_threshold: int = 10
+    """Threshold for the number of factor selections"""
 
     max_loop: int = 10
+    """Maximum number of task implementation loops"""
 
     knowledge_base_path: Union[str, None] = None
+    """Path to the knowledge base"""
+
     new_knowledge_base_path: Union[str, None] = None
+    """Path to the new knowledge base"""
 
     python_bin: str = "python"
+    """Path to the Python binary"""
 
 
 FACTOR_IMPLEMENT_SETTINGS = FactorImplementSettings()

rdagent/components/coder/factor_coder/factor.py

@@ -9,9 +9,11 @@ from typing import Tuple, Union
 import pandas as pd
 from filelock import FileLock
 
+from rdagent.app.kaggle.conf import KAGGLE_IMPLEMENT_SETTING
 from rdagent.components.coder.factor_coder.config import FACTOR_IMPLEMENT_SETTINGS
 from rdagent.core.exception import CodeFormatError, CustomRuntimeError, NoOutputError
 from rdagent.core.experiment import Experiment, FBWorkspace, Task
+from rdagent.core.utils import cache_with_pickle
 from rdagent.log import rdagent_logger as logger
 from rdagent.oai.llm_utils import md5_hash
 
@@ -24,16 +26,21 @@ class FactorTask(Task):
         factor_name,
         factor_description,
         factor_formulation,
+        *args,
         variables: dict = {},
         resource: str = None,
         factor_implementation: bool = False,
+        **kwargs,
     ) -> None:
-        self.factor_name = factor_name
+        self.factor_name = (
+            factor_name  # TODO: remove it in the later version. Keep it only for pickle version compatibility
+        )
         self.factor_description = factor_description
         self.factor_formulation = factor_formulation
         self.variables = variables
         self.factor_resources = resource
         self.factor_implementation = factor_implementation
+        super().__init__(name=factor_name, *args, **kwargs)
 
     def get_task_information(self):
         return f"""factor_name: {self.factor_name}
@@ -65,7 +72,6 @@ class FactorFBWorkspace(FBWorkspace):
     """
 
     # TODO: (Xiao) think raising errors may get better information for processing
-    FB_FROM_CACHE = "The factor value has been executed and stored in the instance variable."
     FB_EXEC_SUCCESS = "Execution succeeded without error."
     FB_CODE_NOT_SET = "code is not set."
     FB_EXECUTION_SUCCEEDED = "Execution succeeded without error."
@@ -75,39 +81,38 @@ class FactorFBWorkspace(FBWorkspace):
     def __init__(
         self,
         *args,
-        executed_factor_value_dataframe=None,
-        raise_exception=False,
+        raise_exception: bool = False,
         **kwargs,
     ) -> None:
         super().__init__(*args, **kwargs)
-        self.executed_factor_value_dataframe = executed_factor_value_dataframe
         self.raise_exception = raise_exception
 
-    @staticmethod
-    def link_data_to_workspace(data_path: Path, workspace_path: Path):
-        data_path = Path(data_path)
-        workspace_path = Path(workspace_path)
-        for data_file_path in data_path.iterdir():
-            workspace_data_file_path = workspace_path / data_file_path.name
-            if workspace_data_file_path.exists():
-                workspace_data_file_path.unlink()
-            subprocess.run(
-                ["ln", "-s", data_file_path, workspace_data_file_path],
-                check=False,
-            )
+    def hash_func(self, data_type: str = "Debug") -> str:
+        return (
+            md5_hash(data_type + self.code_dict["factor.py"])
+            if ("factor.py" in self.code_dict and not self.raise_exception)
+            else None
+        )
 
-    def execute(self, store_result: bool = False, data_type: str = "Debug") -> Tuple[str, pd.DataFrame]:
+    @cache_with_pickle(hash_func)
+    def execute(self, data_type: str = "Debug") -> Tuple[str, pd.DataFrame]:
         """
         execute the implementation and get the factor value by the following steps:
         1. make the directory in workspace path
         2. write the code to the file in the workspace path
         3. link all the source data to the workspace path folder
-        4. execute the code
+        if call_factor_py is True:
+            4. execute the code
+        else:
+            4. generate a script from template to import the factor.py dump get the factor value to result.h5
         5. read the factor value from the output file in the workspace path folder
         returns the execution feedback as a string and the factor value as a pandas dataframe
 
-        parameters:
-        store_result: if True, store the factor value in the instance variable, this feature is to be used in the gt implementation to avoid multiple execution on the same gt implementation
+
+        Regarding the cache mechanism:
+        1. We will store the function's return value to ensure it behaves as expected.
+        - The cached information will include a tuple with the following: (execution_feedback, executed_factor_value_dataframe, Optional[Exception])
+
         """
         super().execute()
         if self.code_dict is None or "factor.py" not in self.code_dict:
@@ -116,40 +121,38 @@ class FactorFBWorkspace(FBWorkspace):
             else:
                 return self.FB_CODE_NOT_SET, None
         with FileLock(self.workspace_path / "execution.lock"):
-            if FACTOR_IMPLEMENT_SETTINGS.enable_execution_cache:
-                # NOTE: cache the result for the same code and same data type
-                target_file_name = md5_hash(data_type + self.code_dict["factor.py"])
-                cache_file_path = Path(FACTOR_IMPLEMENT_SETTINGS.cache_location) / f"{target_file_name}.pkl"
-                Path(FACTOR_IMPLEMENT_SETTINGS.cache_location).mkdir(exist_ok=True, parents=True)
-                if cache_file_path.exists() and not self.raise_exception:
-                    cached_res = pickle.load(open(cache_file_path, "rb"))
-                    if store_result and cached_res[1] is not None:
-                        self.executed_factor_value_dataframe = cached_res[1]
-                    return cached_res
-
-            if self.executed_factor_value_dataframe is not None:
-                return self.FB_FROM_CACHE, self.executed_factor_value_dataframe
-
-            source_data_path = (
-                Path(
-                    FACTOR_IMPLEMENT_SETTINGS.data_folder_debug,
+            if self.target_task.version == 1:
+                source_data_path = (
+                    Path(
+                        FACTOR_IMPLEMENT_SETTINGS.data_folder_debug,
+                    )
+                    if data_type == "Debug"  # FIXME: (yx) don't think we should use a debug tag for this.
+                    else Path(
+                        FACTOR_IMPLEMENT_SETTINGS.data_folder,
+                    )
                 )
-                if data_type == "Debug"
-                else Path(
-                    FACTOR_IMPLEMENT_SETTINGS.data_folder,
-                )
-            )
+            elif self.target_task.version == 2:
+                # TODO you can change the name of the data folder for a better understanding
+                source_data_path = Path(KAGGLE_IMPLEMENT_SETTING.local_data_path) / KAGGLE_IMPLEMENT_SETTING.competition
 
             source_data_path.mkdir(exist_ok=True, parents=True)
             code_path = self.workspace_path / f"factor.py"
 
-            self.link_data_to_workspace(source_data_path, self.workspace_path)
+            self.link_all_files_in_folder_to_workspace(source_data_path, self.workspace_path)
 
             execution_feedback = self.FB_EXECUTION_SUCCEEDED
             execution_success = False
+            execution_error = None
+
+            if self.target_task.version == 1:
+                execution_code_path = code_path
+            elif self.target_task.version == 2:
+                execution_code_path = self.workspace_path / f"{uuid.uuid4()}.py"
+                execution_code_path.write_text((Path(__file__).parent / "factor_execution_template.txt").read_text())
+
             try:
                 subprocess.check_output(
-                    f"{FACTOR_IMPLEMENT_SETTINGS.python_bin} {code_path}",
+                    f"{FACTOR_IMPLEMENT_SETTINGS.python_bin} {execution_code_path}",
                     shell=True,
                     cwd=self.workspace_path,
                     stderr=subprocess.STDOUT,
@@ -161,7 +164,7 @@ class FactorFBWorkspace(FBWorkspace):
 
                 execution_feedback = (
                     e.output.decode()
-                    .replace(str(code_path.parent.absolute()), r"/path/to")
+                    .replace(str(execution_code_path.parent.absolute()), r"/path/to")
                     .replace(str(site.getsitepackages()[0]), r"/path/to/site-packages")
                 )
                 if len(execution_feedback) > 2000:
@@ -170,10 +173,14 @@ class FactorFBWorkspace(FBWorkspace):
                     )
                 if self.raise_exception:
                     raise CustomRuntimeError(execution_feedback)
+                else:
+                    execution_error = CustomRuntimeError(execution_feedback)
             except subprocess.TimeoutExpired:
                 execution_feedback += f"Execution timeout error and the timeout is set to {FACTOR_IMPLEMENT_SETTINGS.file_based_execution_timeout} seconds."
                 if self.raise_exception:
                     raise CustomRuntimeError(execution_feedback)
+                else:
+                    execution_error = CustomRuntimeError(execution_feedback)
 
             workspace_output_file_path = self.workspace_path / "result.h5"
             if workspace_output_file_path.exists() and execution_success:
@@ -188,15 +195,9 @@ class FactorFBWorkspace(FBWorkspace):
                 executed_factor_value_dataframe = None
                 if self.raise_exception:
                     raise NoOutputError(execution_feedback)
+                else:
+                    execution_error = NoOutputError(execution_feedback)
 
-            if store_result and executed_factor_value_dataframe is not None:
-                self.executed_factor_value_dataframe = executed_factor_value_dataframe
-
-        if FACTOR_IMPLEMENT_SETTINGS.enable_execution_cache:
-            pickle.dump(
-                (execution_feedback, executed_factor_value_dataframe),
-                open(cache_file_path, "wb"),
-            )
         return execution_feedback, executed_factor_value_dataframe
 
     def __str__(self) -> str:
@@ -218,3 +219,4 @@ class FactorFBWorkspace(FBWorkspace):
 
 
 FactorExperiment = Experiment
+FeatureExperiment = Experiment

rdagent/components/coder/factor_coder/factor_execution_template.txt

@@ -0,0 +1,15 @@
+import os
+
+import numpy as np
+import pandas as pd
+from factor import feature_engineering_cls
+
+if os.path.exists("X_valid.pkl"):
+    valid_df = pd.read_pickle("X_valid.pkl").head(1000)
+else:
+    raise FileNotFoundError("No valid data found.")
+
+cls = feature_engineering_cls()
+cls.fit(valid_df)
+new_feat = cls.transform(valid_df)
+new_feat.to_hdf("result.h5", key="data", mode="w")

rdagent/components/coder/factor_coder/prompts.yaml

@@ -118,6 +118,13 @@ evolving_strategy_factor_implementation_v2_user: |-
   {{ similar_component_knowledge.implementation.code }}
   {% endfor %}
   {% endif %}
+  {% if latest_attempt_to_latest_successful_execution is not none %}
+  You have tried to correct your former failed code but still met some errors. Here is the latest attempt to the latest successful execution, try not to get the same error to your new code:
+  =====Your latest attempt=====
+  {{ latest_attempt_to_latest_successful_execution.implementation.code }}
+  =====Feedback to your latest attempt=====
+  {{ latest_attempt_to_latest_successful_execution.feedback }}
+  {% endif %}
 
 evolving_strategy_error_summary_v2_system: |-
   User is trying to implement some factors in the following scenario:
@@ -212,7 +219,7 @@ evaluator_final_decision_v1_system: |-
   The implementation final decision is considered in the following logic:
   1. If the value and the ground truth value are exactly the same under a small tolerance, the implementation is considered correct.
   2. If the value and the ground truth value have a high correlation on ic or rank ic, the implementation is considered correct.
-  3. If no ground truth value is not provided, the implementation is considered correct if the code execution is successful and the code feedback is align with the scenario and factor description.
+  3. If no ground truth value is provided, the implementation is considered correct if the code executes successfully (assuming the data provided is correct). Any exceptions, including those actively raised, are considered faults of the code. Additionally, the code feedback must align with the scenario and factor description.
 
   Please response the critic in the json format. Here is an example structure for the JSON output, please strictly follow the format:
   {

rdagent/components/coder/model_coder/CoSTEER/__init__.py

@@ -70,7 +70,7 @@ class ModelCoSTEER(Developer[ModelExperiment]):
         self.rag = ModelRAGStrategy(model_knowledge_base)
 
         # init intermediate items
-        model_experiment = ModelEvolvingItem(sub_tasks=exp.sub_tasks)
+        model_experiment = ModelEvolvingItem.from_experiment(exp)
 
         self.evolve_agent = ModelRAGEvoAgent(
             max_loop=self.max_loop,

rdagent/components/coder/model_coder/CoSTEER/evaluators.py

@@ -4,7 +4,6 @@ from pathlib import Path
 from typing import List, Tuple
 
 import numpy as np
-import torch
 from jinja2 import Environment, StrictUndefined
 
 from rdagent.components.coder.model_coder.conf import MODEL_IMPL_SETTINGS
@@ -19,41 +18,43 @@ from rdagent.core.experiment import Task, Workspace
 from rdagent.core.prompts import Prompts
 from rdagent.core.utils import multiprocessing_wrapper
 from rdagent.log import rdagent_logger as logger
+from rdagent.oai.llm_conf import LLM_SETTINGS
 from rdagent.oai.llm_utils import APIBackend
 
 evaluate_prompts = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")
 
 
-def shape_evaluator(prediction: torch.Tensor, target_shape: Tuple = None) -> Tuple[str, bool]:
+def shape_evaluator(prediction: np.ndarray, target_shape: Tuple = None) -> Tuple[str, bool]:
     if target_shape is None or prediction is None:
-        return "No output generated from the model. No shape evaluation conducted.", False
+        return (
+            "No output generated from the model. No shape evaluation conducted.",
+            False,
+        )
     pre_shape = prediction.shape
 
     if pre_shape == target_shape:
         return "The shape of the output is correct.", True
     else:
-        return f"The shape of the output is incorrect. Expected {target_shape}, but got {pre_shape}.", False
-
-
-def reshape_tensor(original_tensor, target_shape):
-    new_tensor = torch.zeros(target_shape)
-    for i, dim in enumerate(original_tensor.shape):
-        new_tensor = new_tensor.narrow(i, 0, dim).copy_(original_tensor)
-
-    return new_tensor
+        return (
+            f"The shape of the output is incorrect. Expected {target_shape}, but got {pre_shape}.",
+            False,
+        )
 
 
 def value_evaluator(
-    prediction: torch.Tensor,
-    target: torch.Tensor,
-) -> Tuple[torch.Tensor, bool]:
+    prediction: np.ndarray,
+    target: np.ndarray,
+) -> Tuple[np.ndarray, bool]:
     if prediction is None:
         return "No output generated from the model. Skip value evaluation", False
     elif target is None:
-        return "No ground truth output provided. Value evaluation not impractical", False
+        return (
+            "No ground truth output provided. Value evaluation not impractical",
+            False,
+        )
     else:
         # Calculate the mean absolute difference
-        diff = torch.mean(torch.abs(target - prediction)).item()
+        diff = np.mean(np.abs(target - prediction))
         return (
             f"The value of the output is correct. The mean absolute difference is {diff}.",
             diff < 0.1,
@@ -80,7 +81,13 @@ class ModelCodeEvaluator(Evaluator):
         system_prompt = (
             Environment(undefined=StrictUndefined)
             .from_string(evaluate_prompts["evaluator_code_feedback"]["system"])
-            .render(scenario=self.scen.get_scenario_all_desc() if self.scen is not None else "No scenario description.")
+            .render(
+                scenario=(
+                    self.scen.get_scenario_all_desc(target_task)
+                    if self.scen is not None
+                    else "No scenario description."
+                )
+            )
         )
 
         execution_feedback_to_render = model_execution_feedback
@@ -103,7 +110,7 @@ class ModelCodeEvaluator(Evaluator):
                     user_prompt=user_prompt,
                     system_prompt=system_prompt,
                 )
-                > RD_AGENT_SETTINGS.chat_token_limit
+                > LLM_SETTINGS.chat_token_limit
             ):
                 execution_feedback_to_render = execution_feedback_to_render[len(execution_feedback_to_render) // 2 :]
             else:
@@ -136,7 +143,13 @@ class ModelFinalEvaluator(Evaluator):
         system_prompt = (
             Environment(undefined=StrictUndefined)
             .from_string(evaluate_prompts["evaluator_final_feedback"]["system"])
-            .render(scenario=self.scen.get_scenario_all_desc() if self.scen is not None else "No scenario description.")
+            .render(
+                scenario=(
+                    self.scen.get_scenario_all_desc(target_task)
+                    if self.scen is not None
+                    else "No scenario description."
+                )
+            )
         )
 
         execution_feedback_to_render = model_execution_feedback
@@ -159,7 +172,7 @@ class ModelFinalEvaluator(Evaluator):
                     user_prompt=user_prompt,
                     system_prompt=system_prompt,
                 )
-                > RD_AGENT_SETTINGS.chat_token_limit
+                > LLM_SETTINGS.chat_token_limit
             ):
                 execution_feedback_to_render = execution_feedback_to_render[len(execution_feedback_to_render) // 2 :]
             else:
@@ -251,7 +264,7 @@ class ModelCoderEvaluator(Evaluator):
         param_init_value = 0.6
 
         assert isinstance(implementation, ModelFBWorkspace)
-        model_execution_feedback, gen_tensor = implementation.execute(
+        model_execution_feedback, gen_np_array = implementation.execute(
             batch_size=batch_size,
             num_features=num_features,
             num_timesteps=num_timesteps,
@@ -260,7 +273,7 @@ class ModelCoderEvaluator(Evaluator):
         )
         if gt_implementation is not None:
             assert isinstance(gt_implementation, ModelFBWorkspace)
-            _, gt_tensor = gt_implementation.execute(
+            _, gt_np_array = gt_implementation.execute(
                 batch_size=batch_size,
                 num_features=num_features,
                 num_timesteps=num_timesteps,
@@ -268,10 +281,10 @@ class ModelCoderEvaluator(Evaluator):
                 param_init_value=param_init_value,
             )
         else:
-            gt_tensor = None
+            gt_np_array = None
 
-        shape_feedback, shape_decision = shape_evaluator(gen_tensor, (batch_size, 1))
-        value_feedback, value_decision = value_evaluator(gt_tensor, gen_tensor)
+        shape_feedback, shape_decision = shape_evaluator(gen_np_array, (batch_size, 1))
+        value_feedback, value_decision = value_evaluator(gen_np_array, gt_np_array)
         code_feedback, _ = ModelCodeEvaluator(scen=self.scen).evaluate(
             target_task=target_task,
             implementation=implementation,

rdagent/components/coder/model_coder/CoSTEER/evolvable_subjects.py

@@ -27,3 +27,10 @@ class ModelEvolvingItem(ModelExperiment, EvolvableSubjects):
             )
         else:
             self.sub_gt_implementations = sub_gt_implementations
+
+    @classmethod
+    def from_experiment(cls, exp: ModelExperiment) -> "ModelEvolvingItem":
+        ei = cls(sub_tasks=exp.sub_tasks)
+        ei.based_experiments = exp.based_experiments
+        ei.experiment_workspace = exp.experiment_workspace
+        return ei

rdagent/components/coder/model_coder/CoSTEER/evolving_strategy.py

@@ -11,12 +11,18 @@ from rdagent.components.coder.model_coder.CoSTEER.evolvable_subjects import (
 from rdagent.components.coder.model_coder.CoSTEER.knowledge_management import (
     ModelQueriedKnowledge,
 )
-from rdagent.components.coder.model_coder.model import ModelFBWorkspace, ModelTask
+from rdagent.components.coder.model_coder.model import (
+    ModelExperiment,
+    ModelFBWorkspace,
+    ModelTask,
+)
 from rdagent.core.conf import RD_AGENT_SETTINGS
 from rdagent.core.evolving_framework import EvolvingStrategy
 from rdagent.core.prompts import Prompts
 from rdagent.core.utils import multiprocessing_wrapper
+from rdagent.oai.llm_conf import LLM_SETTINGS
 from rdagent.oai.llm_utils import APIBackend
+from rdagent.scenarios.kaggle.experiment.kaggle_experiment import KG_MODEL_MAPPING
 
 coder_prompts = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")
 
@@ -26,8 +32,25 @@ class ModelCoderEvolvingStrategy(EvolvingStrategy):
         self,
         target_task: ModelTask,
         queried_knowledge: ModelQueriedKnowledge = None,
+        current_exp: ModelExperiment = None,  # Add this parameter
     ) -> str:
         model_information_str = target_task.get_task_information()
+        model_type = target_task.model_type
+
+        if len(current_exp.based_experiments) == 0:
+            current_code = None
+        else:
+            current_code = ""
+            sota_exp_code_dict = current_exp.based_experiments[-1].experiment_workspace.code_dict
+            if target_task.version == 2:
+                if model_type in KG_MODEL_MAPPING:
+                    current_code = sota_exp_code_dict.get(KG_MODEL_MAPPING[model_type], None)
+                elif "model.py" in sota_exp_code_dict:
+                    current_code = sota_exp_code_dict["model.py"]
+                else:
+                    current_code = None
+            elif target_task.version == 1:
+                current_code = sota_exp_code_dict.get("model.py", None)
 
         if queried_knowledge is not None and model_information_str in queried_knowledge.success_task_to_knowledge_dict:
             return queried_knowledge.success_task_to_knowledge_dict[model_information_str].implementation
@@ -55,6 +78,7 @@ class ModelCoderEvolvingStrategy(EvolvingStrategy):
                 .render(
                     scenario=self.scen.get_scenario_all_desc(),
                     queried_former_failed_knowledge=queried_former_failed_knowledge_to_render,
+                    current_code=current_code,
                 )
             )
 
@@ -77,7 +101,7 @@ class ModelCoderEvolvingStrategy(EvolvingStrategy):
                         user_prompt=user_prompt,
                         system_prompt=system_prompt,
                     )
-                    < RD_AGENT_SETTINGS.chat_token_limit
+                    < LLM_SETTINGS.chat_token_limit
                 ):
                     break
                 elif len(queried_former_failed_knowledge_to_render) > 1:
@@ -119,15 +143,14 @@ class ModelCoderEvolvingStrategy(EvolvingStrategy):
 
         result = multiprocessing_wrapper(
             [
-                (self.implement_one_model, (evo.sub_tasks[target_index], queried_knowledge))
+                (self.implement_one_model, (evo.sub_tasks[target_index], queried_knowledge, evo))
                 for target_index in to_be_finished_task_index
             ],
             n=RD_AGENT_SETTINGS.multi_proc_n,
         )
 
         for index, target_index in enumerate(to_be_finished_task_index):
-            if evo.sub_workspace_list[target_index] is None:
-                evo.sub_workspace_list[target_index] = ModelFBWorkspace(target_task=evo.sub_tasks[target_index])
+            evo.sub_workspace_list[target_index] = ModelFBWorkspace(target_task=evo.sub_tasks[target_index])
             evo.sub_workspace_list[target_index].inject_code(**{"model.py": result[index]})
 
         evo.corresponding_selection = to_be_finished_task_index

rdagent/components/coder/model_coder/CoSTEER/knowledge_management.py

@@ -1,11 +1,13 @@
+from pathlib import Path
+
 from rdagent.components.coder.model_coder.conf import MODEL_IMPL_SETTINGS
 from rdagent.components.coder.model_coder.CoSTEER.evaluators import ModelCoderFeedback
 from rdagent.components.coder.model_coder.model import ModelTask
 from rdagent.core.evolving_framework import (
     EvolvableSubjects,
+    EvolvingKnowledgeBase,
     EvoStep,
     Knowledge,
-    KnowledgeBase,
     QueriedKnowledge,
     RAGStrategy,
 )
@@ -49,13 +51,15 @@ class ModelQueriedKnowledge(QueriedKnowledge):
         self.working_task_to_similar_successful_knowledge_dict = dict()
 
 
-class ModelKnowledgeBase(KnowledgeBase):
-    def __init__(self) -> None:
+class ModelKnowledgeBase(EvolvingKnowledgeBase):
+    def __init__(self, path: str | Path = None) -> None:
         self.implementation_trace: dict[str, ModelKnowledge] = dict()
         self.success_task_info_set: set[str] = set()
 
         self.task_to_embedding = dict()
 
+        super().__init__(path)
+
     def query(self) -> QueriedKnowledge | None:
         """
         Query the knowledge base to get the queried knowledge. So far is handled in RAG strategy.

rdagent/components/coder/model_coder/conf.py

@@ -10,10 +10,6 @@ class ModelImplSettings(BaseSettings):
 
     coder_use_cache: bool = False
 
-    cache_location: str = str(
-        (Path().cwd() / "git_ignore_folder" / "model_implementation_execution_cache").absolute(),
-    )
-
     knowledge_base_path: Union[str, None] = None
     new_knowledge_base_path: Union[str, None] = None
 
@@ -23,7 +19,5 @@ class ModelImplSettings(BaseSettings):
     query_similar_success_limit: int = 5
     fail_task_trial_limit: int = 20
 
-    enable_execution_cache: bool = True  # whether to enable the execution cache
-
 
 MODEL_IMPL_SETTINGS = ModelImplSettings()

rdagent/components/coder/model_coder/main.py

@@ -1,92 +0,0 @@
-"""
-This file will be removed in the future and replaced by
-- rdagent/app/model_implementation/eval.py
-"""
-
-import os
-
-# randomly generate a input graph, node_feature and edge_index
-# 1000 nodes, 128 dim node feature, 2000 edges
-import torch
-from dotenv import load_dotenv
-
-from rdagent.components.coder.model_coder.CoSTEER.evaluators import (
-    shape_evaluator,
-    value_evaluator,
-)
-from rdagent.oai.llm_utils import APIBackend
-
-assert load_dotenv()
-formula_info = {
-    "name": "Anti-Symmetric Deep Graph Network (A-DGN)",
-    "description": "A framework for stable and non-dissipative DGN design. It ensures long-range information preservation between nodes and prevents gradient vanishing or explosion during training.",
-    "formulation": "x_u^{(l)} = x_u^{(l-1)} + \\epsilon \\sigma \\left( W^T x_u^{(l-1)} + \\Phi(X^{(l-1)}, N_u) + b \\right)",
-    "variables": {
-        "x_u^{(l)}": "The state of node u at layer l",
-        "\\epsilon": "The step size in the Euler discretization",
-        "\\sigma": "A monotonically non-decreasing activation function",
-        "W": "An anti-symmetric weight matrix",
-        "X^{(l-1)}": "The node feature matrix at layer l-1",
-        "N_u": "The set of neighbors of node u",
-        "b": "A bias vector",
-    },
-}
-
-system_prompt = "You are an assistant whose job is to answer user's question."
-user_prompt = "With the following given information, write a python code using pytorch and torch_geometric to implement the model. This model is in the graph learning field, only have one layer. The input will be node_feature [num_nodes, dim_feature] and edge_index [2, num_edges], and they should be loaded from the files 'node_features.pt' and 'edge_index.pt'. There is not edge attribute or edge weight as input. The model should detect the node_feature and edge_index shape, if there is Linear transformation layer in the model, the input and output shape should be consistent. The in_channels is the dimension of the node features. You code should contain additional 'if __name__ == '__main__', where you should load the node_feature and edge_index from the files and run the model, and save the output to a file 'llm_output.pt'. Implement the model forward function based on the following information: model formula information. 1. model name: {}, 2. model description: {}, 3. model formulation: {}, 4. model variables: {}. You must complete the forward function as far as you can do.".format(
-    formula_info["name"],
-    formula_info["description"],
-    formula_info["formulation"],
-    formula_info["variables"],
-)
-
-resp = APIBackend(use_chat_cache=False).build_messages_and_create_chat_completion(user_prompt, system_prompt)
-
-print(resp)
-
-# take the code part from the response and save it to a file, the code is covered in the ```python``` block
-code = resp.split("```python")[1].split("```")[0]
-with open("llm_code.py", "w") as f:
-    f.write(code)
-
-average_shape_eval = []
-average_value_eval = []
-for test_mode in ["zeros", "ones", "randn"]:
-    if test_mode == "zeros":
-        node_feature = torch.zeros(1000, 128)
-    elif test_mode == "ones":
-        node_feature = torch.ones(1000, 128)
-    elif test_mode == "randn":
-        node_feature = torch.randn(1000, 128)
-    edge_index = torch.randint(0, 1000, (2, 2000))
-
-    torch.save(node_feature, "node_features.pt")
-    torch.save(edge_index, "edge_index.pt")
-
-    try:
-        os.system("python llm_code.py")
-    except:
-        print("Error in running the LLM code")
-    os.system("python gt_code.py")
-    os.system("rm edge_index.pt")
-    os.system("rm node_features.pt")
-    # load the output and print the shape
-
-    try:
-        llm_output = torch.load("llm_output.pt")
-    except:
-        llm_output = None
-    gt_output = torch.load("gt_output.pt")
-
-    average_shape_eval.append(shape_evaluator(llm_output, gt_output)[1])
-    average_value_eval.append(value_evaluator(llm_output, gt_output)[1])
-
-    print("Shape evaluation: ", average_shape_eval[-1])
-    print("Value evaluation:super().develop(task_l) ", average_value_eval[-1])
-
-    os.system("rm llm_output.pt")
-    os.system("rm gt_output.pt")
-os.system("rm llm_code.py")
-
-print("Average shape evaluation: ", sum(average_shape_eval) / len(average_shape_eval))
-print("Average value evaluation: ", sum(average_value_eval) / len(average_value_eval))

rdagent/components/coder/model_coder/model.py

@@ -1,18 +1,14 @@
-import json
 import pickle
 import site
 import traceback
-import uuid
 from pathlib import Path
-from typing import Any, Dict, Optional
-
-import torch
+from typing import Dict, Optional
 
 from rdagent.components.coder.model_coder.conf import MODEL_IMPL_SETTINGS
-from rdagent.core.exception import CodeFormatError
 from rdagent.core.experiment import Experiment, FBWorkspace, Task
+from rdagent.core.utils import cache_with_pickle
 from rdagent.oai.llm_utils import md5_hash
-from rdagent.utils import get_module_by_module_path
+from rdagent.utils.env import KGDockerEnv, QTDockerEnv
 
 
 class ModelTask(Task):
@@ -20,31 +16,32 @@ class ModelTask(Task):
         self,
         name: str,
         description: str,
-        formulation: str,
         architecture: str,
-        variables: Dict[str, str],
+        *args,
         hyperparameters: Dict[str, str],
+        formulation: str = None,
+        variables: Dict[str, str] = None,
         model_type: Optional[str] = None,
+        **kwargs,
     ) -> None:
-        self.name: str = name
         self.description: str = description
         self.formulation: str = formulation
         self.architecture: str = architecture
         self.variables: str = variables
         self.hyperparameters: str = hyperparameters
-        self.model_type: str = (
-            model_type  # Tabular for tabular model, TimesSeries for time series model, Graph for graph model
-        )
+        self.model_type: str = model_type  # Tabular for tabular model, TimesSeries for time series model, Graph for graph model, XGBoost for XGBoost model
+        super().__init__(name=name, *args, **kwargs)
 
     def get_task_information(self):
-        return f"""name: {self.name}
+        task_desc = f"""name: {self.name}
 description: {self.description}
-formulation: {self.formulation}
-architecture: {self.architecture}
-variables: {self.variables}
-hyperparameters: {self.hyperparameters}
-model_type: {self.model_type}
 """
+        task_desc += f"formulation: {self.formulation}\n" if self.formulation else ""
+        task_desc += f"architecture: {self.architecture}\n"
+        task_desc += f"variables: {self.variables}\n" if self.variables else ""
+        task_desc += f"hyperparameters: {self.hyperparameters}\n"
+        task_desc += f"model_type: {self.model_type}\n"
+        return task_desc
 
     @staticmethod
     def from_dict(dict):
@@ -66,14 +63,30 @@ class ModelFBWorkspace(FBWorkspace):
         - the `model.py` that contains a variable named `model_cls` which indicates the implemented model structure
             - `model_cls` is a instance of `torch.nn.Module`;
 
+    We support two ways of interface:
+        (version 1) for qlib we'll make a script to import the model in the implementation in file `model.py` after setting the cwd into the directory
+            - from model import model_cls
+            - initialize the model by initializing it `model_cls(input_dim=INPUT_DIM)`
+            - And then verify the model.
 
-    We'll import the model in the implementation in file `model.py` after setting the cwd into the directory
-    - from model import model_cls
-    - initialize the model by initializing it `model_cls(input_dim=INPUT_DIM)`
-    - And then verify the model.
-
+        (version 2) for kaggle we'll make a script to call the fit and predict function in the implementation in file `model.py` after setting the cwd into the directory
     """
 
+    def hash_func(
+        self,
+        batch_size: int = 8,
+        num_features: int = 10,
+        num_timesteps: int = 4,
+        num_edges: int = 20,
+        input_value: float = 1.0,
+        param_init_value: float = 1.0,
+    ) -> str:
+        target_file_name = f"{batch_size}_{num_features}_{num_timesteps}_{input_value}_{param_init_value}"
+        for code_file_name in sorted(list(self.code_dict.keys())):
+            target_file_name = f"{target_file_name}_{self.code_dict[code_file_name]}"
+        return md5_hash(target_file_name)
+
+    @cache_with_pickle(hash_func)
     def execute(
         self,
         batch_size: int = 8,
@@ -85,58 +98,38 @@ class ModelFBWorkspace(FBWorkspace):
     ):
         super().execute()
         try:
-            if MODEL_IMPL_SETTINGS.enable_execution_cache:
-                # NOTE: cache the result for the same code
-                target_file_name = md5_hash(
-                    f"{batch_size}_{num_features}_{num_timesteps}_{input_value}_{param_init_value}_{self.code_dict['model.py']}"
-                )
-                cache_file_path = Path(MODEL_IMPL_SETTINGS.cache_location) / f"{target_file_name}.pkl"
-                Path(MODEL_IMPL_SETTINGS.cache_location).mkdir(exist_ok=True, parents=True)
-                if cache_file_path.exists():
-                    return pickle.load(open(cache_file_path, "rb"))
-            mod = get_module_by_module_path(str(self.workspace_path / "model.py"))
-            model_cls = mod.model_cls
+            qtde = QTDockerEnv() if self.target_task.version == 1 else KGDockerEnv()
+            qtde.prepare()
 
-            if self.target_task.model_type == "Tabular":
-                input_shape = (batch_size, num_features)
-                m = model_cls(num_features=input_shape[1])
-                data = torch.full(input_shape, input_value)
-            elif self.target_task.model_type == "TimeSeries":
-                input_shape = (batch_size, num_features, num_timesteps)
-                m = model_cls(num_features=input_shape[1], num_timesteps=input_shape[2])
-                data = torch.full(input_shape, input_value)
-            elif self.target_task.model_type == "Graph":
-                node_feature = torch.randn(batch_size, num_features)
-                edge_index = torch.randint(0, batch_size, (2, num_edges))
-                m = model_cls(num_features=num_features)
-                data = (node_feature, edge_index)
-            else:
-                raise ValueError(f"Unsupported model type: {self.target_task.model_type}")
+            if self.target_task.version == 1:
+                dump_code = f"""
+MODEL_TYPE = "{self.target_task.model_type}"
+BATCH_SIZE = {batch_size}
+NUM_FEATURES = {num_features}
+NUM_TIMESTEPS = {num_timesteps}
+NUM_EDGES = {num_edges}
+INPUT_VALUE = {input_value}
+PARAM_INIT_VALUE = {param_init_value}
+{(Path(__file__).parent / 'model_execute_template_v1.txt').read_text()}
+"""
+            elif self.target_task.version == 2:
+                dump_code = (Path(__file__).parent / "model_execute_template_v2.txt").read_text()
 
-            # Initialize all parameters of `m` to `param_init_value`
-            for _, param in m.named_parameters():
-                param.data.fill_(param_init_value)
-
-            # Execute the model
-            if self.target_task.model_type == "Graph":
-                out = m(*data)
-            else:
-                out = m(data)
-
-            execution_model_output = out.cpu().detach()
-            execution_feedback_str = f"Execution successful, output tensor shape: {execution_model_output.shape}"
-
-            if MODEL_IMPL_SETTINGS.enable_execution_cache:
-                pickle.dump((execution_feedback_str, execution_model_output), open(cache_file_path, "wb"))
+            log, results = qtde.dump_python_code_run_and_get_results(
+                code=dump_code,
+                dump_file_names=["execution_feedback_str.pkl", "execution_model_output.pkl"],
+                local_path=str(self.workspace_path),
+                env={},
+                code_dump_file_py_name="model_test",
+            )
+            if results is None:
+                raise RuntimeError(f"Error in running the model code: {log}")
+            [execution_feedback_str, execution_model_output] = results
 
         except Exception as e:
             execution_feedback_str = f"Execution error: {e}\nTraceback: {traceback.format_exc()}"
             execution_model_output = None
 
-        code_path = self.workspace_path / f"model.py"
-        execution_feedback_str = execution_feedback_str.replace(str(code_path.parent.absolute()), r"/path/to").replace(
-            str(site.getsitepackages()[0]), r"/path/to/site-packages"
-        )
         if len(execution_feedback_str) > 2000:
             execution_feedback_str = (
                 execution_feedback_str[:1000] + "....hidden long error message...." + execution_feedback_str[-1000:]

rdagent/components/coder/model_coder/model_execute_template_v1.txt

@@ -0,0 +1,44 @@
+# MODEL_TYPE = "Tabular"
+# BATCH_SIZE = 32
+# NUM_FEATURES = 10
+# NUM_TIMESTEPS = 4
+# NUM_EDGES = 20
+# INPUT_VALUE = 1.0
+# PARAM_INIT_VALUE = 1.0
+
+import pickle
+
+import torch
+from model import model_cls
+
+if MODEL_TYPE == "Tabular":
+    input_shape = (BATCH_SIZE, NUM_FEATURES)
+    m = model_cls(num_features=input_shape[1])
+    data = torch.full(input_shape, INPUT_VALUE)
+elif MODEL_TYPE == "TimeSeries":
+    input_shape = (BATCH_SIZE, NUM_FEATURES, NUM_TIMESTEPS)
+    m = model_cls(num_features=input_shape[1], num_timesteps=input_shape[2])
+    data = torch.full(input_shape, INPUT_VALUE)
+elif MODEL_TYPE == "Graph":
+    node_feature = torch.randn(BATCH_SIZE, NUM_FEATURES)
+    edge_index = torch.randint(0, BATCH_SIZE, (2, NUM_EDGES))
+    m = model_cls(num_features=NUM_FEATURES)
+    data = (node_feature, edge_index)
+else:
+    raise ValueError(f"Unsupported model type: {MODEL_TYPE}")
+
+# Initialize all parameters of `m` to `param_init_value`
+for _, param in m.named_parameters():
+    param.data.fill_(PARAM_INIT_VALUE)
+
+# Execute the model
+if MODEL_TYPE == "Graph":
+    out = m(*data)
+else:
+    out = m(data)
+
+execution_model_output = out.cpu().detach().numpy()
+execution_feedback_str = f"Execution successful, output tensor shape: {execution_model_output.shape}"
+
+pickle.dump(execution_model_output, open("execution_model_output.pkl", "wb"))
+pickle.dump(execution_feedback_str, open("execution_feedback_str.pkl", "wb"))

rdagent/components/coder/model_coder/model_execute_template_v2.txt

@@ -0,0 +1,20 @@
+import os
+import pickle
+
+import numpy as np
+import pandas as pd
+import torch
+from model import fit, predict
+
+train_X = pd.DataFrame(np.random.randn(8, 30), columns=[f"{i}" for i in range(30)])
+train_y = pd.Series(np.random.randint(0, 2, 8))
+valid_X = pd.DataFrame(np.random.randn(8, 30), columns=[f"{i}" for i in range(30)])
+valid_y = pd.Series(np.random.randint(0, 2, 8))
+
+model = fit(train_X, train_y, valid_X, valid_y)
+execution_model_output = predict(model, valid_X).cpu().detach().numpy()
+
+execution_feedback_str = f"Execution successful, output numpy ndarray shape: {execution_model_output.shape}"
+
+pickle.dump(execution_model_output, open("execution_model_output.pkl", "wb"))
+pickle.dump(execution_feedback_str, open("execution_feedback_str.pkl", "wb"))

rdagent/components/coder/model_coder/prompts.yaml

@@ -15,7 +15,7 @@ extract_model_formulation_system: |-
             "hyperparameter_name_2": "value of hyperparameter 2",
             "hyperparameter_name_3": "value of hyperparameter 3"
         },
-        "model_type": "Tabular or TimeSeries or Graph"  # Should be one of "Tabular", "TimeSeries", or "Graph"
+        "model_type": "Tabular or TimeSeries or Graph or XGBoost"  # Should be one of "Tabular", "TimeSeries", "Graph", or "XGBoost"
     }
     }
     Eg. 
@@ -34,7 +34,7 @@ extract_model_formulation_system: |-
             "hyperparameter_name_2": "value of hyperparameter 2",
             "hyperparameter_name_3": "value of hyperparameter 3"
         },
-        "model_type": "Tabular or TimeSeries or Graph"  # Should be one of "Tabular", "TimeSeries", or "Graph"
+        "model_type": "Tabular or TimeSeries or Graph or RandomForest or XGBoost"  # If torch & Neural network models are required, the choice should be one of "Tabular", "TimeSeries", or "Graph" 
     }
     }
     such format content should be begin with ```json and end with ``` and the content should be in json format.
@@ -52,6 +52,16 @@ evolving_strategy_model_coder:
 
         Your must write your code based on your former latest attempt below which consists of your former code and code feedback, you should read the former attempt carefully and must not modify the right part of your former code.
 
+        {% if current_code is not none %}
+        User has write some code before. You should write the new code based on this code. Here is the latest code:
+        ```python
+        {{ current_code }}
+        ```
+        Your code should be very similar to the former code which means your code should be ninety more percent same as the former code! You should not modify the right part of the code.
+        {% else %}
+        User has not write any code before. You should write the new code from scratch.
+        {% endif %}
+
         {% if queried_former_failed_knowledge|length != 0 %}
         --------------Your former latest attempt:---------------
         =====Code to the former implementation=====
@@ -99,7 +109,7 @@ evaluator_code_feedback:
         The user will provide the source python code and the execution error message if execution failed.
         The user might provide you the ground truth code for you to provide the critic. You should not leak the ground truth code to the user in any form but you can use it to provide the critic.
 
-        User has also compared the output generated by the user's code and the ground truth code. The user will provide you some analyze result comparing two output. You may find some error in the code which caused the difference between the two output.
+        User has also compared the output generated by the user's code and the ground truth code. The user will provide you some analysis results comparing two output. You may find some error in the code which caused the difference between the two output.
 
         If the ground truth code is provided, your critic should only consider checking whether the user's code is align with the ground truth code since the ground truth is definitely correct.
         If the ground truth code is not provided, your critic should consider checking whether the user's code is reasonable and correct to the description and to the scenario.

rdagent/components/coder/model_coder/task_loader.py

@@ -115,8 +115,8 @@ class ModelExperimentLoaderFromDict(ModelTaskLoader):
 
 
 class ModelExperimentLoaderFromPDFfiles(ModelTaskLoader):
-    def load(self, file_or_folder_path: Path) -> dict:
-        docs_dict = load_and_process_pdfs_by_langchain(Path(file_or_folder_path))  # dict{file_path:content}
+    def load(self, file_or_folder_path: str) -> dict:
+        docs_dict = load_and_process_pdfs_by_langchain(file_or_folder_path)  # dict{file_path:content}
         model_dict = extract_model_from_docs(
             docs_dict
         )  # dict{file_name: dict{model_name: dict{description, formulation, variables}}}
@@ -124,14 +124,3 @@ class ModelExperimentLoaderFromPDFfiles(ModelTaskLoader):
             model_dict
         )  # dict {model_name: dict{description, formulation, variables}}
         return ModelExperimentLoaderFromDict().load(model_dict)
-
-
-def main(path="../test_doc"):
-    doc_dict = load_and_process_pdfs_by_langchain(Path(path))
-    print(doc_dict.keys())  # if you run code like "python -u", the print content will be truncated
-
-
-import fire
-
-if __name__ == "__main__":
-    fire.Fire(main)

rdagent/components/document_reader/document_reader.py

@@ -1,12 +1,14 @@
 from __future__ import annotations
 
+import io
 from pathlib import Path
 from typing import TYPE_CHECKING
 
 import fitz
+import requests
 from azure.ai.formrecognizer import DocumentAnalysisClient
 from azure.core.credentials import AzureKeyCredential
-from langchain.document_loaders import PyPDFDirectoryLoader, PyPDFLoader
+from langchain_community.document_loaders import PyPDFDirectoryLoader, PyPDFLoader
 from PIL import Image
 
 if TYPE_CHECKING:
@@ -15,7 +17,7 @@ if TYPE_CHECKING:
 from rdagent.core.conf import RD_AGENT_SETTINGS
 
 
-def load_documents_by_langchain(path: Path) -> list:
+def load_documents_by_langchain(path: str) -> list:
     """Load documents from the specified path.
 
     Args:
@@ -24,7 +26,10 @@ def load_documents_by_langchain(path: Path) -> list:
     Returns:
         list: A list of loaded documents.
     """
-    loader = PyPDFDirectoryLoader(str(path), silent_errors=True) if path.is_dir() else PyPDFLoader(str(path))
+    if Path(path).is_dir():
+        loader = PyPDFDirectoryLoader(path, silent_errors=True)
+    else:
+        loader = PyPDFLoader(path)
     return loader.load()
 
 
@@ -41,7 +46,10 @@ def process_documents_by_langchain(docs: list[Document]) -> dict[str, str]:
     content_dict = {}
 
     for doc in docs:
-        doc_name = str(Path(doc.metadata["source"]).resolve())
+        if Path(doc.metadata["source"]).exists():
+            doc_name = str(Path(doc.metadata["source"]).resolve())
+        else:
+            doc_name = doc.metadata["source"]
         doc_content = doc.page_content
 
         if doc_name not in content_dict:
@@ -52,7 +60,7 @@ def process_documents_by_langchain(docs: list[Document]) -> dict[str, str]:
     return content_dict
 
 
-def load_and_process_pdfs_by_langchain(path: Path) -> dict[str, str]:
+def load_and_process_pdfs_by_langchain(path: str) -> dict[str, str]:
     return process_documents_by_langchain(load_documents_by_langchain(path))
 
 
@@ -101,8 +109,11 @@ def load_and_process_pdfs_by_azure_document_intelligence(path: Path) -> dict[str
     return content_dict
 
 
-def extract_first_page_screenshot_from_pdf(pdf_path: Path) -> Image:
-    doc = fitz.open(pdf_path)
+def extract_first_page_screenshot_from_pdf(pdf_path: str) -> Image:
+    if not Path(pdf_path).exists():
+        doc = fitz.open(stream=io.BytesIO(requests.get(pdf_path).content), filetype="pdf")
+    else:
+        doc = fitz.open(pdf_path)
     page = doc.load_page(0)
     pix = page.get_pixmap()
     image = Image.frombytes("RGB", [pix.width, pix.height], pix.samples)

rdagent/components/knowledge_management/graph.py

@@ -12,6 +12,7 @@ from rdagent.components.knowledge_management.vector_base import (
     VectorBase,
     cosine,
 )
+from rdagent.core.knowledge_base import KnowledgeBase
 from rdagent.oai.llm_utils import APIBackend
 
 Node = KnowledgeMetaData
@@ -47,14 +48,14 @@ class UndirectedNode(Node):
         )
 
 
-class Graph:
+class Graph(KnowledgeBase):
     """
     base Graph class for Knowledge Graph Search
     """
 
     def __init__(self, path: str | Path | None = None) -> None:
-        self.path = path
         self.nodes = {}
+        super().__init__(path=path)
 
     def size(self) -> int:
         return len(self.nodes)
@@ -77,22 +78,6 @@ class Graph:
                 return node
         return None
 
-    @classmethod
-    def load(cls: type[Graph], path: str | Path) -> Graph:
-        """use pickle as the default load method"""
-        path = path if isinstance(path, Path) else Path(path)
-        if not path.exists():
-            return cls(path=path)
-
-        with path.open("rb") as f:
-            return pickle.load(f)
-
-    def save(self, path: str | Path) -> None:
-        """use pickle as the default save method"""
-        Path.mkdir(path.parent, exist_ok=True)
-        with path.open("wb") as f:
-            pickle.dump(self, f)
-
     @staticmethod
     def batch_embedding(nodes: list[Node]) -> list[Node]:
         contents = [node.content for node in nodes]
@@ -119,8 +104,8 @@ class UndirectedGraph(Graph):
     """
 
     def __init__(self, path: str | Path | None = None) -> None:
-        super().__init__(path=path)
         self.vector_base: VectorBase = PDVectorBase()
+        super().__init__(path=path)
 
     def __str__(self) -> str:
         return f"UndirectedGraph(nodes={self.nodes})"
@@ -174,16 +159,6 @@ class UndirectedGraph(Graph):
 
             node.add_neighbor(neighbor)
 
-    @classmethod
-    def load(cls: type[UndirectedGraph], path: str | Path) -> UndirectedGraph:
-        """use pickle as the default load method"""
-        path = path if isinstance(path, Path) else Path(path)
-        if not path.exists():
-            return cls(path=path)
-
-        with path.open("rb") as f:
-            return pickle.load(f)
-
     def add_nodes(self, node: UndirectedNode, neighbors: list[UndirectedNode]) -> None:
         if not neighbors:
             self.add_node(node)

rdagent/components/knowledge_management/vector_base.py

@@ -5,6 +5,7 @@ from typing import List, Tuple, Union
 import pandas as pd
 from scipy.spatial.distance import cosine
 
+from rdagent.core.knowledge_base import KnowledgeBase
 from rdagent.log import rdagent_logger as logger
 from rdagent.oai.llm_utils import APIBackend
 
@@ -68,14 +69,11 @@ def contents_to_documents(contents: List[str], label: str = None) -> List[Docume
     return docs
 
 
-class VectorBase:
+class VectorBase(KnowledgeBase):
     """
     This class is used for handling vector storage and query
     """
 
-    def __init__(self, vector_df_path: Union[str, Path] = None, **kwargs):
-        pass
-
     def add(self, document: Union[Document, List[Document]]):
         """
         add new node to vector_df
@@ -104,28 +102,15 @@ class VectorBase:
         """
         pass
 
-    def load(self, **kwargs):
-        """load vector_df"""
-
-    def save(self, **kwargs):
-        """save vector_df"""
-
 
 class PDVectorBase(VectorBase):
     """
     Implement of VectorBase using Pandas
     """
 
-    def __init__(self, vector_df_path: Union[str, Path] = None):
-        super().__init__(vector_df_path)
-
-        if vector_df_path:
-            try:
-                self.vector_df = self.load(vector_df_path)
-            except FileNotFoundError:
-                self.vector_df = pd.DataFrame(columns=["id", "label", "content", "embedding"])
-        else:
-            self.vector_df = pd.DataFrame(columns=["id", "label", "content", "embedding"])
+    def __init__(self, path: Union[str, Path] = None):
+        self.vector_df = pd.DataFrame(columns=["id", "label", "content", "embedding"])
+        super().__init__(path)
 
     def shape(self):
         return self.vector_df.shape
@@ -196,10 +181,3 @@ class PDVectorBase(VectorBase):
         for _, similar_docs in most_similar_docs.iterrows():
             docs.append(Document().from_dict(similar_docs.to_dict()))
         return docs, searched_similarities.to_list()
-
-    def load(self, vector_df_path, **kwargs):
-        vector_df = pd.read_pickle(vector_df_path)
-        return vector_df
-
-    def save(self, vector_df_path, **kwargs):
-        self.vector_df.to_pickle(vector_df_path)

rdagent/components/proposal/model_proposal.py

@@ -39,7 +39,7 @@ class ModelHypothesisGen(HypothesisGen):
             Environment(undefined=StrictUndefined)
             .from_string(ModelHypothesisGen.prompts["hypothesis_gen"]["system_prompt"])
             .render(
-                targets="model",
+                targets="model tuning",
                 scenario=self.scen.get_scenario_all_desc(),
                 hypothesis_output_format=context_dict["hypothesis_output_format"],
                 hypothesis_specification=context_dict["hypothesis_specification"],
@@ -49,8 +49,7 @@ class ModelHypothesisGen(HypothesisGen):
             Environment(undefined=StrictUndefined)
             .from_string(ModelHypothesisGen.prompts["hypothesis_gen"]["user_prompt"])
             .render(
-                targets="model",
-                hypothesis_and_feedback=context_dict["hypothesis_and_feedback"],
+                targets="model tuning",
                 RAG=context_dict["RAG"],
             )
         )
@@ -82,7 +81,7 @@ class ModelHypothesis2Experiment(Hypothesis2Experiment[ModelExperiment]):
             Environment(undefined=StrictUndefined)
             .from_string(ModelHypothesis2Experiment.prompts["hypothesis2experiment"]["system_prompt"])
             .render(
-                targets="model",
+                targets="feature engineering and model building",
                 scenario=trace.scen.get_scenario_all_desc(),
                 experiment_output_format=context["experiment_output_format"],
             )
@@ -91,7 +90,7 @@ class ModelHypothesis2Experiment(Hypothesis2Experiment[ModelExperiment]):
             Environment(undefined=StrictUndefined)
             .from_string(ModelHypothesis2Experiment.prompts["hypothesis2experiment"]["user_prompt"])
             .render(
-                targets="model",
+                targets="feature engineering and model building",
                 target_hypothesis=context["target_hypothesis"],
                 hypothesis_and_feedback=context["hypothesis_and_feedback"],
                 target_list=context["target_list"],

rdagent/components/proposal/prompts.yaml

@@ -1,21 +1,25 @@
 hypothesis_gen:
   system_prompt: |-
-    The user is trying to generate new hypothesis on the {{targets}} in data-driven research and development.
-    The {{targets}} are used in a certain scenario, the scenario is as follows:
-    {{ scenario }}
-    The user has made several hypothesis on this scenario and did several evaluation on them. The user will provide this information to you. Check if a new hypothesis has already been proposed. If it is already generated and you agree with it, just use it. If you don't agree, generate a better one.
-    To help you generate new hypothesis, the user has prepared some additional information for you. You should use this information to help generate new {{targets}}.
-    Please generate the output following the format and specifications below:
+    The user is working on generating new hypotheses for the {{targets}} in a data-driven research and development process. 
+    The {{targets}} are used in the following scenario:
+    {{scenario}}
+    The user has already proposed several hypotheses and conducted evaluations on them. This information will be provided to you. Your task is to check whether a similar hypothesis has already been generated. 
+    If one exists and you agree with it, feel free to use it. If you disagree, please generate an improved version.
+    {% if hypothesis_specification %}
+    To assist you in formulating new hypotheses, the user has provided some additional information: {{hypothesis_specification}}.
+    **Important:** If the hypothesis_specification outlines the next steps you need to follow, ensure you adhere to those instructions.
+    {% endif %}
+    Please generate the output using the following format and specifications:
     {{ hypothesis_output_format }}
-    Here are the specifications: {{ hypothesis_specification }}
 
   user_prompt: |-
-    If it is not the first round, then the user has made several hypothesis on this scenario and did several evaluation on them.
-    The former hypothesis and the corresponding feedbacks are as follows (focus on the last one & the new hypothesis that it provides and reasoning to see if you agree):
-    {{ hypothesis_and_feedback }}
-    To help you generate new {{targets}}, we have prepared the following information for you:
-    {{ RAG }}
-    Please generate the new hypothesis based on the information above. Also generate the relevant keys for the reasoning and the distilled knowledge that follows. For those keys, in particular for knowledge, explain in the context of the specific scenario to build up domain knowledge in the specific field rather than genearl knowledge.
+    {% if RAG %}
+    To assist you in generating new {{targets}}, we have provided the following information: {{RAG}}.
+    **Note:** The provided RAG is for reference only. 
+    You must carefully assess whether the RAG aligns with the {{targets}}. 
+    If it does not, it should not be used. Exercise caution and make your own judgment.
+    {% endif %}
+    Also generate the relevant keys for the reasoning and the distilled knowledge that follows. For those keys, in particular for knowledge, explain in the context of the specific scenario to build up domain knowledge in the specific field rather than general knowledge.
 
 hypothesis2experiment:
   system_prompt: |-
@@ -36,8 +40,4 @@ hypothesis2experiment:
     {{ target_hypothesis }}
     The former hypothesis and the corresponding feedbacks are as follows:
     {{ hypothesis_and_feedback }}
-    The former proposed {{targets}} on similar hypothesis are as follows:
-    {{ target_list }}
-    To help you generate new {{targets}}, we have prepared the following information for you:
-    {{ RAG }}
     Please generate the new {{targets}} based on the information above.

rdagent/components/runner/__init__.py

@@ -1,8 +1,7 @@
 import pickle
 from pathlib import Path
-from typing import Tuple
+from typing import Any, Tuple
 
-from rdagent.components.runner.conf import RUNNER_SETTINGS
 from rdagent.core.developer import Developer
 from rdagent.core.experiment import ASpecificExp, Experiment
 from rdagent.oai.llm_utils import md5_hash
@@ -18,16 +17,8 @@ class CachedRunner(Developer[ASpecificExp]):
         task_info_str = "\n".join(task_info_list)
         return md5_hash(task_info_str)
 
-    def get_cache_result(self, exp: Experiment) -> Tuple[bool, object]:
-        task_info_key = self.get_cache_key(exp)
-        Path(RUNNER_SETTINGS.cache_path).mkdir(parents=True, exist_ok=True)
-        cache_path = Path(RUNNER_SETTINGS.cache_path) / f"{task_info_key}.pkl"
-        if cache_path.exists():
-            return True, pickle.load(open(cache_path, "rb"))
-        else:
-            return False, None
-
-    def dump_cache_result(self, exp: Experiment, result: object):
-        task_info_key = self.get_cache_key(exp)
-        cache_path = Path(RUNNER_SETTINGS.cache_path) / f"{task_info_key}.pkl"
-        pickle.dump(result, open(cache_path, "wb"))
+    def assign_cached_result(self, exp: Experiment, cached_res: Experiment) -> Experiment:
+        if exp.based_experiments and exp.based_experiments[-1].result is None:
+            exp.based_experiments[-1].result = cached_res.based_experiments[-1].result
+        exp.result = cached_res.result
+        return exp

rdagent/components/runner/conf.py

@@ -1,22 +0,0 @@
-from __future__ import annotations
-
-from pathlib import Path
-
-from dotenv import load_dotenv
-from pydantic_settings import BaseSettings
-
-# make sure that env variable is loaded while calling Config()
-load_dotenv(verbose=True, override=True)
-
-from pydantic_settings import BaseSettings
-
-
-class RunnerSettings(BaseSettings):
-    class Config:
-        env_prefix = "RUNNER_"  # Use MODEL_CODER_ as prefix for environment variables
-
-    cache_result: bool = True  # whether to cache the result of the docker execution
-    cache_path: str = str(Path.cwd() / "runner_cache/")  # the path to store the cache
-
-
-RUNNER_SETTINGS = RunnerSettings()

rdagent/components/workflow/conf.py

@@ -14,6 +14,8 @@ class BasePropSetting(BaseSettings):
     """
 
     scen: str = ""
+    knowledge_base: str = ""
+    knowledge_base_path: str = ""
     hypothesis_gen: str = ""
     hypothesis2experiment: str = ""
     coder: str = ""

rdagent/components/workflow/rd_loop.py

@@ -3,6 +3,7 @@ Model workflow with session control
 It is from `rdagent/app/qlib_rd_loop/model.py` and try to replace `rdagent/app/qlib_rd_loop/RDAgent.py`
 """
 
+import time
 from typing import Any
 
 from rdagent.components.workflow.conf import BasePropSetting
@@ -16,10 +17,12 @@ from rdagent.core.proposal import (
 from rdagent.core.scenario import Scenario
 from rdagent.core.utils import import_class
 from rdagent.log import rdagent_logger as logger
+from rdagent.log.time import measure_time
 from rdagent.utils.workflow import LoopBase, LoopMeta
 
 
 class RDLoop(LoopBase, metaclass=LoopMeta):
+    @measure_time
     def __init__(self, PROP_SETTING: BasePropSetting):
         with logger.tag("init"):
             scen: Scenario = import_class(PROP_SETTING.scen)()
@@ -41,30 +44,35 @@ class RDLoop(LoopBase, metaclass=LoopMeta):
             self.trace = Trace(scen=scen)
             super().__init__()
 
+    @measure_time
     def propose(self, prev_out: dict[str, Any]):
         with logger.tag("r"):  # research
             hypothesis = self.hypothesis_gen.gen(self.trace)
             logger.log_object(hypothesis, tag="hypothesis generation")
         return hypothesis
 
+    @measure_time
     def exp_gen(self, prev_out: dict[str, Any]):
         with logger.tag("r"):  # research
             exp = self.hypothesis2experiment.convert(prev_out["propose"], self.trace)
             logger.log_object(exp.sub_tasks, tag="experiment generation")
         return exp
 
+    @measure_time
     def coding(self, prev_out: dict[str, Any]):
         with logger.tag("d"):  # develop
             exp = self.coder.develop(prev_out["exp_gen"])
             logger.log_object(exp.sub_workspace_list, tag="coder result")
         return exp
 
+    @measure_time
     def running(self, prev_out: dict[str, Any]):
         with logger.tag("ef"):  # evaluate and feedback
             exp = self.runner.develop(prev_out["coding"])
             logger.log_object(exp, tag="runner result")
         return exp
 
+    @measure_time
     def feedback(self, prev_out: dict[str, Any]):
         feedback = self.summarizer.generate_feedback(prev_out["running"], prev_out["propose"], self.trace)
         with logger.tag("ef"):  # evaluate and feedback

rdagent/core/conf.py

@@ -2,15 +2,11 @@ from __future__ import annotations
 
 from pathlib import Path
 
-from dotenv import load_dotenv
 from pydantic_settings import BaseSettings
 
 # TODO: use pydantic for other modules in Qlib
 # from pydantic_settings import BaseSettings
 
-# make sure that env variable is loaded while calling Config()
-load_dotenv(verbose=True, override=True)
-
 
 class RDAgentSettings(BaseSettings):
     # TODO: (xiao) I think LLMSetting may be a better name.
@@ -18,86 +14,14 @@ class RDAgentSettings(BaseSettings):
     # Log configs
     # TODO: (xiao) think it can be a separate config.
     log_trace_path: str | None = None
-    log_llm_chat_content: bool = True
-
-    use_azure: bool = False
-    use_azure_token_provider: bool = False
-    managed_identity_client_id: str | None = None
-    max_retry: int = 10
-    retry_wait_seconds: int = 1
-    dump_chat_cache: bool = False
-    use_chat_cache: bool = False
-    dump_embedding_cache: bool = False
-    use_embedding_cache: bool = False
-    prompt_cache_path: str = str(Path.cwd() / "prompt_cache.db")
-    session_cache_folder_location: str = str(Path.cwd() / "session_cache_folder/")
-    max_past_message_include: int = 10
-
-    # Chat configs
-    openai_api_key: str = ""  # TODO: simplify the key design.
-    chat_openai_api_key: str = ""
-    chat_azure_api_base: str = ""
-    chat_azure_api_version: str = ""
-    chat_model: str = ""
-    chat_max_tokens: int = 3000
-    chat_temperature: float = 0.5
-    chat_stream: bool = True
-    chat_seed: int | None = None
-    chat_frequency_penalty: float = 0.0
-    chat_presence_penalty: float = 0.0
-    chat_token_limit: int = (
-        100000  # 100000 is the maximum limit of gpt4, which might increase in the future version of gpt
-    )
-    default_system_prompt: str = "You are an AI assistant who helps to answer user's questions."
-
-    # Embedding configs
-    embedding_openai_api_key: str = ""
-    embedding_azure_api_base: str = ""
-    embedding_azure_api_version: str = ""
-    embedding_model: str = ""
-
-    # offline llama2 related config
-    use_llama2: bool = False
-    llama2_ckpt_dir: str = "Llama-2-7b-chat"
-    llama2_tokenizer_path: str = "Llama-2-7b-chat/tokenizer.model"
-    llams2_max_batch_size: int = 8
 
     # azure document intelligence configs
     azure_document_intelligence_key: str = ""
     azure_document_intelligence_endpoint: str = ""
-
-    # server served endpoints
-    use_gcr_endpoint: bool = False
-    gcr_endpoint_type: str = "llama2_70b"  # or "llama3_70b", "phi2", "phi3_4k", "phi3_128k"
-
-    llama2_70b_endpoint: str = ""
-    llama2_70b_endpoint_key: str = ""
-    llama2_70b_endpoint_deployment: str = ""
-
-    llama3_70b_endpoint: str = ""
-    llama3_70b_endpoint_key: str = ""
-    llama3_70b_endpoint_deployment: str = ""
-
-    phi2_endpoint: str = ""
-    phi2_endpoint_key: str = ""
-    phi2_endpoint_deployment: str = ""
-
-    phi3_4k_endpoint: str = ""
-    phi3_4k_endpoint_key: str = ""
-    phi3_4k_endpoint_deployment: str = ""
-
-    phi3_128k_endpoint: str = ""
-    phi3_128k_endpoint_key: str = ""
-    phi3_128k_endpoint_deployment: str = ""
-
-    gcr_endpoint_temperature: float = 0.7
-    gcr_endpoint_top_p: float = 0.9
-    gcr_endpoint_do_sample: bool = False
-    gcr_endpoint_max_token: int = 100
-
     # factor extraction conf
-    max_input_duplicate_factor_group: int = 600
+    max_input_duplicate_factor_group: int = 300
     max_output_duplicate_factor_group: int = 20
+    max_kmeans_group_number: int = 40
 
     # workspace conf
     workspace_path: Path = Path.cwd() / "git_ignore_folder" / "RD-Agent_workspace"
@@ -105,5 +29,15 @@ class RDAgentSettings(BaseSettings):
     # multi processing conf
     multi_proc_n: int = 1
 
+    # pickle cache conf
+    cache_with_pickle: bool = True  # whether to use pickle cache
+    pickle_cache_folder_path_str: str = str(
+        Path.cwd() / "pickle_cache/",
+    )  # the path of the folder to store the pickle cache
+    use_file_lock: bool = (
+        True  # when calling the function with same parameters, whether to use file lock to avoid
+        # executing the function multiple times
+    )
+
 
 RD_AGENT_SETTINGS = RDAgentSettings()

rdagent/core/evolving_agent.py

@@ -78,6 +78,8 @@ class RAGEvoAgent(EvoAgent):
             )
             # TODO: Due to design issues, we have chosen to ignore this mypy error.
             logger.log_object(evo.sub_workspace_list, tag="evolving code")  # type: ignore[attr-defined]
+            for sw in evo.sub_workspace_list:  # type: ignore[attr-defined]
+                logger.info(f"evolving code workspace: {sw}")
 
             # 4. Pack evolve results
             es = EvoStep(evo, queried_knowledge)

rdagent/core/evolving_framework.py

@@ -5,6 +5,8 @@ from abc import ABC, abstractmethod
 from dataclasses import dataclass
 from typing import TYPE_CHECKING, Any
 
+from rdagent.core.knowledge_base import KnowledgeBase
+
 if TYPE_CHECKING:
     from rdagent.core.evaluation import Feedback
     from rdagent.core.scenario import Scenario
@@ -18,7 +20,7 @@ class QueriedKnowledge:
     pass
 
 
-class KnowledgeBase(ABC):
+class EvolvingKnowledgeBase(KnowledgeBase):
     @abstractmethod
     def query(
         self,
@@ -78,7 +80,7 @@ class EvolvingStrategy(ABC):
 class RAGStrategy(ABC):
     """Retrieval Augmentation Generation Strategy"""
 
-    def __init__(self, knowledgebase: KnowledgeBase) -> None:
+    def __init__(self, knowledgebase: EvolvingKnowledgeBase) -> None:
         self.knowledgebase = knowledgebase
 
     @abstractmethod

rdagent/core/experiment.py

@@ -1,11 +1,14 @@
 from __future__ import annotations
 
+import os
+import platform
 import shutil
 import uuid
 from abc import ABC, abstractmethod
+from collections.abc import Sequence
 from copy import deepcopy
 from pathlib import Path
-from typing import Any, Generic, Sequence, TypeVar
+from typing import Any, Generic, TypeVar
 
 from rdagent.core.conf import RD_AGENT_SETTINGS
 
@@ -15,6 +18,15 @@ This file contains the all the class about organizing the task in RD-Agent.
 
 
 class Task(ABC):
+    def __init__(self, name: str, version: int = 1) -> None:
+        """
+        The version of the task, default is 1
+        Because qlib tasks execution and kaggle tasks execution are different, we need to distinguish them.
+        TODO: We may align them in the future.
+        """
+        self.version = version
+        self.name = name
+
     @abstractmethod
     def get_task_information(self) -> str:
         """
@@ -103,6 +115,19 @@ class FBWorkspace(Workspace):
         """
         self.workspace_path.mkdir(parents=True, exist_ok=True)
 
+    @staticmethod
+    def link_all_files_in_folder_to_workspace(data_path: Path, workspace_path: Path) -> None:
+        data_path = Path(data_path).absolute()  # in case of relative path that will be invalid when we change cwd.
+        workspace_path = Path(workspace_path)
+        for data_file_path in data_path.iterdir():
+            workspace_data_file_path = workspace_path / data_file_path.name
+            if workspace_data_file_path.exists():
+                workspace_data_file_path.unlink()
+            if platform.system() == "Linux":
+                os.symlink(data_file_path, workspace_data_file_path)
+            if platform.system() == "Windows":
+                os.link(data_file_path, workspace_data_file_path)
+
     def inject_code(self, **files: str) -> None:
         """
         Inject the code into the folder.
@@ -113,6 +138,9 @@ class FBWorkspace(Workspace):
         self.prepare()
         for k, v in files.items():
             self.code_dict[k] = v
+            target_file_path = self.workspace_path / k
+            if not target_file_path.parent.exists():
+                target_file_path.parent.mkdir(parents=True, exist_ok=True)
             with Path.open(self.workspace_path / k, "w") as f:
                 f.write(v)
 
@@ -129,9 +157,10 @@ class FBWorkspace(Workspace):
         """
         Load the workspace from the folder
         """
-        for file_path in folder_path.iterdir():
-            if file_path.suffix in {".py", ".yaml"}:
-                self.inject_code(**{file_path.name: file_path.read_text()})
+        for file_path in folder_path.rglob("*"):
+            if file_path.suffix in (".py", ".yaml", ".md"):
+                relative_path = file_path.relative_to(folder_path)
+                self.inject_code(**{str(relative_path): file_path.read_text()})
 
     def copy(self) -> FBWorkspace:
         """
@@ -143,7 +172,7 @@ class FBWorkspace(Workspace):
         """
         Clear the workspace
         """
-        shutil.rmtree(self.workspace_path)
+        shutil.rmtree(self.workspace_path, ignore_errors=True)
         self.code_dict = {}
 
     def execute(self) -> object | None:
@@ -154,20 +183,32 @@ class FBWorkspace(Workspace):
         self.inject_code(**self.code_dict)
         return None
 
+    def __str__(self) -> str:
+        return f"Workspace[{self.workspace_path=}" + (
+            "]" if self.target_task is None else f",{self.target_task.name=}]"
+        )
+
 
 ASpecificWSForExperiment = TypeVar("ASpecificWSForExperiment", bound=Workspace)
 ASpecificWSForSubTasks = TypeVar("ASpecificWSForSubTasks", bound=Workspace)
 
 
-class Experiment(ABC, Generic[ASpecificTask, ASpecificWSForExperiment, ASpecificWSForSubTasks]):
+class Experiment(
+    ABC,
+    Generic[ASpecificTask, ASpecificWSForExperiment, ASpecificWSForSubTasks],
+):
     """
     The experiment is a sequence of tasks and the implementations of the tasks after generated by the Developer.
     """
 
-    def __init__(self, sub_tasks: Sequence[ASpecificTask]) -> None:
+    def __init__(
+        self,
+        sub_tasks: Sequence[ASpecificTask],
+        based_experiments: Sequence[ASpecificWSForExperiment] = [],
+    ) -> None:
         self.sub_tasks = sub_tasks
         self.sub_workspace_list: list[ASpecificWSForSubTasks | None] = [None] * len(self.sub_tasks)
-        self.based_experiments: Sequence[ASpecificWSForExperiment] = []
+        self.based_experiments: Sequence[ASpecificWSForExperiment] = based_experiments
         self.result: object = None  # The result of the experiment, can be different types in different scenarios.
         self.experiment_workspace: ASpecificWSForExperiment | None = None
 

rdagent/core/knowledge_base.py

@@ -0,0 +1,27 @@
+from pathlib import Path
+
+import dill as pickle  # type: ignore[import-untyped]
+
+from rdagent.log import rdagent_logger as logger
+
+
+class KnowledgeBase:
+    def __init__(self, path: str | Path | None = None) -> None:
+        self.path = Path(path) if path else None
+        self.load()
+
+    def load(self) -> None:
+        if self.path is not None and self.path.exists():
+            with self.path.open("rb") as f:
+                loaded = pickle.load(f)
+                if isinstance(loaded, dict):
+                    self.__dict__.update(loaded)
+                else:
+                    self.__dict__.update(loaded.__dict__)
+
+    def dump(self) -> None:
+        if self.path is not None:
+            self.path.parent.mkdir(parents=True, exist_ok=True)
+            pickle.dump(self.__dict__, self.path.open("wb"))
+        else:
+            logger.warning("KnowledgeBase path is not set, dump failed.")

rdagent/core/prompts.py

@@ -1,12 +1,11 @@
-from pathlib import Path  # noqa: I001
-from typing import Dict
+from pathlib import Path
 
 import yaml
 
 from rdagent.core.utils import SingletonBaseClass
 
 
-class Prompts(SingletonBaseClass, Dict[str, str]):
+class Prompts(SingletonBaseClass, dict[str, str]):
     def __init__(self, file_path: Path) -> None:
         super().__init__()
         with file_path.open(encoding="utf8") as file:

rdagent/core/proposal.py

@@ -9,6 +9,7 @@ from typing import TYPE_CHECKING, Generic, TypeVar
 
 from rdagent.core.evaluation import Feedback
 from rdagent.core.experiment import ASpecificExp, Experiment
+from rdagent.core.knowledge_base import KnowledgeBase
 from rdagent.core.scenario import Scenario
 
 if TYPE_CHECKING:
@@ -83,12 +84,14 @@ Reason: {self.reason}"""
 
 
 ASpecificScen = TypeVar("ASpecificScen", bound=Scenario)
+ASpecificKB = TypeVar("ASpecificKB", bound=KnowledgeBase)
 
 
-class Trace(Generic[ASpecificScen]):
-    def __init__(self, scen: ASpecificScen) -> None:
+class Trace(Generic[ASpecificScen, ASpecificKB]):
+    def __init__(self, scen: ASpecificScen, knowledge_base: ASpecificKB | None = None) -> None:
         self.scen: ASpecificScen = scen
         self.hist: list[tuple[Hypothesis, Experiment, HypothesisFeedback]] = []
+        self.knowledge_base: ASpecificKB | None = knowledge_base
 
     def get_sota_hypothesis_and_experiment(self) -> tuple[Hypothesis | None, Experiment | None]:
         """Access the last experiment result, sub-task, and the corresponding hypothesis."""

rdagent/core/scenario.py

@@ -1,5 +1,7 @@
 from abc import ABC, abstractmethod
 
+from rdagent.core.experiment import Task
+
 
 class Scenario(ABC):
     @property
@@ -7,10 +9,21 @@ class Scenario(ABC):
     def background(self) -> str:
         """Background information"""
 
+    # TODO: We have to change all the sub classes to override get_source_data_desc instead of `source_data`
+    def get_source_data_desc(self, task: Task | None = None) -> str:  # noqa: ARG002
+        """
+        Source data description
+
+        The choice of data may vary based on the specific task at hand.
+        """
+        return ""
+
     @property
-    @abstractmethod
     def source_data(self) -> str:
-        """Source data description"""
+        """
+        A convenient shortcut for describing source data
+        """
+        return self.get_source_data_desc()
 
     @property
     @abstractmethod
@@ -33,8 +46,12 @@ class Scenario(ABC):
         """Rich style description to present"""
 
     @abstractmethod
-    def get_scenario_all_desc(self) -> str:
-        """Combine all the description together"""
+    def get_scenario_all_desc(self, task: Task | None = None) -> str:
+        """
+        Combine all descriptions together
+
+        The scenario description varies based on the task being performed.
+        """
 
     @property
     def experiment_setting(self) -> str | None:

rdagent/core/utils.py

@@ -1,14 +1,21 @@
 from __future__ import annotations
 
+import functools
 import importlib
 import json
 import multiprocessing as mp
 import pickle
+import random
 from collections.abc import Callable
+from pathlib import Path
 from typing import Any, ClassVar, NoReturn, cast
 
+from filelock import FileLock
 from fuzzywuzzy import fuzz  # type: ignore[import-untyped]
 
+from rdagent.core.conf import RD_AGENT_SETTINGS
+from rdagent.oai.llm_conf import LLM_SETTINGS
+
 
 class RDAgentException(Exception):  # noqa: N818
     pass
@@ -30,7 +37,7 @@ class SingletonBaseClass:
             raise RDAgentException(exception_message)
         class_name = [(-1, f"{cls.__module__}.{cls.__name__}")]
         args_l = [(i, args[i]) for i in args]
-        kwargs_l = list(sorted(kwargs.items()))
+        kwargs_l = sorted(kwargs.items())
         all_args = class_name + args_l + kwargs_l
         kwargs_hash = hash(tuple(all_args))
         if kwargs_hash not in cls._instance_dict:
@@ -81,11 +88,48 @@ def import_class(class_path: str) -> Any:
     return getattr(module, class_name)
 
 
+class CacheSeedGen:
+    """
+    It is a global seed generator to generate a sequence of seeds.
+    This will support the feature `use_auto_chat_cache_seed_gen` claim
+
+    NOTE:
+    - This seed is specifically for the cache and is different from a regular seed.
+    - If the cache is removed, setting the same seed will not produce the same QA trace.
+    """
+
+    def __init__(self) -> None:
+        self.set_seed(LLM_SETTINGS.init_chat_cache_seed)
+
+    def set_seed(self, seed: int) -> None:
+        random.seed(seed)
+
+    def get_next_seed(self) -> int:
+        """generate next random int"""
+        return random.randint(0, 10000)  # noqa: S311
+
+
+LLM_CACHE_SEED_GEN = CacheSeedGen()
+
+
+def _subprocess_wrapper(f: Callable, seed: int, args: list) -> Any:
+    """
+    It is a function wrapper. To ensure the subprocess has a fixed start seed.
+    """
+
+    LLM_CACHE_SEED_GEN.set_seed(seed)
+    return f(*args)
+
+
 def multiprocessing_wrapper(func_calls: list[tuple[Callable, tuple]], n: int) -> list:
     """It will use multiprocessing to call the functions in func_calls with the given parameters.
     The results equals to `return  [f(*args) for f, args in func_calls]`
     It will not call multiprocessing if `n=1`
 
+    NOTE:
+    We coooperate with chat_cache_seed feature
+    We ensure get the same seed trace even we have multiple number of seed
+
     Parameters
     ----------
     func_calls : List[Tuple[Callable, Tuple]]
@@ -100,6 +144,58 @@ def multiprocessing_wrapper(func_calls: list[tuple[Callable, tuple]], n: int) ->
     """
     if n == 1:
         return [f(*args) for f, args in func_calls]
-    with mp.Pool(processes=n) as pool:
-        results = [pool.apply_async(f, args) for f, args in func_calls]
+
+    with mp.Pool(processes=max(1, min(n, len(func_calls)))) as pool:
+        results = [
+            pool.apply_async(_subprocess_wrapper, args=(f, LLM_CACHE_SEED_GEN.get_next_seed(), args))
+            for f, args in func_calls
+        ]
         return [result.get() for result in results]
+
+
+def cache_with_pickle(hash_func: Callable, post_process_func: Callable | None = None) -> Callable:
+    """
+    This decorator will cache the return value of the function with pickle.
+    The cache key is generated by the hash_func. The hash function returns a string or None.
+    If it returns None, the cache will not be used. The cache will be stored in the folder
+    specified by RD_AGENT_SETTINGS.pickle_cache_folder_path_str with name hash_key.pkl.
+    The post_process_func will be called with the original arguments and the cached result
+    to give each caller a chance to process the cached result. The post_process_func should
+    return the final result.
+    """
+
+    def cache_decorator(func: Callable) -> Callable:
+        @functools.wraps(func)
+        def cache_wrapper(*args: Any, **kwargs: Any) -> Any:
+            if not RD_AGENT_SETTINGS.cache_with_pickle:
+                return func(*args, **kwargs)
+
+            target_folder = Path(RD_AGENT_SETTINGS.pickle_cache_folder_path_str) / f"{func.__module__}.{func.__name__}"
+            target_folder.mkdir(parents=True, exist_ok=True)
+            hash_key = hash_func(*args, **kwargs)
+
+            if hash_key is None:
+                return func(*args, **kwargs)
+
+            cache_file = target_folder / f"{hash_key}.pkl"
+            lock_file = target_folder / f"{hash_key}.lock"
+
+            if cache_file.exists():
+                with cache_file.open("rb") as f:
+                    cached_res = pickle.load(f)
+                return post_process_func(*args, cached_res=cached_res, **kwargs) if post_process_func else cached_res
+
+            if RD_AGENT_SETTINGS.use_file_lock:
+                with FileLock(lock_file):
+                    result = func(*args, **kwargs)
+            else:
+                result = func(*args, **kwargs)
+
+            with cache_file.open("wb") as f:
+                pickle.dump(result, f)
+
+            return result
+
+        return cache_wrapper
+
+    return cache_decorator

rdagent/log/storage.py

@@ -68,10 +68,10 @@ class FileStorage(Storage):
     def iter_msg(self, watch: bool = False) -> Generator[Message, None, None]:
         msg_l = []
         for file in self.path.glob("**/*.log"):
-            tag = ".".join(str(file.relative_to(self.path)).replace("/", ".").split(".")[:-3])
+            tag = ".".join(file.relative_to(self.path).as_posix().replace("/", ".").split(".")[:-3])
             pid = file.parent.name
 
-            with file.open("r") as f:
+            with file.open("r", encoding="utf-8") as f:
                 content = f.read()
 
             matches, next_matches = self.log_pattern.finditer(content), self.log_pattern.finditer(content)
@@ -100,7 +100,7 @@ class FileStorage(Storage):
                 msg_l.append(m)
 
         for file in self.path.glob("**/*.pkl"):
-            tag = ".".join(str(file.relative_to(self.path)).replace("/", ".").split(".")[:-3])
+            tag = ".".join(file.relative_to(self.path).as_posix().replace("/", ".").split(".")[:-3])
             pid = file.parent.name
 
             with file.open("rb") as f:

rdagent/log/time.py

@@ -0,0 +1,19 @@
+import time
+from functools import wraps
+
+from rdagent.log import rdagent_logger as logger
+
+
+def measure_time(method):
+    @wraps(method)
+    def timed(*args, **kwargs):
+        start_time = time.time()
+        result = method(*args, **kwargs)
+        end_time = time.time()
+        duration = end_time - start_time
+        method_name = method.__name__
+        # logger.log_object(f"{method_name} took {duration:.2f} sec")
+        logger.info(f"{method_name} took {duration:.2f} sec")
+        return result
+
+    return timed

rdagent/log/ui/app.py

@@ -2,6 +2,7 @@ import argparse
 import textwrap
 from collections import defaultdict
 from datetime import datetime, timezone
+from importlib.resources import files as rfiles
 from pathlib import Path
 from typing import Callable, Type
 
@@ -10,9 +11,8 @@ import plotly.express as px
 import plotly.graph_objects as go
 import streamlit as st
 from plotly.subplots import make_subplots
-from st_btn_select import st_btn_select
 from streamlit import session_state as state
-from streamlit.delta_generator import DeltaGenerator
+from streamlit_theme import st_theme
 
 from rdagent.components.coder.factor_coder.CoSTEER.evaluators import (
     FactorSingleFeedback,
@@ -21,14 +21,16 @@ from rdagent.components.coder.factor_coder.factor import FactorFBWorkspace, Fact
 from rdagent.components.coder.model_coder.CoSTEER.evaluators import ModelCoderFeedback
 from rdagent.components.coder.model_coder.model import ModelFBWorkspace, ModelTask
 from rdagent.core.proposal import Hypothesis, HypothesisFeedback
+from rdagent.core.scenario import Scenario
 from rdagent.log.base import Message
 from rdagent.log.storage import FileStorage
 from rdagent.log.ui.qlib_report_figure import report_figure
 from rdagent.scenarios.data_mining.experiment.model_experiment import DMModelScenario
 from rdagent.scenarios.general_model.scenario import GeneralModelScenario
-from rdagent.scenarios.qlib.experiment.factor_experiment import (
-    QlibFactorExperiment,
-    QlibFactorScenario,
+from rdagent.scenarios.kaggle.experiment.scenario import KGScenario
+from rdagent.scenarios.qlib.experiment.factor_experiment import QlibFactorScenario
+from rdagent.scenarios.qlib.experiment.factor_from_report_experiment import (
+    QlibFactorFromReportScenario,
 )
 from rdagent.scenarios.qlib.experiment.model_experiment import (
     QlibModelExperiment,
@@ -41,6 +43,7 @@ st.set_page_config(layout="wide", page_title="RD-Agent", page_icon="🎓", initi
 # 获取log_path参数
 parser = argparse.ArgumentParser(description="RD-Agent Streamlit App")
 parser.add_argument("--log_dir", type=str, help="Path to the log directory")
+parser.add_argument("--debug", action="store_true", help="Enable debug mode")
 args = parser.parse_args()
 if args.log_dir:
     main_log_path = Path(args.log_dir)
@@ -51,21 +54,24 @@ else:
     main_log_path = None
 
 
-SELECTED_METRICS = [
+QLIB_SELECTED_METRICS = [
     "IC",
     "1day.excess_return_without_cost.annualized_return",
     "1day.excess_return_without_cost.information_ratio",
     "1day.excess_return_without_cost.max_drawdown",
 ]
 
-if "log_type" not in state:
-    state.log_type = "Qlib Model"
+SIMILAR_SCENARIOS = (QlibModelScenario, DMModelScenario, QlibFactorScenario, QlibFactorFromReportScenario, KGScenario)
 
 if "log_path" not in state:
     if main_log_path:
         state.log_path = next(main_log_path.iterdir()).relative_to(main_log_path)
     else:
-        state.log_path = ""
+        state.log_path = None
+        st.toast(":red[**Please Set Log Path!**]", icon="⚠️")
+
+if "scenario" not in state:
+    state.scenario = None
 
 if "fs" not in state:
     state.fs = None
@@ -82,6 +88,9 @@ if "current_tags" not in state:
 if "lround" not in state:
     state.lround = 0  # RD Loop Round
 
+if "times" not in state:
+    state.times = defaultdict(lambda: defaultdict(list))
+
 if "erounds" not in state:
     state.erounds = defaultdict(int)  # Evolving Rounds in each RD Loop
 
@@ -104,23 +113,6 @@ if "alpha158_metrics" not in state:
     state.alpha158_metrics = None
 
 
-def refresh():
-    if main_log_path:
-        state.fs = FileStorage(main_log_path / state.log_path).iter_msg()
-    else:
-        state.fs = FileStorage(state.log_path).iter_msg()
-    state.msgs = defaultdict(lambda: defaultdict(list))
-    state.lround = 0
-    state.erounds = defaultdict(int)
-    state.e_decisions = defaultdict(lambda: defaultdict(tuple))
-    state.hypotheses = defaultdict(None)
-    state.h_decisions = defaultdict(bool)
-    state.metric_series = []
-    state.last_msg = None
-    state.current_tags = []
-    state.alpha158_metrics = None
-
-
 def should_display(msg: Message):
     for t in state.excluded_tags:
         if t in msg.tag.split("."):
@@ -150,24 +142,28 @@ def get_msgs_until(end_func: Callable[[Message], bool] = lambda _: True):
                     # Update Summary Info
                     if "model runner result" in tags or "factor runner result" in tags or "runner result" in tags:
                         # factor baseline exp metrics
-                        if state.log_type == "Qlib Factor" and state.alpha158_metrics is None:
-                            sms = msg.content.based_experiments[0].result.loc[SELECTED_METRICS]
+                        if isinstance(state.scenario, QlibFactorScenario) and state.alpha158_metrics is None:
+                            sms = msg.content.based_experiments[0].result.loc[QLIB_SELECTED_METRICS]
                             sms.name = "alpha158"
                             state.alpha158_metrics = sms
 
                         # common metrics
                         if msg.content.result is None:
-                            state.metric_series.append(pd.Series([None], index=["AUROC"], name=f"Round {state.lround}"))
+                            if isinstance(state.scenario, DMModelScenario):
+                                state.metric_series.append(
+                                    pd.Series([None], index=["AUROC"], name=f"Round {state.lround}")
+                                )
                         else:
-                            if len(msg.content.result) < 4:
-                                ps = msg.content.result
-                                ps.index = ["AUROC"]
-                                ps.name = f"Round {state.lround}"
-                                state.metric_series.append(ps)
-                            else:
-                                sms = msg.content.result.loc[SELECTED_METRICS]
-                                sms.name = f"Round {state.lround}"
-                                state.metric_series.append(sms)
+                            sms = msg.content.result
+                            if isinstance(state.scenario, DMModelScenario):
+                                sms.index = ["AUROC"]
+                            elif isinstance(
+                                state.scenario, (QlibModelScenario, QlibFactorFromReportScenario, QlibFactorScenario)
+                            ):
+                                sms = sms.loc[QLIB_SELECTED_METRICS]
+
+                            sms.name = f"Round {state.lround}"
+                            state.metric_series.append(sms)
                     elif "hypothesis generation" in tags:
                         state.hypotheses[state.lround] = msg.content
                     elif "ef" in tags and "feedback" in tags:
@@ -176,7 +172,9 @@ def get_msgs_until(end_func: Callable[[Message], bool] = lambda _: True):
                         if "evolving code" in tags:
                             msg.content = [i for i in msg.content if i]
                         if "evolving feedback" in tags:
+                            total_len = len(msg.content)
                             msg.content = [i for i in msg.content if i]
+                            none_num = total_len - len(msg.content)
                             if len(msg.content) != len(state.msgs[state.lround]["d.evolving code"][-1].content):
                                 st.toast(":red[**Evolving Feedback Length Error!**]", icon="‼️")
                             right_num = 0
@@ -184,9 +182,24 @@ def get_msgs_until(end_func: Callable[[Message], bool] = lambda _: True):
                                 if wsf.final_decision:
                                     right_num += 1
                             wrong_num = len(msg.content) - right_num
-                            state.e_decisions[state.lround][state.erounds[state.lround]] = (right_num, wrong_num)
+                            state.e_decisions[state.lround][state.erounds[state.lround]] = (
+                                right_num,
+                                wrong_num,
+                                none_num,
+                            )
 
                     state.msgs[state.lround][msg.tag].append(msg)
+
+                    # Update Times
+                    if "init" in tags:
+                        state.times[state.lround]["init"].append(msg.timestamp)
+                    if "r" in tags:
+                        state.times[state.lround]["r"].append(msg.timestamp)
+                    if "d" in tags:
+                        state.times[state.lround]["d"].append(msg.timestamp)
+                    if "ef" in tags:
+                        state.times[state.lround]["ef"].append(msg.timestamp)
+
                     # Stop Getting Logs
                     if end_func(msg):
                         break
@@ -195,6 +208,39 @@ def get_msgs_until(end_func: Callable[[Message], bool] = lambda _: True):
                 break
 
 
+def refresh(same_trace: bool = False):
+    if state.log_path is None:
+        st.toast(":red[**Please Set Log Path!**]", icon="⚠️")
+        return
+
+    if main_log_path:
+        state.fs = FileStorage(main_log_path / state.log_path).iter_msg()
+    else:
+        state.fs = FileStorage(state.log_path).iter_msg()
+
+    # detect scenario
+    if not same_trace:
+        get_msgs_until(lambda m: not isinstance(m.content, str))
+        if state.last_msg is None or not isinstance(state.last_msg.content, Scenario):
+            st.toast(":red[**No Scenario Info detected**]", icon="❗")
+            state.scenario = None
+        else:
+            state.scenario = state.last_msg.content
+            st.toast(f":green[**Scenario Info detected**] *{type(state.scenario).__name__}*", icon="✅")
+
+    state.msgs = defaultdict(lambda: defaultdict(list))
+    state.lround = 0
+    state.erounds = defaultdict(int)
+    state.e_decisions = defaultdict(lambda: defaultdict(tuple))
+    state.hypotheses = defaultdict(None)
+    state.h_decisions = defaultdict(bool)
+    state.metric_series = []
+    state.last_msg = None
+    state.current_tags = []
+    state.alpha158_metrics = None
+    state.times = defaultdict(lambda: defaultdict(list))
+
+
 def evolving_feedback_window(wsf: FactorSingleFeedback | ModelCoderFeedback):
     if isinstance(wsf, FactorSingleFeedback):
         ffc, efc, cfc, vfc = st.tabs(
@@ -231,14 +277,30 @@ def evolving_feedback_window(wsf: FactorSingleFeedback | ModelCoderFeedback):
 
 
 def display_hypotheses(hypotheses: dict[int, Hypothesis], decisions: dict[int, bool], success_only: bool = False):
+    name_dict = {
+        "hypothesis": "RD-Agent proposes the hypothesis⬇️",
+        "concise_justification": "because the reason⬇️",
+        "concise_observation": "based on the observation⬇️",
+        "concise_knowledge": "Knowledge⬇️ gained after practice",
+    }
     if success_only:
         shd = {k: v.__dict__ for k, v in hypotheses.items() if decisions[k]}
     else:
         shd = {k: v.__dict__ for k, v in hypotheses.items()}
     df = pd.DataFrame(shd).T
+
+    if "concise_observation" in df.columns and "concise_justification" in df.columns:
+        df["concise_observation"], df["concise_justification"] = df["concise_justification"], df["concise_observation"]
+        df.rename(
+            columns={"concise_observation": "concise_justification", "concise_justification": "concise_observation"},
+            inplace=True,
+        )
     if "reason" in df.columns:
         df.drop(["reason"], axis=1, inplace=True)
-    df.columns = df.columns.map(lambda x: x.replace("_", " ").capitalize())
+    if "concise_reason" in df.columns:
+        df.drop(["concise_reason"], axis=1, inplace=True)
+
+    df.columns = df.columns.map(lambda x: name_dict.get(x, x))
 
     def style_rows(row):
         if decisions[row.name]:
@@ -246,7 +308,7 @@ def display_hypotheses(hypotheses: dict[int, Hypothesis], decisions: dict[int, b
         return [""] * len(row)
 
     def style_columns(col):
-        if col.name != "Hypothesis":
+        if col.name != name_dict.get("hypothesis", "hypothesis"):
             return ["font-style: italic;"] * len(col)
         return ["font-weight: bold;"] * len(col)
 
@@ -302,8 +364,10 @@ def metrics_window(df: pd.DataFrame, R: int, C: int, *, height: int = 300, color
 
 
 def summary_window():
-    if state.log_type in ["Qlib Model", "Data Mining", "Qlib Factor"]:
+    if isinstance(state.scenario, SIMILAR_SCENARIOS):
         st.header("Summary📊", divider="rainbow", anchor="_summary")
+        if state.lround == 0:
+            return
         with st.container():
             # TODO: not fixed height
             with st.container():
@@ -322,7 +386,7 @@ def summary_window():
                 display_hypotheses(state.hypotheses, state.h_decisions, show_true_only)
 
             with chart_c:
-                if state.log_type == "Qlib Factor" and state.alpha158_metrics is not None:
+                if isinstance(state.scenario, QlibFactorScenario) and state.alpha158_metrics is not None:
                     df = pd.DataFrame([state.alpha158_metrics] + state.metric_series)
                 else:
                     df = pd.DataFrame(state.metric_series)
@@ -336,40 +400,40 @@ def summary_window():
                     st.table(df.iloc[0])
                 elif df.shape[0] > 1:
                     if df.shape[1] == 1:
-                        # suhan's scenario
                         fig = px.line(df, x=df.index, y=df.columns, markers=True)
                         fig.update_layout(xaxis_title="Loop Round", yaxis_title=None)
                         st.plotly_chart(fig)
                     else:
                         metrics_window(df, 1, 4, height=300, colors=["red", "blue", "orange", "green"])
 
-    elif state.log_type == "Model from Paper" and len(state.msgs[state.lround]["d.evolving code"]) > 0:
+    elif isinstance(state.scenario, GeneralModelScenario):
         with st.container(border=True):
             st.subheader("Summary📊", divider="rainbow", anchor="_summary")
+            if len(state.msgs[state.lround]["d.evolving code"]) > 0:
+                # pass
+                ws: list[FactorFBWorkspace | ModelFBWorkspace] = state.msgs[state.lround]["d.evolving code"][-1].content
+                # All Tasks
 
-            # pass
-            ws: list[FactorFBWorkspace | ModelFBWorkspace] = state.msgs[state.lround]["d.evolving code"][-1].content
-            # All Tasks
+                tab_names = [
+                    w.target_task.factor_name if isinstance(w.target_task, FactorTask) else w.target_task.name
+                    for w in ws
+                ]
+                for j in range(len(ws)):
+                    if state.msgs[state.lround]["d.evolving feedback"][-1].content[j].final_decision:
+                        tab_names[j] += "✔️"
+                    else:
+                        tab_names[j] += "❌"
 
-            tab_names = [
-                w.target_task.factor_name if isinstance(w.target_task, FactorTask) else w.target_task.name for w in ws
-            ]
-            for j in range(len(ws)):
-                if state.msgs[state.lround]["d.evolving feedback"][-1].content[j].final_decision:
-                    tab_names[j] += "✔️"
-                else:
-                    tab_names[j] += "❌"
+                wtabs = st.tabs(tab_names)
+                for j, w in enumerate(ws):
+                    with wtabs[j]:
+                        # Evolving Code
+                        for k, v in w.code_dict.items():
+                            with st.expander(f":green[`{k}`]", expanded=False):
+                                st.code(v, language="python")
 
-            wtabs = st.tabs(tab_names)
-            for j, w in enumerate(ws):
-                with wtabs[j]:
-                    # Evolving Code
-                    for k, v in w.code_dict.items():
-                        with st.expander(f":green[`{k}`]", expanded=False):
-                            st.code(v, language="python")
-
-                    # Evolving Feedback
-                    evolving_feedback_window(state.msgs[state.lround]["d.evolving feedback"][-1].content[j])
+                        # Evolving Feedback
+                        evolving_feedback_window(state.msgs[state.lround]["d.evolving feedback"][-1].content[j])
 
 
 def tabs_hint():
@@ -379,7 +443,6 @@ def tabs_hint():
     )
 
 
-# TODO: when tab names are too long, some tabs are not shown
 def tasks_window(tasks: list[FactorTask | ModelTask]):
     if isinstance(tasks[0], FactorTask):
         st.markdown("**Factor Tasks🚩**")
@@ -392,12 +455,13 @@ def tasks_window(tasks: list[FactorTask | ModelTask]):
                 # st.markdown(f"**Factor Name**: {ft.factor_name}")
                 st.markdown(f"**Description**: {ft.factor_description}")
                 st.latex("Formulation")
-                st.latex(f"{ft.factor_formulation}")
+                st.latex(ft.factor_formulation)
 
                 mks = "| Variable | Description |\n| --- | --- |\n"
-                for v, d in ft.variables.items():
-                    mks += f"| ${v}$ | {d} |\n"
-                st.markdown(mks)
+                if isinstance(ft.variables, dict):
+                    for v, d in ft.variables.items():
+                        mks += f"| ${v}$ | {d} |\n"
+                    st.markdown(mks)
 
     elif isinstance(tasks[0], ModelTask):
         st.markdown("**Model Tasks🚩**")
@@ -411,159 +475,20 @@ def tasks_window(tasks: list[FactorTask | ModelTask]):
                 st.markdown(f"**Model Type**: {mt.model_type}")
                 st.markdown(f"**Description**: {mt.description}")
                 st.latex("Formulation")
-                st.latex(f"{mt.formulation}")
+                st.latex(mt.formulation)
 
                 mks = "| Variable | Description |\n| --- | --- |\n"
-                for v, d in mt.variables.items():
-                    mks += f"| ${v}$ | {d} |\n"
-                st.markdown(mks)
-
-
-# Config Sidebar
-with st.sidebar:
-    st.markdown(
-        """
-# RD-Agent🤖
-## [Scenario Description](#_scenario)
-## [Summary](#_summary)
-- [**Hypotheses**](#_hypotheses)
-- [**Metrics**](#_metrics)
-## [RD-Loops](#_rdloops)
-- [**Research**](#_research)
-- [**Development**](#_development)
-- [**Feedback**](#_feedback)
-"""
-    )
-
-    st.selectbox(
-        ":green[**Scenario**]", ["Qlib Model", "Data Mining", "Qlib Factor", "Model from Paper"], key="log_type"
-    )
-
-    with st.popover(":orange[**Config⚙️**]"):
-        with st.container(border=True):
-            st.markdown(":blue[**log path**]")
-            if main_log_path:
-                if st.toggle("Manual Input"):
-                    st.text_input("log path", key="log_path", on_change=refresh)
-                else:
-                    folders = [
-                        folder.relative_to(main_log_path) for folder in main_log_path.iterdir() if folder.is_dir()
-                    ]
-                    st.selectbox(f"Select from `{main_log_path}`", folders, key="log_path", on_change=refresh)
-            else:
-                st.text_input("log path", key="log_path", on_change=refresh)
-
-        with st.container(border=True):
-            st.markdown(":blue[**excluded configs**]")
-            st.multiselect("excluded log tags", ["llm_messages"], ["llm_messages"], key="excluded_tags")
-            st.multiselect("excluded log types", ["str", "dict", "list"], ["str"], key="excluded_types")
-
-    if st.button("All Loops"):
-        if not state.fs:
-            refresh()
-        get_msgs_until(lambda m: False)
-
-    if st.button("Next Loop"):
-        if not state.fs:
-            refresh()
-        get_msgs_until(lambda m: "ef.feedback" in m.tag)
-
-    if st.button("One Evolving"):
-        if not state.fs:
-            refresh()
-        get_msgs_until(lambda m: "d.evolving feedback" in m.tag)
-
-    if st.button("refresh logs", help="clear all log messages in cache"):
-        refresh()
-    debug = st.toggle("debug", value=False)
-
-    if debug:
-        if st.button("Single Step Run"):
-            if not state.fs:
-                refresh()
-            get_msgs_until()
-
-
-# Debug Info Window
-if debug:
-    with st.expander(":red[**Debug Info**]", expanded=True):
-        dcol1, dcol2 = st.columns([1, 3])
-        with dcol1:
-            st.markdown(
-                f"**trace type**: {state.log_type}\n\n"
-                f"**log path**: {state.log_path}\n\n"
-                f"**excluded tags**: {state.excluded_tags}\n\n"
-                f"**excluded types**: {state.excluded_types}\n\n"
-                f":blue[**message id**]: {sum(sum(len(tmsgs) for tmsgs in rmsgs.values()) for rmsgs in state.msgs.values())}\n\n"
-                f":blue[**round**]: {state.lround}\n\n"
-                f":blue[**evolving round**]: {state.erounds[state.lround]}\n\n"
-            )
-        with dcol2:
-            if state.last_msg:
-                st.write(state.last_msg)
-                if isinstance(state.last_msg.content, list):
-                    st.write(state.last_msg.content[0])
-                elif not isinstance(state.last_msg.content, str):
-                    st.write(state.last_msg.content.__dict__)
-
-
-# Main Window
-header_c1, header_c3 = st.columns([1, 6], vertical_alignment="center")
-with st.container():
-    with header_c1:
-        st.image("https://img-prod-cms-rt-microsoft-com.akamaized.net/cms/api/am/imageFileData/RE1Mu3b?ver=5c31")
-    with header_c3:
-        st.markdown(
-            """
-        <h1>
-            RD-Agent:<br>LLM-based autonomous evolving agents for industrial data-driven R&D
-        </h1>
-        """,
-            unsafe_allow_html=True,
-        )
-
-# Project Info
-with st.container():
-    image_c, scen_c = st.columns([3, 3], vertical_alignment="center")
-    with image_c:
-        st.image("./docs/_static/flow.png")
-    with scen_c:
-        st.header("Scenario Description📖", divider="violet", anchor="_scenario")
-        # TODO: other scenarios
-        if state.log_type == "Qlib Model":
-            st.markdown(QlibModelScenario().rich_style_description, unsafe_allow_html=True)
-        elif state.log_type == "Data Mining":
-            st.markdown(DMModelScenario().rich_style_description)
-        elif state.log_type == "Qlib Factor":
-            st.markdown(QlibFactorScenario().rich_style_description, unsafe_allow_html=True)
-        elif state.log_type == "Model from Paper":
-            st.markdown(GeneralModelScenario().rich_style_description, unsafe_allow_html=True)
-
-
-# Summary Window
-summary_window()
-
-# R&D Loops Window
-if state.log_type in ["Qlib Model", "Data Mining", "Qlib Factor"]:
-    st.header("R&D Loops♾️", divider="rainbow", anchor="_rdloops")
-
-if state.log_type in ["Qlib Model", "Data Mining", "Qlib Factor"]:
-    if len(state.msgs) > 1:
-        r_options = list(state.msgs.keys())
-        if 0 in r_options:
-            r_options.remove(0)
-        round = st_btn_select(options=r_options, index=state.lround - 1)
-    else:
-        round = 1
-else:
-    round = 1
+                if mt.variables:
+                    for v, d in mt.variables.items():
+                        mks += f"| ${v}$ | {d} |\n"
+                    st.markdown(mks)
 
 
 def research_window():
     with st.container(border=True):
-        title = "Research🔍" if state.log_type in ["Qlib Model", "Data Mining", "Qlib Factor"] else "Research🔍 (reader)"
+        title = "Research🔍" if isinstance(state.scenario, SIMILAR_SCENARIOS) else "Research🔍 (reader)"
         st.subheader(title, divider="blue", anchor="_research")
-        if state.log_type in ["Qlib Model", "Data Mining", "Qlib Factor"]:
+        if isinstance(state.scenario, SIMILAR_SCENARIOS):
             # pdf image
             if pim := state.msgs[round]["r.extract_factors_and_implement.load_pdf_screenshot"]:
                 for i in range(min(2, len(pim))):
@@ -582,7 +507,7 @@ def research_window():
             if eg := state.msgs[round]["r.experiment generation"]:
                 tasks_window(eg[0].content)
 
-        elif state.log_type == "Model from Paper":
+        elif isinstance(state.scenario, GeneralModelScenario):
             # pdf image
             c1, c2 = st.columns([2, 3])
             with c1:
@@ -593,14 +518,26 @@ def research_window():
             # loaded model exp
             with c2:
                 if mem := state.msgs[round]["d.load_experiment"]:
+                    # 'load_experiment' should in 'r' now, but old version trace may in 'd', so we need to check both
+                    # TODO: modify the way to get one message with a specific tag like 'load_experiment' in the future
+                    me: QlibModelExperiment = mem[0].content
+                    tasks_window(me.sub_tasks)
+                elif mem := state.msgs[round]["r.load_experiment"]:
                     me: QlibModelExperiment = mem[0].content
                     tasks_window(me.sub_tasks)
 
 
 def feedback_window():
-    if state.log_type in ["Qlib Model", "Data Mining", "Qlib Factor"]:
+    if isinstance(state.scenario, SIMILAR_SCENARIOS):
         with st.container(border=True):
             st.subheader("Feedback📝", divider="orange", anchor="_feedback")
+
+            if state.lround > 0 and isinstance(
+                state.scenario, (QlibModelScenario, QlibFactorScenario, QlibFactorFromReportScenario, KGScenario)
+            ):
+                with st.expander("**Config⚙️**", expanded=True):
+                    st.markdown(state.scenario.experiment_setting, unsafe_allow_html=True)
+
             if fbr := state.msgs[round]["ef.Quantitative Backtesting Chart"]:
                 st.markdown("**Returns📈**")
                 fig = report_figure(fbr[0].content)
@@ -617,26 +554,22 @@ def feedback_window():
 - **Reason**: {h.reason}"""
                 )
 
-
-if state.log_type in ["Qlib Model", "Data Mining", "Qlib Factor"]:
-    rf_c, d_c = st.columns([2, 2])
-elif state.log_type == "Model from Paper":
-    rf_c = st.container()
-    d_c = st.container()
+            if isinstance(state.scenario, KGScenario):
+                if fbe := state.msgs[round]["ef.runner result"]:
+                    submission_path = fbe[0].content.experiment_workspace.workspace_path / "submission.csv"
+                    st.markdown(
+                        f":green[**Exp Workspace**]: {str(fbe[0].content.experiment_workspace.workspace_path.absolute())}"
+                    )
+                    st.download_button(
+                        label="**Download** submission.csv",
+                        data=submission_path.read_bytes(),
+                        file_name="submission.csv",
+                        mime="text/csv",
+                    )
 
 
-with rf_c:
-    research_window()
-    feedback_window()
-
-
-# Development Window (Evolving)
-with d_c.container(border=True):
-    title = (
-        "Development🛠️"
-        if state.log_type in ["Qlib Model", "Data Mining", "Qlib Factor"]
-        else "Development🛠️ (evolving coder)"
-    )
+def evolving_window():
+    title = "Development🛠️" if isinstance(state.scenario, SIMILAR_SCENARIOS) else "Development🛠️ (evolving coder)"
     st.subheader(title, divider="green", anchor="_development")
 
     # Evolving Status
@@ -647,17 +580,20 @@ with d_c.container(border=True):
         e_status_mks += "|--" * state.erounds[round] + "|\n"
         for ei, estatus in es.items():
             if not estatus:
-                estatus = (0, 0)
-            e_status_mks += "| " + "✔️<br>" * estatus[0] + "❌<br>" * estatus[1] + " "
+                estatus = (0, 0, 0)
+            e_status_mks += "| " + "🕙<br>" * estatus[2] + "✔️<br>" * estatus[0] + "❌<br>" * estatus[1] + " "
         e_status_mks += "|\n"
         st.markdown(e_status_mks, unsafe_allow_html=True)
 
     # Evolving Tabs
     if state.erounds[round] > 0:
         if state.erounds[round] > 1:
-            st.markdown("**🔄️Evolving Rounds**")
-            evolving_round = st_btn_select(
-                options=range(1, state.erounds[round] + 1), index=state.erounds[round] - 1, key="show_eround"
+            evolving_round = st.radio(
+                "**🔄️Evolving Rounds**",
+                horizontal=True,
+                options=range(1, state.erounds[round] + 1),
+                index=state.erounds[round] - 1,
+                key="show_eround",
             )
         else:
             evolving_round = 1
@@ -691,8 +627,189 @@ with d_c.container(border=True):
                     evolving_feedback_window(state.msgs[round]["d.evolving feedback"][evolving_round - 1].content[j])
 
 
-with st.container(border=True):
-    st.subheader("Disclaimer", divider="gray")
-    st.markdown(
-        "This content is AI-generated and may not be fully accurate or up-to-date; please verify with a professional for critical matters."
-    )
+toc = """
+## [Scenario Description📖](#_scenario)
+## [Summary📊](#_summary)
+- [**Metrics📈**](#_metrics)
+- [**Hypotheses🏅**](#_hypotheses)
+## [RD-Loops♾️](#_rdloops)
+- [**Research🔍**](#_research)
+- [**Development🛠️**](#_development)
+- [**Feedback📝**](#_feedback)
+"""
+if isinstance(state.scenario, GeneralModelScenario):
+    toc = """
+## [Scenario Description📖](#_scenario)
+### [Summary📊](#_summary)
+### [Research🔍](#_research)
+### [Development🛠️](#_development)
+"""
+# Config Sidebar
+with st.sidebar:
+    st.markdown("# RD-Agent🤖  [:grey[@GitHub]](https://github.com/microsoft/RD-Agent)")
+    st.subheader(":blue[Table of Content]", divider="blue")
+    st.markdown(toc)
+    st.subheader(":orange[Control Panel]", divider="red")
+
+    with st.container(border=True):
+        if main_log_path:
+            lc1, lc2 = st.columns([1, 2], vertical_alignment="center")
+            with lc1:
+                st.markdown(":blue[**Log Path**]")
+            with lc2:
+                manually = st.toggle("Manual Input")
+            if manually:
+                st.text_input("log path", key="log_path", on_change=refresh, label_visibility="collapsed")
+            else:
+                folders = [folder.relative_to(main_log_path) for folder in main_log_path.iterdir() if folder.is_dir()]
+                st.selectbox(f"**Select from `{main_log_path}`**", folders, key="log_path", on_change=refresh)
+        else:
+            st.text_input(":blue[**log path**]", key="log_path", on_change=refresh)
+
+    c1, c2 = st.columns([1, 1], vertical_alignment="center")
+    with c1:
+        if st.button(":green[**All Loops**]", use_container_width=True):
+            if not state.fs:
+                refresh()
+            get_msgs_until(lambda m: False)
+        if st.button("**Reset**", use_container_width=True):
+            refresh(same_trace=True)
+    with c2:
+        if st.button(":green[Next Loop]", use_container_width=True):
+            if not state.fs:
+                refresh()
+            get_msgs_until(lambda m: "ef.feedback" in m.tag)
+
+        if st.button("Next Step", use_container_width=True):
+            if not state.fs:
+                refresh()
+            get_msgs_until(lambda m: "d.evolving feedback" in m.tag)
+
+    with st.popover(":orange[**Config⚙️**]", use_container_width=True):
+        st.multiselect("excluded log tags", ["llm_messages"], ["llm_messages"], key="excluded_tags")
+        st.multiselect("excluded log types", ["str", "dict", "list"], ["str"], key="excluded_types")
+
+    if args.debug:
+        debug = st.toggle("debug", value=False)
+
+        if debug:
+            if st.button("Single Step Run", use_container_width=True):
+                get_msgs_until()
+    else:
+        debug = False
+
+
+# Debug Info Window
+if debug:
+    with st.expander(":red[**Debug Info**]", expanded=True):
+        dcol1, dcol2 = st.columns([1, 3])
+        with dcol1:
+            st.markdown(
+                f"**log path**: {state.log_path}\n\n"
+                f"**excluded tags**: {state.excluded_tags}\n\n"
+                f"**excluded types**: {state.excluded_types}\n\n"
+                f":blue[**message id**]: {sum(sum(len(tmsgs) for tmsgs in rmsgs.values()) for rmsgs in state.msgs.values())}\n\n"
+                f":blue[**round**]: {state.lround}\n\n"
+                f":blue[**evolving round**]: {state.erounds[state.lround]}\n\n"
+            )
+        with dcol2:
+            if state.last_msg:
+                st.write(state.last_msg)
+                if isinstance(state.last_msg.content, list):
+                    st.write(state.last_msg.content[0])
+                elif not isinstance(state.last_msg.content, str):
+                    st.write(state.last_msg.content.__dict__)
+
+
+if state.log_path and state.fs is None:
+    refresh()
+
+# Main Window
+header_c1, header_c3 = st.columns([1, 6], vertical_alignment="center")
+with st.container():
+    with header_c1:
+        st.image("https://img-prod-cms-rt-microsoft-com.akamaized.net/cms/api/am/imageFileData/RE1Mu3b?ver=5c31")
+    with header_c3:
+        st.markdown(
+            """
+        <h1>
+            RD-Agent:<br>LLM-based autonomous evolving agents for industrial data-driven R&D
+        </h1>
+        """,
+            unsafe_allow_html=True,
+        )
+
+# Project Info
+with st.container():
+    image_c, scen_c = st.columns([3, 3], vertical_alignment="center")
+    with image_c:
+        img_path = rfiles("rdagent.log.ui").joinpath("flow.png")
+        st.image(str(img_path), use_column_width=True)
+    with scen_c:
+        st.header("Scenario Description📖", divider="violet", anchor="_scenario")
+        if state.scenario is not None:
+            theme = st_theme()
+            if theme:
+                theme = theme.get("base", "light")
+            css = f"""
+<style>
+    a[href="#_rdloops"], a[href="#_research"], a[href="#_development"], a[href="#_feedback"], a[href="#_scenario"], a[href="#_summary"], a[href="#_hypotheses"], a[href="#_metrics"] {{
+        color: {"black" if theme == "light" else "white"};
+    }}
+</style>
+"""
+            st.markdown(state.scenario.rich_style_description + css, unsafe_allow_html=True)
+
+
+def show_times(round: int):
+    for k, v in state.times[round].items():
+        if len(v) > 1:
+            diff = v[-1] - v[0]
+        else:
+            diff = v[0] - v[0]
+        total_seconds = diff.seconds
+        seconds = total_seconds % 60
+        minutes = total_seconds // 60
+        st.markdown(f"**:blue[{k}]**: :red[**{minutes}**] minutes :orange[**{seconds}**] seconds")
+
+
+if state.scenario is not None:
+    summary_window()
+
+    # R&D Loops Window
+    if isinstance(state.scenario, SIMILAR_SCENARIOS):
+        st.header("R&D Loops♾️", divider="rainbow", anchor="_rdloops")
+        if len(state.msgs) > 1:
+            r_options = list(state.msgs.keys())
+            if 0 in r_options:
+                r_options.remove(0)
+            round = st.radio("**Loops**", horizontal=True, options=r_options, index=state.lround - 1)
+        else:
+            round = 1
+
+        show_times(round)
+        rf_c, d_c = st.columns([2, 2])
+    elif isinstance(state.scenario, GeneralModelScenario):
+        show_times(round)
+
+        rf_c = st.container()
+        d_c = st.container()
+        round = 1
+    else:
+        st.error("Unknown Scenario!")
+        st.stop()
+
+    with rf_c:
+        research_window()
+        feedback_window()
+
+    with d_c.container(border=True):
+        evolving_window()
+
+
+st.markdown("<br><br><br>", unsafe_allow_html=True)
+st.markdown("#### Disclaimer")
+st.markdown(
+    "*This content is AI-generated and may not be fully accurate or up-to-date; please verify with a professional for critical matters.*",
+    unsafe_allow_html=True,
+)

rdagent/log/ui/web.py

@@ -36,7 +36,7 @@ class WebView(View):
 
     def display(self, s: Storage, watch: bool = False):
         for msg in s.iter_msg():  # iterate overtime
-            # NOTE:  iter_msg will correctly seperate the information.
+            # NOTE:  iter_msg will correctly separate the information.
             # TODO: msg may support streaming mode.
             self.ui.consume_msg(msg)
 
@@ -291,10 +291,6 @@ class WorkspaceWindow(StWindow):
             self.container.markdown(f"`{k}`")
             self.container.code(v, language="python")
 
-        # executed_factor_value_dataframe
-        # if isinstance(ws, FactorFBWorkspace):
-        #     self.container.dataframe(ws.executed_factor_value_dataframe)
-
 
 class QlibFactorExpWindow(StWindow):
     def __init__(self, container: DeltaGenerator, show_task_info: bool = False):
@@ -586,7 +582,7 @@ class TraceWindow(StWindow):
         self.show_llm = show_llm
         self.show_common_logs = show_common_logs
         image_c, scen_c = container.columns([2, 3], vertical_alignment="center")
-        image_c.image("scen.jpg")
+        image_c.image("scen.png")
         scen_c.container(border=True).markdown(QlibModelScenario().rich_style_description)
         top_container = container.container()
         col1, col2 = top_container.columns([2, 3])
@@ -622,9 +618,11 @@ class TraceWindow(StWindow):
             self.hypothesis_decisions[self.hypotheses[-1]] = msg.content.decision
             self.summary_c.markdown(
                 "\n".join(
-                    f"{id+1}. :green[{self.hypotheses[id].hypothesis}]\n\t>*{self.hypotheses[id].concise_reason}*"
-                    if d
-                    else f"{id+1}. {self.hypotheses[id].hypothesis}\n\t>*{self.hypotheses[id].concise_reason}*"
+                    (
+                        f"{id+1}. :green[{self.hypotheses[id].hypothesis}]\n\t>*{self.hypotheses[id].concise_reason}*"
+                        if d
+                        else f"{id+1}. {self.hypotheses[id].hypothesis}\n\t>*{self.hypotheses[id].concise_reason}*"
+                    )
                     for id, (h, d) in enumerate(self.hypothesis_decisions.items())
                 )
             )

rdagent/oai/llm_conf.py

@@ -0,0 +1,93 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+from pydantic_settings import BaseSettings
+
+
+class LLMSettings(BaseSettings):
+    log_llm_chat_content: bool = True
+
+    use_azure: bool = False
+    use_azure_token_provider: bool = False
+    managed_identity_client_id: str | None = None
+    max_retry: int = 10
+    retry_wait_seconds: int = 1
+    dump_chat_cache: bool = False
+    use_chat_cache: bool = False
+    dump_embedding_cache: bool = False
+    use_embedding_cache: bool = False
+    prompt_cache_path: str = str(Path.cwd() / "prompt_cache.db")
+    max_past_message_include: int = 10
+
+    # Behavior of returning answers to the same question when caching is enabled
+    use_auto_chat_cache_seed_gen: bool = False
+    """
+    `_create_chat_completion_inner_function` provdies a feature to pass in a seed to affect the cache hash key
+    We want to enable a auto seed generator to get different default seed for `_create_chat_completion_inner_function`
+    if seed is not given.
+    So the cache will only not miss you ask the same question on same round.
+    """
+    init_chat_cache_seed: int = 42
+
+    # Chat configs
+    openai_api_key: str = ""  # TODO: simplify the key design.
+    chat_openai_api_key: str = ""
+    chat_azure_api_base: str = ""
+    chat_azure_api_version: str = ""
+    chat_model: str = "gpt-4-turbo"
+    chat_max_tokens: int = 3000
+    chat_temperature: float = 0.5
+    chat_stream: bool = True
+    chat_seed: int | None = None
+    chat_frequency_penalty: float = 0.0
+    chat_presence_penalty: float = 0.0
+    chat_token_limit: int = (
+        100000  # 100000 is the maximum limit of gpt4, which might increase in the future version of gpt
+    )
+    default_system_prompt: str = "You are an AI assistant who helps to answer user's questions."
+
+    # Embedding configs
+    embedding_openai_api_key: str = ""
+    embedding_azure_api_base: str = ""
+    embedding_azure_api_version: str = ""
+    embedding_model: str = ""
+    embedding_max_str_num: int = 50
+
+    # offline llama2 related config
+    use_llama2: bool = False
+    llama2_ckpt_dir: str = "Llama-2-7b-chat"
+    llama2_tokenizer_path: str = "Llama-2-7b-chat/tokenizer.model"
+    llams2_max_batch_size: int = 8
+
+    # server served endpoints
+    use_gcr_endpoint: bool = False
+    gcr_endpoint_type: str = "llama2_70b"  # or "llama3_70b", "phi2", "phi3_4k", "phi3_128k"
+
+    llama2_70b_endpoint: str = ""
+    llama2_70b_endpoint_key: str = ""
+    llama2_70b_endpoint_deployment: str = ""
+
+    llama3_70b_endpoint: str = ""
+    llama3_70b_endpoint_key: str = ""
+    llama3_70b_endpoint_deployment: str = ""
+
+    phi2_endpoint: str = ""
+    phi2_endpoint_key: str = ""
+    phi2_endpoint_deployment: str = ""
+
+    phi3_4k_endpoint: str = ""
+    phi3_4k_endpoint_key: str = ""
+    phi3_4k_endpoint_deployment: str = ""
+
+    phi3_128k_endpoint: str = ""
+    phi3_128k_endpoint_key: str = ""
+    phi3_128k_endpoint_deployment: str = ""
+
+    gcr_endpoint_temperature: float = 0.7
+    gcr_endpoint_top_p: float = 0.9
+    gcr_endpoint_do_sample: bool = False
+    gcr_endpoint_max_token: int = 100
+
+
+LLM_SETTINGS = LLMSettings()

rdagent/oai/llm_utils.py

@@ -1,10 +1,9 @@
 from __future__ import annotations
 
-import datetime
 import hashlib
 import json
-import multiprocessing
 import os
+import random
 import re
 import sqlite3
 import ssl
@@ -13,15 +12,15 @@ import urllib.request
 import uuid
 from copy import deepcopy
 from pathlib import Path
-from typing import Any
+from typing import Any, Optional
 
 import numpy as np
 import tiktoken
 
-from rdagent.core.conf import RD_AGENT_SETTINGS
-from rdagent.core.utils import SingletonBaseClass
+from rdagent.core.utils import LLM_CACHE_SEED_GEN, SingletonBaseClass
 from rdagent.log import LogColors
 from rdagent.log import rdagent_logger as logger
+from rdagent.oai.llm_conf import LLM_SETTINGS
 
 DEFAULT_QLIB_DOT_PATH = Path("./")
 
@@ -90,7 +89,7 @@ class SQliteLazyCache(SingletonBaseClass):
         self.cache_location = cache_location
         db_file_exist = Path(cache_location).exists()
         # TODO: sqlite3 does not support multiprocessing.
-        self.conn = sqlite3.connect(cache_location)
+        self.conn = sqlite3.connect(cache_location, timeout=20)
         self.c = self.conn.cursor()
         if not db_file_exist:
             self.c.execute(
@@ -109,6 +108,14 @@ class SQliteLazyCache(SingletonBaseClass):
                 )
                 """,
             )
+            self.c.execute(
+                """
+                CREATE TABLE message_cache (
+                    conversation_id TEXT PRIMARY KEY,
+                    message TEXT
+                )
+                """,
+            )
             self.conn.commit()
 
     def chat_get(self, key: str) -> str | None:
@@ -144,42 +151,37 @@ class SQliteLazyCache(SingletonBaseClass):
             )
         self.conn.commit()
 
+    def message_get(self, conversation_id: str) -> list[str]:
+        self.c.execute("SELECT message FROM message_cache WHERE conversation_id=?", (conversation_id,))
+        result = self.c.fetchone()
+        if result is None:
+            return []
+        return json.loads(result[0])
+
+    def message_set(self, conversation_id: str, message_value: list[str]) -> None:
+        self.c.execute(
+            "INSERT OR REPLACE INTO message_cache (conversation_id, message) VALUES (?, ?)",
+            (conversation_id, json.dumps(message_value)),
+        )
+        self.conn.commit()
+
 
 class SessionChatHistoryCache(SingletonBaseClass):
     def __init__(self) -> None:
         """load all history conversation json file from self.session_cache_location"""
-        self.cfg = RD_AGENT_SETTINGS
-        self.session_cache_location = Path(self.cfg.session_cache_folder_location)
-        self.cache = {}
-        if not self.session_cache_location.exists():
-            logger.warning(f"Directory {self.session_cache_location} does not exist.")
-            self.session_cache_location.mkdir(parents=True, exist_ok=True)
-        json_files = [f for f in self.session_cache_location.iterdir() if f.suffix == ".json"]
-        if not json_files:
-            logger.info(f"No JSON files found in {self.session_cache_location}.")
-        for file_path in json_files:
-            conversation_id = file_path.stem
-            with file_path.open("r") as f:
-                conversation_content = json.load(f)
-                self.cache[conversation_id] = conversation_content["content"]
+        self.cache = SQliteLazyCache(cache_location=LLM_SETTINGS.prompt_cache_path)
 
     def message_get(self, conversation_id: str) -> list[str]:
-        return self.cache.get(conversation_id, [])
+        return self.cache.message_get(conversation_id)
 
     def message_set(self, conversation_id: str, message_value: list[str]) -> None:
-        self.cache[conversation_id] = message_value
-        conversation_path = self.session_cache_location / conversation_id
-        conversation_path = conversation_path.with_suffix(".json")
-        current_time = datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%d-%H-%M-%S")
-        with conversation_path.open("w") as f:
-            json.dump({"content": message_value, "last_modified_time": current_time}, f)
+        self.cache.message_set(conversation_id, message_value)
 
 
 class ChatSession:
     def __init__(self, api_backend: Any, conversation_id: str | None = None, system_prompt: str | None = None) -> None:
         self.conversation_id = str(uuid.uuid4()) if conversation_id is None else conversation_id
-        self.cfg = RD_AGENT_SETTINGS
-        self.system_prompt = system_prompt if system_prompt is not None else self.cfg.default_system_prompt
+        self.system_prompt = system_prompt if system_prompt is not None else LLM_SETTINGS.default_system_prompt
         self.api_backend = api_backend
 
     def build_chat_completion_message(self, user_prompt: str) -> list[dict[str, Any]]:
@@ -231,6 +233,15 @@ class ChatSession:
 
 
 class APIBackend:
+    """
+    This is a unified interface for different backends.
+
+    (xiao) thinks integrate all kinds of API in a single class is not a good design.
+    So we should split them into different classes in `oai/backends/` in the future.
+    """
+
+    # FIXME: (xiao) We should avoid using self.xxxx.
+    # Instead, we can use LLM_SETTINGS directly. If it's difficult to support different backend settings, we can split them into multiple BaseSettings.
     def __init__(  # noqa: C901, PLR0912, PLR0915
         self,
         *,
@@ -247,37 +258,36 @@ class APIBackend:
         use_embedding_cache: bool | None = None,
         dump_embedding_cache: bool | None = None,
     ) -> None:
-        self.cfg = RD_AGENT_SETTINGS
-        if self.cfg.use_llama2:
+        if LLM_SETTINGS.use_llama2:
             self.generator = Llama.build(
-                ckpt_dir=self.cfg.llama2_ckpt_dir,
-                tokenizer_path=self.cfg.llama2_tokenizer_path,
-                max_seq_len=self.cfg.max_tokens,
-                max_batch_size=self.cfg.llams2_max_batch_size,
+                ckpt_dir=LLM_SETTINGS.llama2_ckpt_dir,
+                tokenizer_path=LLM_SETTINGS.llama2_tokenizer_path,
+                max_seq_len=LLM_SETTINGS.max_tokens,
+                max_batch_size=LLM_SETTINGS.llams2_max_batch_size,
             )
             self.encoder = None
-        elif self.cfg.use_gcr_endpoint:
-            gcr_endpoint_type = self.cfg.gcr_endpoint_type
+        elif LLM_SETTINGS.use_gcr_endpoint:
+            gcr_endpoint_type = LLM_SETTINGS.gcr_endpoint_type
             if gcr_endpoint_type == "llama2_70b":
-                self.gcr_endpoint_key = self.cfg.llama2_70b_endpoint_key
-                self.gcr_endpoint_deployment = self.cfg.llama2_70b_endpoint_deployment
-                self.gcr_endpoint = self.cfg.llama2_70b_endpoint
+                self.gcr_endpoint_key = LLM_SETTINGS.llama2_70b_endpoint_key
+                self.gcr_endpoint_deployment = LLM_SETTINGS.llama2_70b_endpoint_deployment
+                self.gcr_endpoint = LLM_SETTINGS.llama2_70b_endpoint
             elif gcr_endpoint_type == "llama3_70b":
-                self.gcr_endpoint_key = self.cfg.llama3_70b_endpoint_key
-                self.gcr_endpoint_deployment = self.cfg.llama3_70b_endpoint_deployment
-                self.gcr_endpoint = self.cfg.llama3_70b_endpoint
+                self.gcr_endpoint_key = LLM_SETTINGS.llama3_70b_endpoint_key
+                self.gcr_endpoint_deployment = LLM_SETTINGS.llama3_70b_endpoint_deployment
+                self.gcr_endpoint = LLM_SETTINGS.llama3_70b_endpoint
             elif gcr_endpoint_type == "phi2":
-                self.gcr_endpoint_key = self.cfg.phi2_endpoint_key
-                self.gcr_endpoint_deployment = self.cfg.phi2_endpoint_deployment
-                self.gcr_endpoint = self.cfg.phi2_endpoint
+                self.gcr_endpoint_key = LLM_SETTINGS.phi2_endpoint_key
+                self.gcr_endpoint_deployment = LLM_SETTINGS.phi2_endpoint_deployment
+                self.gcr_endpoint = LLM_SETTINGS.phi2_endpoint
             elif gcr_endpoint_type == "phi3_4k":
-                self.gcr_endpoint_key = self.cfg.phi3_4k_endpoint_key
-                self.gcr_endpoint_deployment = self.cfg.phi3_4k_endpoint_deployment
-                self.gcr_endpoint = self.cfg.phi3_4k_endpoint
+                self.gcr_endpoint_key = LLM_SETTINGS.phi3_4k_endpoint_key
+                self.gcr_endpoint_deployment = LLM_SETTINGS.phi3_4k_endpoint_deployment
+                self.gcr_endpoint = LLM_SETTINGS.phi3_4k_endpoint
             elif gcr_endpoint_type == "phi3_128k":
-                self.gcr_endpoint_key = self.cfg.phi3_128k_endpoint_key
-                self.gcr_endpoint_deployment = self.cfg.phi3_128k_endpoint_deployment
-                self.gcr_endpoint = self.cfg.phi3_128k_endpoint
+                self.gcr_endpoint_key = LLM_SETTINGS.phi3_128k_endpoint_key
+                self.gcr_endpoint_deployment = LLM_SETTINGS.phi3_128k_endpoint_deployment
+                self.gcr_endpoint = LLM_SETTINGS.phi3_128k_endpoint
             else:
                 error_message = f"Invalid gcr_endpoint_type: {gcr_endpoint_type}"
                 raise ValueError(error_message)
@@ -286,46 +296,48 @@ class APIBackend:
                 "Authorization": ("Bearer " + self.gcr_endpoint_key),
                 "azureml-model-deployment": self.gcr_endpoint_deployment,
             }
-            self.gcr_endpoint_temperature = self.cfg.gcr_endpoint_temperature
-            self.gcr_endpoint_top_p = self.cfg.gcr_endpoint_top_p
-            self.gcr_endpoint_do_sample = self.cfg.gcr_endpoint_do_sample
-            self.gcr_endpoint_max_token = self.cfg.gcr_endpoint_max_token
+            self.gcr_endpoint_temperature = LLM_SETTINGS.gcr_endpoint_temperature
+            self.gcr_endpoint_top_p = LLM_SETTINGS.gcr_endpoint_top_p
+            self.gcr_endpoint_do_sample = LLM_SETTINGS.gcr_endpoint_do_sample
+            self.gcr_endpoint_max_token = LLM_SETTINGS.gcr_endpoint_max_token
             if not os.environ.get("PYTHONHTTPSVERIFY", "") and hasattr(ssl, "_create_unverified_context"):
                 ssl._create_default_https_context = ssl._create_unverified_context  # noqa: SLF001
             self.encoder = None
         else:
-            self.use_azure = self.cfg.use_azure
-            self.use_azure_token_provider = self.cfg.use_azure_token_provider
-            self.managed_identity_client_id = self.cfg.managed_identity_client_id
+            self.use_azure = LLM_SETTINGS.use_azure
+            self.use_azure_token_provider = LLM_SETTINGS.use_azure_token_provider
+            self.managed_identity_client_id = LLM_SETTINGS.managed_identity_client_id
 
             # Priority: chat_api_key/embedding_api_key > openai_api_key > os.environ.get("OPENAI_API_KEY")
             # TODO: Simplify the key design. Consider Pandatic's field alias & priority.
             self.chat_api_key = (
                 chat_api_key
-                or self.cfg.chat_openai_api_key
-                or self.cfg.openai_api_key
+                or LLM_SETTINGS.chat_openai_api_key
+                or LLM_SETTINGS.openai_api_key
                 or os.environ.get("OPENAI_API_KEY")
             )
             self.embedding_api_key = (
                 embedding_api_key
-                or self.cfg.embedding_openai_api_key
-                or self.cfg.openai_api_key
+                or LLM_SETTINGS.embedding_openai_api_key
+                or LLM_SETTINGS.openai_api_key
                 or os.environ.get("OPENAI_API_KEY")
             )
 
-            self.chat_model = self.cfg.chat_model if chat_model is None else chat_model
+            self.chat_model = LLM_SETTINGS.chat_model if chat_model is None else chat_model
             self.encoder = tiktoken.encoding_for_model(self.chat_model)
-            self.chat_api_base = self.cfg.chat_azure_api_base if chat_api_base is None else chat_api_base
-            self.chat_api_version = self.cfg.chat_azure_api_version if chat_api_version is None else chat_api_version
-            self.chat_stream = self.cfg.chat_stream
-            self.chat_seed = self.cfg.chat_seed
+            self.chat_api_base = LLM_SETTINGS.chat_azure_api_base if chat_api_base is None else chat_api_base
+            self.chat_api_version = (
+                LLM_SETTINGS.chat_azure_api_version if chat_api_version is None else chat_api_version
+            )
+            self.chat_stream = LLM_SETTINGS.chat_stream
+            self.chat_seed = LLM_SETTINGS.chat_seed
 
-            self.embedding_model = self.cfg.embedding_model if embedding_model is None else embedding_model
+            self.embedding_model = LLM_SETTINGS.embedding_model if embedding_model is None else embedding_model
             self.embedding_api_base = (
-                self.cfg.embedding_azure_api_base if embedding_api_base is None else embedding_api_base
+                LLM_SETTINGS.embedding_azure_api_base if embedding_api_base is None else embedding_api_base
             )
             self.embedding_api_version = (
-                self.cfg.embedding_azure_api_version if embedding_api_version is None else embedding_api_version
+                LLM_SETTINGS.embedding_azure_api_version if embedding_api_version is None else embedding_api_version
             )
 
             if self.use_azure:
@@ -363,20 +375,22 @@ class APIBackend:
                 self.chat_client = openai.OpenAI(api_key=self.chat_api_key)
                 self.embedding_client = openai.OpenAI(api_key=self.embedding_api_key)
 
-        self.dump_chat_cache = self.cfg.dump_chat_cache if dump_chat_cache is None else dump_chat_cache
-        self.use_chat_cache = self.cfg.use_chat_cache if use_chat_cache is None else use_chat_cache
+        self.dump_chat_cache = LLM_SETTINGS.dump_chat_cache if dump_chat_cache is None else dump_chat_cache
+        self.use_chat_cache = LLM_SETTINGS.use_chat_cache if use_chat_cache is None else use_chat_cache
         self.dump_embedding_cache = (
-            self.cfg.dump_embedding_cache if dump_embedding_cache is None else dump_embedding_cache
+            LLM_SETTINGS.dump_embedding_cache if dump_embedding_cache is None else dump_embedding_cache
+        )
+        self.use_embedding_cache = (
+            LLM_SETTINGS.use_embedding_cache if use_embedding_cache is None else use_embedding_cache
         )
-        self.use_embedding_cache = self.cfg.use_embedding_cache if use_embedding_cache is None else use_embedding_cache
         if self.dump_chat_cache or self.use_chat_cache or self.dump_embedding_cache or self.use_embedding_cache:
-            self.cache_file_location = self.cfg.prompt_cache_path
+            self.cache_file_location = LLM_SETTINGS.prompt_cache_path
             self.cache = SQliteLazyCache(cache_location=self.cache_file_location)
 
         # transfer the config to the class if the config is not supposed to change during the runtime
-        self.use_llama2 = self.cfg.use_llama2
-        self.use_gcr_endpoint = self.cfg.use_gcr_endpoint
-        self.retry_wait_seconds = self.cfg.retry_wait_seconds
+        self.use_llama2 = LLM_SETTINGS.use_llama2
+        self.use_gcr_endpoint = LLM_SETTINGS.use_gcr_endpoint
+        self.retry_wait_seconds = LLM_SETTINGS.retry_wait_seconds
 
     def build_chat_session(
         self,
@@ -397,7 +411,10 @@ class APIBackend:
         *,
         shrink_multiple_break: bool = False,
     ) -> list[dict]:
-        """build the messages to avoid implementing several redundant lines of code"""
+        """
+        build the messages to avoid implementing several redundant lines of code
+
+        """
         if former_messages is None:
             former_messages = []
         # shrink multiple break will recursively remove multiple breaks(more than 2)
@@ -407,14 +424,14 @@ class APIBackend:
             if system_prompt is not None:
                 while "\n\n\n" in system_prompt:
                     system_prompt = system_prompt.replace("\n\n\n", "\n\n")
-        system_prompt = self.cfg.default_system_prompt if system_prompt is None else system_prompt
+        system_prompt = LLM_SETTINGS.default_system_prompt if system_prompt is None else system_prompt
         messages = [
             {
                 "role": "system",
                 "content": system_prompt,
             },
         ]
-        messages.extend(former_messages[-1 * self.cfg.max_past_message_include :])
+        messages.extend(former_messages[-1 * LLM_SETTINGS.max_past_message_include :])
         messages.append(
             {
                 "role": "user",
@@ -436,7 +453,10 @@ class APIBackend:
         if former_messages is None:
             former_messages = []
         messages = self.build_messages(
-            user_prompt, system_prompt, former_messages, shrink_multiple_break=shrink_multiple_break
+            user_prompt,
+            system_prompt,
+            former_messages,
+            shrink_multiple_break=shrink_multiple_break,
         )
         return self._try_create_chat_completion_or_embedding(
             messages=messages,
@@ -485,7 +505,7 @@ class APIBackend:
         **kwargs: Any,
     ) -> Any:
         assert not (chat_completion and embedding), "chat_completion and embedding cannot be True at the same time"
-        max_retry = self.cfg.max_retry if self.cfg.max_retry is not None else max_retry
+        max_retry = LLM_SETTINGS.max_retry if LLM_SETTINGS.max_retry is not None else max_retry
         for i in range(max_retry):
             try:
                 if embedding:
@@ -524,21 +544,25 @@ class APIBackend:
             filtered_input_content_list = input_content_list
 
         if len(filtered_input_content_list) > 0:
-            if self.use_azure:
-                response = self.embedding_client.embeddings.create(
-                    model=self.embedding_model,
-                    input=filtered_input_content_list,
-                )
-            else:
-                response = self.embedding_client.embeddings.create(
-                    model=self.embedding_model,
-                    input=filtered_input_content_list,
-                )
-            for index, data in enumerate(response.data):
-                content_to_embedding_dict[filtered_input_content_list[index]] = data.embedding
+            for sliced_filtered_input_content_list in [
+                filtered_input_content_list[i : i + LLM_SETTINGS.embedding_max_str_num]
+                for i in range(0, len(filtered_input_content_list), LLM_SETTINGS.embedding_max_str_num)
+            ]:
+                if self.use_azure:
+                    response = self.embedding_client.embeddings.create(
+                        model=self.embedding_model,
+                        input=sliced_filtered_input_content_list,
+                    )
+                else:
+                    response = self.embedding_client.embeddings.create(
+                        model=self.embedding_model,
+                        input=sliced_filtered_input_content_list,
+                    )
+                for index, data in enumerate(response.data):
+                    content_to_embedding_dict[sliced_filtered_input_content_list[index]] = data.embedding
 
-            if self.dump_embedding_cache:
-                self.cache.embedding_set(content_to_embedding_dict)
+                if self.dump_embedding_cache:
+                    self.cache.embedding_set(content_to_embedding_dict)
         return [content_to_embedding_dict[content] for content in input_content_list]
 
     def _build_log_messages(self, messages: list[dict]) -> str:
@@ -563,30 +587,40 @@ class APIBackend:
         *,
         json_mode: bool = False,
         add_json_in_prompt: bool = False,
+        seed: Optional[int] = None,
     ) -> str:
+        """
+        seed : Optional[int]
+            When retrying with cache enabled, it will keep returning the same results.
+            To make retries useful, we need to enable a seed.
+            This seed is different from `self.chat_seed` for GPT. It is for the local cache mechanism enabled by RD-Agent locally.
+        """
+        if seed is None and LLM_SETTINGS.use_auto_chat_cache_seed_gen:
+            seed = LLM_CACHE_SEED_GEN.get_next_seed()
+
         # TODO: we can add this function back to avoid so much `self.cfg.log_llm_chat_content`
-        if self.cfg.log_llm_chat_content:
+        if LLM_SETTINGS.log_llm_chat_content:
             logger.info(self._build_log_messages(messages), tag="llm_messages")
         # TODO: fail to use loguru adaptor due to stream response
         input_content_json = json.dumps(messages)
         input_content_json = (
-            chat_cache_prefix + input_content_json
+            chat_cache_prefix + input_content_json + f"<seed={seed}/>"
         )  # FIXME this is a hack to make sure the cache represents the round index
         if self.use_chat_cache:
             cache_result = self.cache.chat_get(input_content_json)
             if cache_result is not None:
-                if self.cfg.log_llm_chat_content:
+                if LLM_SETTINGS.log_llm_chat_content:
                     logger.info(f"{LogColors.CYAN}Response:{cache_result}{LogColors.END}", tag="llm_messages")
                 return cache_result, None
 
         if temperature is None:
-            temperature = self.cfg.chat_temperature
+            temperature = LLM_SETTINGS.chat_temperature
         if max_tokens is None:
-            max_tokens = self.cfg.chat_max_tokens
+            max_tokens = LLM_SETTINGS.chat_max_tokens
         if frequency_penalty is None:
-            frequency_penalty = self.cfg.chat_frequency_penalty
+            frequency_penalty = LLM_SETTINGS.chat_frequency_penalty
         if presence_penalty is None:
-            presence_penalty = self.cfg.chat_presence_penalty
+            presence_penalty = LLM_SETTINGS.chat_presence_penalty
 
         finish_reason = None
         if self.use_llama2:
@@ -596,7 +630,7 @@ class APIBackend:
                 temperature=temperature,
             )
             resp = response[0]["generation"]["content"]
-            if self.cfg.log_llm_chat_content:
+            if LLM_SETTINGS.log_llm_chat_content:
                 logger.info(f"{LogColors.CYAN}Response:{resp}{LogColors.END}", tag="llm_messages")
         elif self.use_gcr_endpoint:
             body = str.encode(
@@ -618,7 +652,7 @@ class APIBackend:
             req = urllib.request.Request(self.gcr_endpoint, body, self.headers)  # noqa: S310
             response = urllib.request.urlopen(req)  # noqa: S310
             resp = json.loads(response.read().decode())["output"]
-            if self.cfg.log_llm_chat_content:
+            if LLM_SETTINGS.log_llm_chat_content:
                 logger.info(f"{LogColors.CYAN}Response:{resp}{LogColors.END}", tag="llm_messages")
         else:
             kwargs = dict(
@@ -643,7 +677,7 @@ class APIBackend:
             if self.chat_stream:
                 resp = ""
                 # TODO: with logger.config(stream=self.chat_stream): and add a `stream_start` flag to add timestamp for first message.
-                if self.cfg.log_llm_chat_content:
+                if LLM_SETTINGS.log_llm_chat_content:
                     logger.info(f"{LogColors.CYAN}Response:{LogColors.END}", tag="llm_messages")
 
                 for chunk in response:
@@ -652,19 +686,19 @@ class APIBackend:
                         if len(chunk.choices) > 0 and chunk.choices[0].delta.content is not None
                         else ""
                     )
-                    if self.cfg.log_llm_chat_content:
+                    if LLM_SETTINGS.log_llm_chat_content:
                         logger.info(LogColors.CYAN + content + LogColors.END, raw=True, tag="llm_messages")
                     resp += content
                     if len(chunk.choices) > 0 and chunk.choices[0].finish_reason is not None:
                         finish_reason = chunk.choices[0].finish_reason
 
-                if self.cfg.log_llm_chat_content:
+                if LLM_SETTINGS.log_llm_chat_content:
                     logger.info("\n", raw=True, tag="llm_messages")
 
             else:
                 resp = response.choices[0].message.content
                 finish_reason = response.choices[0].finish_reason
-                if self.cfg.log_llm_chat_content:
+                if LLM_SETTINGS.log_llm_chat_content:
                     logger.info(f"{LogColors.CYAN}Response:{resp}{LogColors.END}", tag="llm_messages")
             if json_mode:
                 json.loads(resp)
@@ -709,26 +743,6 @@ class APIBackend:
         return self.calculate_token_from_messages(messages)
 
 
-def calculate_embedding_process(str_list: list) -> list:
-    return APIBackend().create_embedding(str_list)
-
-
-def create_embedding_with_multiprocessing(str_list: list, slice_count: int = 50, nproc: int = 8) -> list:
-    embeddings = []
-
-    pool = multiprocessing.Pool(nproc)
-    result_list = [
-        pool.apply_async(calculate_embedding_process, (str_list[index : index + slice_count],))
-        for index in range(0, len(str_list), slice_count)
-    ]
-    pool.close()
-    pool.join()
-
-    for res in result_list:
-        embeddings.extend(res.get())
-    return embeddings
-
-
 def calculate_embedding_distance_between_str_list(
     source_str_list: list[str],
     target_str_list: list[str],
@@ -736,7 +750,8 @@ def calculate_embedding_distance_between_str_list(
     if not source_str_list or not target_str_list:
         return [[]]
 
-    embeddings = create_embedding_with_multiprocessing(source_str_list + target_str_list, slice_count=50, nproc=8)
+    embeddings = APIBackend().create_embedding(source_str_list + target_str_list)
+
     source_embeddings = embeddings[: len(source_str_list)]
     target_embeddings = embeddings[len(source_str_list) :]
 

rdagent/scenarios/data_mining/developer/model_runner.py

@@ -1,27 +1,12 @@
-import shutil
-import uuid
-from pathlib import Path
-
-import pandas as pd
-
-from rdagent.components.coder.model_coder.model import ModelExperiment, ModelFBWorkspace
 from rdagent.components.runner import CachedRunner
-from rdagent.components.runner.conf import RUNNER_SETTINGS
-from rdagent.core.developer import Developer
 from rdagent.core.exception import ModelEmptyError
-from rdagent.log import rdagent_logger as logger
+from rdagent.core.utils import cache_with_pickle
 from rdagent.scenarios.data_mining.experiment.model_experiment import DMModelExperiment
-from rdagent.utils.env import DMDockerEnv
 
 
 class DMModelRunner(CachedRunner[DMModelExperiment]):
+    @cache_with_pickle(CachedRunner.get_cache_key, CachedRunner.assign_cached_result)
     def develop(self, exp: DMModelExperiment) -> DMModelExperiment:
-        if RUNNER_SETTINGS.cache_result:
-            cache_hit, result = self.get_cache_result(exp)
-            if cache_hit:
-                exp.result = result
-                return exp
-
         if exp.sub_workspace_list[0].code_dict.get("model.py") is None:
             raise ModelEmptyError("model.py is empty")
         # to replace & inject code
@@ -32,7 +17,5 @@ class DMModelRunner(CachedRunner[DMModelExperiment]):
         result = exp.experiment_workspace.execute(run_env=env_to_use)
 
         exp.result = result
-        if RUNNER_SETTINGS.cache_result:
-            self.dump_cache_result(exp, result)
 
         return exp

rdagent/scenarios/data_mining/docker/Dockerfile

@@ -1,4 +1,4 @@
-FROM pytorch/pytorch:latest
+FROM pytorch/pytorch:2.2.1-cuda12.1-cudnn8-runtime
 # For GPU support, please choose the proper tag from https://hub.docker.com/r/pytorch/pytorch/tags
 
 RUN apt-get clean && apt-get update && apt-get install -y \