chore(main): release 0.3.0 (#258)

Release-As: 0.3.0

.commitlintrc.js

@@ -0,0 +1,21 @@
+module.exports = {
+    extends: ["@commitlint/config-conventional"],
+    rules: {
+        // Configuration Format: [level, applicability, value]
+        // level: Error level, usually expressed as a number:
+        //     0 - disable rule
+        //     1 - Warning (does not prevent commits)
+        //     2 - Error (will block the commit)
+        // applicability: the conditions under which the rule applies, commonly used values:
+        //     “always” - always apply the rule
+        //     “never” - never apply the rule
+        // value: the specific value of the rule, e.g. a maximum length of 100.
+        // Refs: https://commitlint.js.org/reference/rules-configuration.html
+      "header-max-length": [2, "always", 100],
+      "type-enum": [
+        2,
+        "always",
+        ["build", "chore", "ci", "docs", "feat", "fix", "perf", "refactor", "revert", "style", "test", "Release-As"]
+      ]
+    }
+  };

.github/workflows/pr.yml

@@ -1,18 +1,5 @@
-concurrency:
-  cancel-in-progress: true
-  group: ${{ github.workflow }}-${{ github.ref }}
-jobs:
-  lint-title:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Check PR Title for Conventional Commit Format
-        run: |
-          if ! echo "${{ github.event.pull_request.title }}" | grep -Pq '^(build|chore|ci|docs|feat|fix|perf|refactor|revert|style|test|Release-As)(\(\w+\))?!?:\s.*'; then
-            echo 'The title does not conform to the Conventional Commit.'
-            echo 'Please refer to "https://www.conventionalcommits.org/"'
-            exit 1
-          fi
 name: Lint pull request title
+
 on:
   pull_request:
     types:
@@ -20,3 +7,29 @@ on:
       - synchronize
       - reopened
       - edited
+
+concurrency:
+  cancel-in-progress: true
+  group: ${{ github.workflow }}-${{ github.ref }}
+
+jobs:
+  lint-title:
+    runs-on: ubuntu-latest
+    steps:
+      # This step is necessary because the lint title uses the .commitlintrc.js file in the project root directory.
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v3
+        with:
+          node-version: '16'
+
+      - name: Install commitlint
+        run: npm install --save-dev @commitlint/{config-conventional,cli}
+
+      - name: Validate PR Title with commitlint
+        env:
+          BODY: ${{ github.event.pull_request.title }}
+        run: |
+          echo "$BODY" | npx commitlint --config .commitlintrc.js

CHANGELOG.md

@@ -1,5 +1,111 @@
 # Changelog
 
+## [0.3.0](https://github.com/microsoft/RD-Agent/compare/v0.2.1...v0.3.0) (2024-10-21)
+
+
+### Features
+
+* add a new template for kaggle ([#289](https://github.com/microsoft/RD-Agent/issues/289)) ([eee3ab5](https://github.com/microsoft/RD-Agent/commit/eee3ab5b25198224826cb7a8a17eab28bd5d1f7d))
+* add download submission.csv button for kaggle scenario ([#317](https://github.com/microsoft/RD-Agent/issues/317)) ([dcdcbe4](https://github.com/microsoft/RD-Agent/commit/dcdcbe46b4858bfb133ae3cca056e7f602d5cf63))
+* add kaggle command ([#271](https://github.com/microsoft/RD-Agent/issues/271)) ([0938394](https://github.com/microsoft/RD-Agent/commit/0938394b7084ffbf3294d8c23d2d34bf7322ca0b))
+* add kaggle tpl: feedback-prize ([#331](https://github.com/microsoft/RD-Agent/issues/331)) ([a288e39](https://github.com/microsoft/RD-Agent/commit/a288e399e6b0beec62729bd7d46b98a55de5ab79))
+* add more templates for kaggle ([#291](https://github.com/microsoft/RD-Agent/issues/291)) ([da752ec](https://github.com/microsoft/RD-Agent/commit/da752ec806e6f5f5679bc27ac1c072ed9a319251))
+* add normal rag into framework ([#360](https://github.com/microsoft/RD-Agent/issues/360)) ([91b0b1f](https://github.com/microsoft/RD-Agent/commit/91b0b1f66c3c1bf757cb64c4cfbdcaafe59eab74))
+* add qlib_factor_strategy ([#307](https://github.com/microsoft/RD-Agent/issues/307)) ([f8f59ff](https://github.com/microsoft/RD-Agent/commit/f8f59ff0a1be4428a68c8c27f220aabad0b6c9f0))
+* Add ranking in kaggle scenario ([#401](https://github.com/microsoft/RD-Agent/issues/401)) ([b16b4be](https://github.com/microsoft/RD-Agent/commit/b16b4beb402e0c27dfb39ee9d2a120f1b56d447c))
+* Add runtime measurement for each step and loop in RDLoop. ([#281](https://github.com/microsoft/RD-Agent/issues/281)) ([83058c8](https://github.com/microsoft/RD-Agent/commit/83058c864ceeec413dd29bf501030d5a7bd34679))
+* add s3e11 kaggle template ([#324](https://github.com/microsoft/RD-Agent/issues/324)) ([8c57524](https://github.com/microsoft/RD-Agent/commit/8c57524bead1c8f655a08763d608eb7a6dd5975e))
+* Added RepoAnalyzer to empower auto-summary of a workspace ([#264](https://github.com/microsoft/RD-Agent/issues/264)) ([0bd349a](https://github.com/microsoft/RD-Agent/commit/0bd349af50b9b881ba1774bdeb4d723529ef2aa9))
+* Added support for loading and storing RAG in Kaggle scenarios. ([#269](https://github.com/microsoft/RD-Agent/issues/269)) ([c4895de](https://github.com/microsoft/RD-Agent/commit/c4895de83f1ed000e563d42b3468a6bd9e5a4965))
+* announce Discord and WeChat ([#367](https://github.com/microsoft/RD-Agent/issues/367)) ([acac507](https://github.com/microsoft/RD-Agent/commit/acac5078a103b71afa6bd6c053b0766a6a7e609d))
+* auto submit result after one kaggle RDLoop ([#345](https://github.com/microsoft/RD-Agent/issues/345)) ([ab55d70](https://github.com/microsoft/RD-Agent/commit/ab55d7052b53a928b84dc5d5d0d2999d90ca9056))
+* better feedback & evaluation ([#346](https://github.com/microsoft/RD-Agent/issues/346)) ([cc9a8c1](https://github.com/microsoft/RD-Agent/commit/cc9a8c1eab3ca89f8c1e5de4a2bb4e7fcc0cc615))
+* Dynamic scenario based on task ([#392](https://github.com/microsoft/RD-Agent/issues/392)) ([665a037](https://github.com/microsoft/RD-Agent/commit/665a037e4fd7326c450e3fa0d0605eea26fd9ef3))
+* Factor Implement Search Enhancement ([#294](https://github.com/microsoft/RD-Agent/issues/294)) ([4ecf25f](https://github.com/microsoft/RD-Agent/commit/4ecf25f0acf2389a172b14d3dab20895daf2ab89))
+* Feature selection v3 to support all actions  ([#280](https://github.com/microsoft/RD-Agent/issues/280)) ([0047641](https://github.com/microsoft/RD-Agent/commit/00476413fbf00e36e71ab3ccb48d4e766b6ccf4d))
+* fix some bugs and add original features' description ([#259](https://github.com/microsoft/RD-Agent/issues/259)) ([1a5f45a](https://github.com/microsoft/RD-Agent/commit/1a5f45a40d821c017bdba14af8c93710707c5ea5))
+* get kaggle notebooks & disscussion text for RAG ([#371](https://github.com/microsoft/RD-Agent/issues/371)) ([cead345](https://github.com/microsoft/RD-Agent/commit/cead3450a14bf4b142ac988c27fa098c7656a95c))
+* Iceberge competition ([#372](https://github.com/microsoft/RD-Agent/issues/372)) ([c10ea4f](https://github.com/microsoft/RD-Agent/commit/c10ea4f5d4cc56a75b47cf23c7084ee189ba1a25))
+* implement isolated model feature selection loop ([#370](https://github.com/microsoft/RD-Agent/issues/370)) ([cf1292d](https://github.com/microsoft/RD-Agent/commit/cf1292de1a0153ca14ea64971e73a1c93f7d89e3))
+* Initial version if Graph RAG in KAGGLE scenario ([#301](https://github.com/microsoft/RD-Agent/issues/301)) ([fd3c0fd](https://github.com/microsoft/RD-Agent/commit/fd3c0fd26eff7d3be72fa4f2a234e33b9f796627))
+* Integrate RAG into the Kaggle scenarios. ([#262](https://github.com/microsoft/RD-Agent/issues/262)) ([be0e48a](https://github.com/microsoft/RD-Agent/commit/be0e48a7dfbee2b5d2947d09115db5db2e5266f1))
+* Kaggle loop update (Feature & Model) ([#241](https://github.com/microsoft/RD-Agent/issues/241)) ([4cf22a6](https://github.com/microsoft/RD-Agent/commit/4cf22a65c964123b4267569ee02c0c7094c54ca4))
+* kaggle templates related ([#287](https://github.com/microsoft/RD-Agent/issues/287)) ([785fdc1](https://github.com/microsoft/RD-Agent/commit/785fdc144d16fa8454b7c9d2e53e78fe7f22a29a))
+* Model context for tuning and selection ([#284](https://github.com/microsoft/RD-Agent/issues/284)) ([f2831e7](https://github.com/microsoft/RD-Agent/commit/f2831e7442510668b0ca75953b3359894803ef3c))
+* Modify FactorRowCountEvaluator and FactorIndexEvaluator to return the ratio ([#328](https://github.com/microsoft/RD-Agent/issues/328)) ([8f43f8e](https://github.com/microsoft/RD-Agent/commit/8f43f8e87a92e05b541e925910608606ec8f6c4b))
+* New competition - Optiver ([#356](https://github.com/microsoft/RD-Agent/issues/356)) ([3705efe](https://github.com/microsoft/RD-Agent/commit/3705efe3b923748655a57d76b7a236e54d361831))
+* random forest for s3e11 ([#347](https://github.com/microsoft/RD-Agent/issues/347)) ([b57846d](https://github.com/microsoft/RD-Agent/commit/b57846d29314e9a5967945d1b4895f0f48c0f5ce))
+* refine the code in model description and fix some bugs in feedback.py ([#288](https://github.com/microsoft/RD-Agent/issues/288)) ([5b124d7](https://github.com/microsoft/RD-Agent/commit/5b124d7372137e4c613eb2749ddcc773922cc7b6))
+* refine the template in several Kaggle competitions ([#343](https://github.com/microsoft/RD-Agent/issues/343)) ([034f238](https://github.com/microsoft/RD-Agent/commit/034f238ed5ec351486b21250eabc75114961936c))
+* Revise to support better hypothesis proposal  ([#390](https://github.com/microsoft/RD-Agent/issues/390)) ([c55ec0a](https://github.com/microsoft/RD-Agent/commit/c55ec0a0f577bbf7fc6228f7b87d2089ded83b31))
+* show workspace in demo ([#348](https://github.com/microsoft/RD-Agent/issues/348)) ([ddf567c](https://github.com/microsoft/RD-Agent/commit/ddf567c551b553788be022e9312c209ef6137d64))
+* support Multi output ([#330](https://github.com/microsoft/RD-Agent/issues/330)) ([3d36c45](https://github.com/microsoft/RD-Agent/commit/3d36c452ff0983800e5343834cc69f24a508ea70))
+* Supporting COVID-19 competition ([#374](https://github.com/microsoft/RD-Agent/issues/374)) ([a1b63db](https://github.com/microsoft/RD-Agent/commit/a1b63db79600edc9a74ba713c9d0be290214a592))
+* supporting Mnist competition ([#375](https://github.com/microsoft/RD-Agent/issues/375)) ([e958a34](https://github.com/microsoft/RD-Agent/commit/e958a34f5632a46ac43bff8e0d07d6ed020fdfc2))
+* Supporting Model Specifications ([#319](https://github.com/microsoft/RD-Agent/issues/319)) ([e126471](https://github.com/microsoft/RD-Agent/commit/e1264719e10b76158a91cd0ef331848e7c2de7c7))
+* supporting various Kaggle competitions & scenarios for RD-Agent ([#409](https://github.com/microsoft/RD-Agent/issues/409)) ([75eea22](https://github.com/microsoft/RD-Agent/commit/75eea22cc3d4e6f5a94c88cce915e27c507f8c50))
+* template for kaggle ([#308](https://github.com/microsoft/RD-Agent/issues/308)) ([ff97cf0](https://github.com/microsoft/RD-Agent/commit/ff97cf0155ab6941e4b5cf7d103575f934b70dc9))
+* use auto gen seed when using LLM cache ([#441](https://github.com/microsoft/RD-Agent/issues/441)) ([ca15365](https://github.com/microsoft/RD-Agent/commit/ca15365d23eeb094f42cf3dc8f5269b2f1c42bd3))
+* use unified pickle cacher & move llm config into a isolated config ([#424](https://github.com/microsoft/RD-Agent/issues/424)) ([2879ecf](https://github.com/microsoft/RD-Agent/commit/2879ecff816d97688b60909a79c7e568d42608a1))
+* xgboost gpu accelerate ([#359](https://github.com/microsoft/RD-Agent/issues/359)) ([56a5b8f](https://github.com/microsoft/RD-Agent/commit/56a5b8f9b2c6726cc64ec5b04b4ce7935d59b572))
+
+
+### Bug Fixes
+
+* a bug of developer& edit s4e8 template ([#338](https://github.com/microsoft/RD-Agent/issues/338)) ([f12ce72](https://github.com/microsoft/RD-Agent/commit/f12ce726e7de96d478a232a3c27f92439820f8b4))
+* actively raised errors aer also considered as negative feedback. ([#268](https://github.com/microsoft/RD-Agent/issues/268)) ([46ec908](https://github.com/microsoft/RD-Agent/commit/46ec908e3594ac5e4cdc4057268e2f8800f5ed1f))
+* bug of saving preprocess cache files ([#310](https://github.com/microsoft/RD-Agent/issues/310)) ([5fb0608](https://github.com/microsoft/RD-Agent/commit/5fb0608f39f113cc9807fb1f381284a0bd4da318))
+* cache ([#383](https://github.com/microsoft/RD-Agent/issues/383)) ([f2a6e75](https://github.com/microsoft/RD-Agent/commit/f2a6e75b36ca96f7733b9c2a7154ac67bd2d7c6f))
+* change css tag of kaggle competition info crawler ([#306](https://github.com/microsoft/RD-Agent/issues/306)) ([1e3d38b](https://github.com/microsoft/RD-Agent/commit/1e3d38bf1ca3654f3a90ff392ecba1dbb4e80224))
+* debug dsagent ([#387](https://github.com/microsoft/RD-Agent/issues/387)) ([8fe9511](https://github.com/microsoft/RD-Agent/commit/8fe9511e606ba148c66f384add6ab94857079541))
+* eval_method cannot catch run factor error ([#260](https://github.com/microsoft/RD-Agent/issues/260)) ([2aaab31](https://github.com/microsoft/RD-Agent/commit/2aaab317ccb7a0121063bcd85fc36c21c7b8a391))
+* fix a bug in competition metric evaluation ([#407](https://github.com/microsoft/RD-Agent/issues/407)) ([94c47d6](https://github.com/microsoft/RD-Agent/commit/94c47d6fd5c3e38fc786a83e6d0d05e8d04498f3))
+* fix a bug in mini case ([#389](https://github.com/microsoft/RD-Agent/issues/389)) ([e75bb57](https://github.com/microsoft/RD-Agent/commit/e75bb5746f63933b750406bbd34ee63c5ba76b9f))
+* fix a bug in model tuning feedback ([#316](https://github.com/microsoft/RD-Agent/issues/316)) ([8aa088d](https://github.com/microsoft/RD-Agent/commit/8aa088da2dc7525a3970c01d01987246f47d6238))
+* fix a bug in scenario.py ([#388](https://github.com/microsoft/RD-Agent/issues/388)) ([999a1eb](https://github.com/microsoft/RD-Agent/commit/999a1eb0eff9088e1b02419db741db4acf8d9ff7))
+* fix a bug in the format of the model input ([#327](https://github.com/microsoft/RD-Agent/issues/327)) ([8f0574e](https://github.com/microsoft/RD-Agent/commit/8f0574eaaadb245b8c38e09ad4821306996d926f))
+* fix a small bug in cache using module name and function name as unique folder name ([#429](https://github.com/microsoft/RD-Agent/issues/429)) ([4f8134a](https://github.com/microsoft/RD-Agent/commit/4f8134a697d952f7ac824d7ebeec64bbc4545ab3))
+* fix a typo ([#362](https://github.com/microsoft/RD-Agent/issues/362)) ([9fafabd](https://github.com/microsoft/RD-Agent/commit/9fafabdf321b818bdd2211a2324d50cd0ebe1c1f))
+* fix cache result logic ([#430](https://github.com/microsoft/RD-Agent/issues/430)) ([5e34263](https://github.com/microsoft/RD-Agent/commit/5e342637dcc862679fd0642c6ba9ef048c984845))
+* fix command injection ([#421](https://github.com/microsoft/RD-Agent/issues/421)) ([52f30a6](https://github.com/microsoft/RD-Agent/commit/52f30a6184af1295be15e855a80b84bc424fc75d))
+* fix json load error ([#386](https://github.com/microsoft/RD-Agent/issues/386)) ([bba55fb](https://github.com/microsoft/RD-Agent/commit/bba55fb48fe105f4847c1b9c476eedc80835f523))
+* fix some bugs in feedback.py and refine the prompt ([#292](https://github.com/microsoft/RD-Agent/issues/292)) ([d834052](https://github.com/microsoft/RD-Agent/commit/d8340527f133dcc649d599d90d6402eddd37859e))
+* fix some bugs in knowledge base ([#378](https://github.com/microsoft/RD-Agent/issues/378)) ([fa6ff8e](https://github.com/microsoft/RD-Agent/commit/fa6ff8e591cf1847df77d73116649c5623161573))
+* fix some bugs in rag ([#399](https://github.com/microsoft/RD-Agent/issues/399)) ([194215c](https://github.com/microsoft/RD-Agent/commit/194215c4559aee5b6ece18d65c95fb30968e2db6))
+* fix some bugs in the entire loop ([#274](https://github.com/microsoft/RD-Agent/issues/274)) ([8a564ec](https://github.com/microsoft/RD-Agent/commit/8a564ece1d87b27ee98b76db317935e802468965))
+* fix some errors in scenario.py, proposal.py and runner.py and several complex competition scenarios([#365](https://github.com/microsoft/RD-Agent/issues/365)) ([2e383b1](https://github.com/microsoft/RD-Agent/commit/2e383b175d8448a67cb470f4e3ae8977d8ec6b5b))
+* improve_execution_time_in_kaggle_loop ([#279](https://github.com/microsoft/RD-Agent/issues/279)) ([4c8f998](https://github.com/microsoft/RD-Agent/commit/4c8f998c76f1e983a5687d2c65d3251750f2a9a0))
+* kaggle data mount problem ([#297](https://github.com/microsoft/RD-Agent/issues/297)) ([795df31](https://github.com/microsoft/RD-Agent/commit/795df311e3f93cd2f3fb51ba5698adaf10f6bd62))
+* Optiver fixes ([#357](https://github.com/microsoft/RD-Agent/issues/357)) ([b054017](https://github.com/microsoft/RD-Agent/commit/b054017463af0d1784407030f2477d212118f341))
+* partial bug in bench ([#368](https://github.com/microsoft/RD-Agent/issues/368)) ([af9808f](https://github.com/microsoft/RD-Agent/commit/af9808f98736a2df07e121c2f6d7bfeb7b7d3581))
+* preprocess output format & some mistake in spelling ([#358](https://github.com/microsoft/RD-Agent/issues/358)) ([b8b2cd6](https://github.com/microsoft/RD-Agent/commit/b8b2cd6ccd3b27aa73de847e50899a8a53b71b8f))
+* rag save file ([#385](https://github.com/microsoft/RD-Agent/issues/385)) ([1cb01dd](https://github.com/microsoft/RD-Agent/commit/1cb01dd6fe595f2f5fb86487601326611dd1a57a))
+* raise error in demo when no Metric in a Loop ([#313](https://github.com/microsoft/RD-Agent/issues/313)) ([e46a78e](https://github.com/microsoft/RD-Agent/commit/e46a78eb69271cb19978aab2f3b976c2870ca082))
+* refactor Bench ([#302](https://github.com/microsoft/RD-Agent/issues/302)) ([78a87f6](https://github.com/microsoft/RD-Agent/commit/78a87f624780ff67c0fa995ae4692678a120f99c))
+* refine some codes ([#353](https://github.com/microsoft/RD-Agent/issues/353)) ([866c2e6](https://github.com/microsoft/RD-Agent/commit/866c2e63ffa3876a3d16ad37f96da41d0558b714))
+* refine the prompt ([#286](https://github.com/microsoft/RD-Agent/issues/286)) ([77966c4](https://github.com/microsoft/RD-Agent/commit/77966c4f5e9f492c437c5b4b78d89c0f875ef0d8))
+* refine the ucb algorithm ([#406](https://github.com/microsoft/RD-Agent/issues/406)) ([14f7d97](https://github.com/microsoft/RD-Agent/commit/14f7d976e03c92d6e727524e0cdad8a03b585016))
+* revert model and make SOTA model available to COSTEER ([#351](https://github.com/microsoft/RD-Agent/issues/351)) ([3b7437b](https://github.com/microsoft/RD-Agent/commit/3b7437b87e685188259779cd85a78a0b592de9de))
+* stop using markup in docker env print ([#336](https://github.com/microsoft/RD-Agent/issues/336)) ([3009889](https://github.com/microsoft/RD-Agent/commit/3009889b5e2605b5427c76f3084e0e58026bb5ae))
+* support seed and fix absolute path ([#278](https://github.com/microsoft/RD-Agent/issues/278)) ([26352e1](https://github.com/microsoft/RD-Agent/commit/26352e13121cad5be95c0de78bb9f5dda4330614))
+* template for kaggle foreset & s4e9 ([#334](https://github.com/microsoft/RD-Agent/issues/334)) ([2393a41](https://github.com/microsoft/RD-Agent/commit/2393a41e7237615ced2c3fdd5c49308236b9f276))
+* test kaggle method ([#296](https://github.com/microsoft/RD-Agent/issues/296)) ([91a6196](https://github.com/microsoft/RD-Agent/commit/91a619618be1d7db660ea2b413a78dfaba9417a1))
+* update code to fix a small bug in model cache md5 hash ([#303](https://github.com/microsoft/RD-Agent/issues/303)) ([b00e4dc](https://github.com/microsoft/RD-Agent/commit/b00e4dc2eff5b16029a2a12a6589eadac5cfd148))
+* update new feature engineering code format ([#272](https://github.com/microsoft/RD-Agent/issues/272)) ([7850b80](https://github.com/microsoft/RD-Agent/commit/7850b8006a7c89d22629b345b4f361b0f35bc60d))
+* Update prompts.yaml to constrain only one model type   ([#341](https://github.com/microsoft/RD-Agent/issues/341)) ([5b5dfee](https://github.com/microsoft/RD-Agent/commit/5b5dfeefbc7eb9dcbd9923544005c5d281262c03))
+* Update runner.py to fix a small bug ([#282](https://github.com/microsoft/RD-Agent/issues/282)) ([8aef3ab](https://github.com/microsoft/RD-Agent/commit/8aef3abcecd6002bd4bfeedcbe2c786d8bbfe2be))
+* Use fixed file name in model costeer & fixing cache ([#311](https://github.com/microsoft/RD-Agent/issues/311)) ([1f910a5](https://github.com/microsoft/RD-Agent/commit/1f910a5248bc576895ed66c2f7b2c3e046a2bc28))
+
+
+### Performance Improvements
+
+* some small upgrade to factor costeer to improve the performance ([#420](https://github.com/microsoft/RD-Agent/issues/420)) ([9eb931f](https://github.com/microsoft/RD-Agent/commit/9eb931ffd971f252380dbd33ad1db259a4f229fd))
+
+
+### Reverts
+
+* Revert feat: Factor Implement Search Enhancement ([#294](https://github.com/microsoft/RD-Agent/issues/294)) ([#305](https://github.com/microsoft/RD-Agent/issues/305)) ([f663cf4](https://github.com/microsoft/RD-Agent/commit/f663cf42a2f75cd52aef1c6b18be7c27f0641fed))
+
 ## [0.2.1](https://github.com/microsoft/RD-Agent/compare/v0.2.0...v0.2.1) (2024-09-10)
 
 

Makefile

@@ -97,7 +97,7 @@ mypy:
 # First deal with the core folder, and then gradually increase the scope of detection,
 # and eventually realize the detection of the complete project.
 ruff:
-	$(PIPRUN) ruff check rdagent/core --ignore FBT001,FBT002   # --exclude rdagent/scripts,git_ignore_folder
+	$(PIPRUN) ruff check rdagent/core --ignore FBT001,FBT002,I001   # --exclude rdagent/scripts,git_ignore_folder
 
 # Check lint with toml-sort.
 toml-sort:

README.md

@@ -1,9 +1,10 @@
-<h3 align="center">
+<h4 align="center">
   <img src="docs/_static/logo.png" alt="RA-Agent logo" style="width:70%; ">
   
-  <a href="https://rdagent.azurewebsites.net" target="_blank">🖥️ Live Demo</a> | <a href="https://rdagent.azurewebsites.net/factor_loop" target="_blank">🎥 Demo Video</a>  | <a href="https://rdagent.readthedocs.io/en/latest/index.html" target="_blank">📖 Documentation</a> | <a href="#-paperwork-list"> 📃 Papers </a>
+  <a href="https://rdagent.azurewebsites.net" target="_blank">🖥️ Live Demo</a> | <a href="https://rdagent.azurewebsites.net/factor_loop" target="_blank">🎥 Demo Video</a> <a href="https://www.youtube.com/watch?v=JJ4JYO3HscM&list=PLALmKB0_N3_i52fhUmPQiL4jsO354uopR" target="_blank">▶️YouTube</a>   | <a href="https://rdagent.readthedocs.io/en/latest/index.html" target="_blank">📖 Documentation</a> | <a href="#-paperwork-list"> 📃 Papers </a>
 </h3>
 
+
 [![CI](https://github.com/microsoft/RD-Agent/actions/workflows/ci.yml/badge.svg)](https://github.com/microsoft/RD-Agent/actions/workflows/ci.yml)
 [![CodeQL](https://github.com/microsoft/RD-Agent/actions/workflows/github-code-scanning/codeql/badge.svg)](https://github.com/microsoft/RD-Agent/actions/workflows/github-code-scanning/codeql)
 [![Dependabot Updates](https://github.com/microsoft/RD-Agent/actions/workflows/dependabot/dependabot-updates/badge.svg)](https://github.com/microsoft/RD-Agent/actions/workflows/dependabot/dependabot-updates)
@@ -17,12 +18,15 @@
 [![pre-commit](https://img.shields.io/badge/pre--commit-enabled-brightgreen?logo=pre-commit)](https://github.com/pre-commit/pre-commit)
 [![Checked with mypy](https://www.mypy-lang.org/static/mypy_badge.svg)](http://mypy-lang.org/)
 [![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)
+[![Chat](https://img.shields.io/badge/chat-discord-blue)](https://discord.gg/ybQ97B6Jjy)
 [![Readthedocs Preview](https://github.com/microsoft/RD-Agent/actions/workflows/readthedocs-preview.yml/badge.svg)](https://github.com/microsoft/RD-Agent/actions/workflows/readthedocs-preview.yml) <!-- this badge is too long, please place it in the last one to make it pretty --> 
 
 # 📰 News
 | 🗞️ News        | 📝 Description                 |
 | --            | ------                        |
-| First release | **RDAgent** is released on Github |
+| Official WeChat group release  | We created a WeChat group, welcome to join! (🗪[QR Code](docs/WeChat_QR_code.jpg)) |
+| Official Discord release  | We launch our first chatting channel in Discord (🗪[![Chat](https://img.shields.io/badge/chat-discord-blue)](https://discord.gg/ybQ97B6Jjy)) |
+| First release | **RDAgent** is released on GitHub |
 
 
 # 🌟 Introduction
@@ -37,9 +41,9 @@ We believe that the automatic evolution of R&D will lead to solutions of signifi
 
 <!-- Tag Cloud -->
 R&D is a very general scenario. The advent of RDAgent can be your
-- 💰 **Automatic Quant Factory** [(🎥Demo Video)](https://rdagent.azurewebsites.net/factor_loop)
-- 🤖 **Data Mining Agent:** Iteratively proposing data [(🎥Demo Video)](https://rdagent.azurewebsites.net/dmm) & models [(🎥Demo Video)](https://rdagent.azurewebsites.net/model_loop) and implementing them by gaining knowledge from data.
-- 🦾 **Research Copilot:** Auto read research papers [(🎥Demo Video)](https://rdagent.azurewebsites.net/report_model) / financial reports [(🎥Demo Video)](https://rdagent.azurewebsites.net/report_factor) and implement model structures or building datasets.
+- 💰 **Automatic Quant Factory** ([🎥Demo Video](https://rdagent.azurewebsites.net/factor_loop)|[▶️YouTube](https://www.youtube.com/watch?v=X4DK2QZKaKY&t=6s))
+- 🤖 **Data Mining Agent:** Iteratively proposing data & models ([🎥Demo Video 1](https://rdagent.azurewebsites.net/model_loop)|[▶️YouTube](https://www.youtube.com/watch?v=dm0dWL49Bc0&t=104s)) ([🎥Demo Video 2](https://rdagent.azurewebsites.net/dmm)|[▶️YouTube](https://www.youtube.com/watch?v=VIaSTZuoZg4))  and implementing them by gaining knowledge from data.
+- 🦾 **Research Copilot:** Auto read research papers ([🎥Demo Video](https://rdagent.azurewebsites.net/report_model)|[▶️YouTube](https://www.youtube.com/watch?v=BiA2SfdKQ7o)) / financial reports ([🎥Demo Video](https://rdagent.azurewebsites.net/report_factor)|[▶️YouTube](https://www.youtube.com/watch?v=ECLTXVcSx-c)) and implement model structures or building datasets.
 - ...
 
 You can click the links above to view the demo. We're continuously adding more methods and scenarios to the project to enhance your R&D processes and boost productivity. 
@@ -164,9 +168,9 @@ The supported scenarios are listed below:
 
 | Scenario/Target | Model Implementation                   | Data Building                                                                      |
 | --              | --                                     | --                                                                                 |
-| **💹 Finance**      | 🤖 [Iteratively Proposing Ideas & Evolving](https://rdagent.azurewebsites.net/model_loop) |  🤖 [Iteratively Proposing Ideas & Evolving](https://rdagent.azurewebsites.net/factor_loop)  <br/>   🦾 [Auto reports reading & implementation](https://rdagent.azurewebsites.net/report_factor)  |
-| **🩺 Medical**      | 🤖 [Iteratively Proposing Ideas & Evolving](https://rdagent.azurewebsites.net/dmm) | -                                                                                  |
-| **🏭 General**      | 🦾 [Auto paper reading & implementation](https://rdagent.azurewebsites.net/report_model)    | -                                                                                  |
+| **💹 Finance**      | 🤖 [Iteratively Proposing Ideas & Evolving](https://rdagent.azurewebsites.net/model_loop)[▶️YouTube](https://www.youtube.com/watch?v=dm0dWL49Bc0&t=104s) |  🤖 [Iteratively Proposing Ideas & Evolving](https://rdagent.azurewebsites.net/factor_loop) [▶️YouTube](https://www.youtube.com/watch?v=X4DK2QZKaKY&t=6s) <br/>   🦾 [Auto reports reading & implementation](https://rdagent.azurewebsites.net/report_factor)[▶️YouTube](https://www.youtube.com/watch?v=ECLTXVcSx-c)  |
+| **🩺 Medical**      | 🤖 [Iteratively Proposing Ideas & Evolving](https://rdagent.azurewebsites.net/dmm)[▶️YouTube](https://www.youtube.com/watch?v=VIaSTZuoZg4) | -                                                                                  |
+| **🏭 General**      | 🦾 [Auto paper reading & implementation](https://rdagent.azurewebsites.net/report_model)[▶️YouTube](https://www.youtube.com/watch?v=BiA2SfdKQ7o)    | -                                                                                  |
 
 Different scenarios vary in entrance and configuration. Please check the detailed setup tutorial in the scenarios documents.
 

constraints/3.10.txt

@@ -71,7 +71,6 @@ isort==5.13.2
 jaraco.classes==3.3.0
 jedi==0.19.1
 jeepney==0.8.0
-Jinja2==3.1.2
 joblib==1.4.2
 json5==0.9.25
 jsonpatch==1.33
@@ -102,7 +101,6 @@ loguru==0.7.2
 loguru-mypy==0.0.4
 lxml==5.0.0
 markdown-it-py==3.0.0
-MarkupSafe==2.1.3
 marshmallow==3.20.1
 matplotlib==3.9.1
 matplotlib-inline==0.1.7
@@ -110,7 +108,6 @@ mdit-py-plugins==0.4.0
 mdurl==0.1.2
 mistune==3.0.2
 more-itertools==10.1.0
-mpmath==1.3.0
 msal==1.30.0
 msal-extensions==1.2.0
 msgpack==1.0.8
@@ -124,24 +121,11 @@ nbconvert==7.16.4
 nbformat==5.10.4
 ndindex==1.8
 nest-asyncio==1.6.0
-networkx==3.2.1
 nh3==0.2.15
 notebook==7.2.1
 notebook_shim==0.2.4
 numexpr==2.10.1
 numpy==1.26.2
-nvidia-cublas-cu12==12.1.3.1
-nvidia-cuda-cupti-cu12==12.1.105
-nvidia-cuda-nvrtc-cu12==12.1.105
-nvidia-cuda-runtime-cu12==12.1.105
-nvidia-cudnn-cu12==8.9.2.26
-nvidia-cufft-cu12==11.0.2.54
-nvidia-curand-cu12==10.3.2.106
-nvidia-cusolver-cu12==11.4.5.107
-nvidia-cusparse-cu12==12.1.0.106
-nvidia-nccl-cu12==2.18.1
-nvidia-nvjitlink-cu12==12.3.101
-nvidia-nvtx-cu12==12.1.105
 oauthlib==3.2.2
 openai==1.6.1
 overrides==7.7.0
@@ -153,7 +137,6 @@ parso==0.8.4
 pathspec==0.12.1
 patsy==0.5.6
 pexpect==4.9.0
-pillow==10.4.0
 pkginfo==1.9.6
 platformdirs==4.1.0
 pluggy==1.3.0
@@ -201,7 +184,6 @@ ruamel.yaml==0.18.5
 ruamel.yaml.clib==0.2.8
 ruff==0.4.5
 scikit-learn==1.5.1
-scipy==1.11.4
 SecretStorage==3.3.3
 semver==3.0.2
 Send2Trash==1.8.3
@@ -226,7 +208,6 @@ sphinxcontrib-serializinghtml==1.1.9
 SQLAlchemy==2.0.24
 stack-data==0.6.3
 statsmodels==0.14.2
-sympy==1.12
 tables==3.9.2
 tabulate==0.9.0
 tenacity==8.2.3
@@ -238,14 +219,11 @@ tinycss2==1.3.0
 toml-sort==0.23.1
 tomli==2.0.1
 tomlkit==0.12.3
-torch==2.1.2
-torch_geometric==2.5.3
 tornado==6.4
 tqdm==4.66.1
 traitlets==5.14.3
 tree-sitter==0.22.3
 tree-sitter-python==0.21.0
-triton==2.1.0
 twine==4.0.2
 typer==0.9.0
 types-psutil==6.0.0.20240621
@@ -253,7 +231,6 @@ types-python-dateutil==2.9.0.20240316
 types-PyYAML==6.0.12.20240724
 types-tqdm==4.66.0.20240417
 typing-inspect==0.9.0
-typing_extensions==4.9.0
 tzdata==2023.4
 uri-template==1.3.0
 urllib3==2.1.0

constraints/3.11.txt

@@ -69,7 +69,6 @@ isort==5.13.2
 jaraco.classes==3.3.0
 jedi==0.19.1
 jeepney==0.8.0
-Jinja2==3.1.2
 joblib==1.4.2
 json5==0.9.25
 jsonpatch==1.33
@@ -100,7 +99,6 @@ loguru==0.7.2
 loguru-mypy==0.0.4
 lxml==5.0.0
 markdown-it-py==3.0.0
-MarkupSafe==2.1.3
 marshmallow==3.20.1
 matplotlib==3.9.1
 matplotlib-inline==0.1.7
@@ -108,7 +106,6 @@ mdit-py-plugins==0.4.0
 mdurl==0.1.2
 mistune==3.0.2
 more-itertools==10.1.0
-mpmath==1.3.0
 msal==1.30.0
 msal-extensions==1.2.0
 msgpack==1.0.8
@@ -122,24 +119,11 @@ nbconvert==7.16.4
 nbformat==5.10.4
 ndindex==1.8
 nest-asyncio==1.6.0
-networkx==3.2.1
 nh3==0.2.15
 notebook==7.2.1
 notebook_shim==0.2.4
 numexpr==2.10.1
 numpy==1.26.2
-nvidia-cublas-cu12==12.1.3.1
-nvidia-cuda-cupti-cu12==12.1.105
-nvidia-cuda-nvrtc-cu12==12.1.105
-nvidia-cuda-runtime-cu12==12.1.105
-nvidia-cudnn-cu12==8.9.2.26
-nvidia-cufft-cu12==11.0.2.54
-nvidia-curand-cu12==10.3.2.106
-nvidia-cusolver-cu12==11.4.5.107
-nvidia-cusparse-cu12==12.1.0.106
-nvidia-nccl-cu12==2.18.1
-nvidia-nvjitlink-cu12==12.3.101
-nvidia-nvtx-cu12==12.1.105
 oauthlib==3.2.2
 openai==1.6.1
 overrides==7.7.0
@@ -151,7 +135,6 @@ parso==0.8.4
 pathspec==0.12.1
 patsy==0.5.6
 pexpect==4.9.0
-pillow==10.4.0
 pkginfo==1.9.6
 platformdirs==4.1.0
 pluggy==1.3.0
@@ -199,7 +182,6 @@ ruamel.yaml==0.18.5
 ruamel.yaml.clib==0.2.8
 ruff==0.4.5
 scikit-learn==1.5.1
-scipy==1.11.4
 SecretStorage==3.3.3
 semver==3.0.2
 Send2Trash==1.8.3
@@ -224,7 +206,6 @@ sphinxcontrib-serializinghtml==1.1.9
 SQLAlchemy==2.0.24
 stack-data==0.6.3
 statsmodels==0.14.2
-sympy==1.12
 tables==3.9.2
 tabulate==0.9.0
 tenacity==8.2.3
@@ -235,14 +216,11 @@ tiktoken==0.7.0
 tinycss2==1.3.0
 toml-sort==0.23.1
 tomlkit==0.12.3
-torch==2.1.2
-torch_geometric==2.5.3
 tornado==6.4
 tqdm==4.66.1
 traitlets==5.14.3
 tree-sitter==0.22.3
 tree-sitter-python==0.21.0
-triton==2.1.0
 twine==4.0.2
 typer==0.9.0
 types-psutil==6.0.0.20240621
@@ -250,7 +228,6 @@ types-python-dateutil==2.9.0.20240316
 types-PyYAML==6.0.12.20240724
 types-tqdm==4.66.0.20240417
 typing-inspect==0.9.0
-typing_extensions==4.9.0
 tzdata==2023.4
 uri-template==1.3.0
 urllib3==2.1.0

docs/installation_and_configuration.rst

@@ -138,8 +138,6 @@ Configuration List
 +------------------------------+--------------------------------------------------+-------------------------+
 | prompt_cache_path            | Path to prompt cache                             | ./prompt_cache.db       |
 +------------------------------+--------------------------------------------------+-------------------------+
-| session_cache_folder_location| Path to session cache                            | ./session_cache_folder  |
-+------------------------------+--------------------------------------------------+-------------------------+
 | max_past_message_include     | Maximum number of past messages to include       | 10                      |
 +------------------------------+--------------------------------------------------+-------------------------+
 

docs/scens/data_agent_fin.rst

@@ -133,6 +133,6 @@ The following environment variables can be set in the `.env` file to customize t
 
 .. autopydantic_settings:: rdagent.components.coder.factor_coder.config.FactorImplementSettings
     :settings-show-field-summary: False
-    :members: coder_use_cache, data_folder, data_folder_debug, cache_location, enable_execution_cache, file_based_execution_timeout, select_method, select_threshold, max_loop, knowledge_base_path, new_knowledge_base_path
+    :members: coder_use_cache, data_folder, data_folder_debug, file_based_execution_timeout, select_method, select_threshold, max_loop, knowledge_base_path, new_knowledge_base_path
     :exclude-members: Config, fail_task_trial_limit, v1_query_former_trace_limit, v1_query_similar_success_limit, v2_query_component_limit, v2_query_error_limit, v2_query_former_trace_limit, v2_error_summary, v2_knowledge_sampler
     :no-index:

docs/scens/data_copilot_fin.rst

@@ -159,6 +159,6 @@ The following environment variables can be set in the `.env` file to customize t
 
 .. autopydantic_settings:: rdagent.components.coder.factor_coder.config.FactorImplementSettings
     :settings-show-field-summary: False
-    :members: coder_use_cache, data_folder, data_folder_debug, cache_location, enable_execution_cache, file_based_execution_timeout, select_method, select_threshold, max_loop, knowledge_base_path, new_knowledge_base_path
+    :members: coder_use_cache, data_folder, data_folder_debug, file_based_execution_timeout, select_method, select_threshold, max_loop, knowledge_base_path, new_knowledge_base_path
     :exclude-members: Config, python_bin, fail_task_trial_limit, v1_query_former_trace_limit, v1_query_similar_success_limit, v2_query_component_limit, v2_query_error_limit, v2_query_former_trace_limit, v2_error_summary, v2_knowledge_sampler
     :no-index:

docs/scens/model_agent_med.rst

@@ -123,6 +123,6 @@ You can try our demo by running the following command:
 
 The following environment variables can be set in the `.env` file to customize the application's behavior:
 
-.. autopydantic_settings:: rdagent.app.data_mining.conf.PropSetting
+.. autopydantic_settings:: rdagent.app.data_mining.conf.MedBasePropSetting
     :settings-show-field-summary: False
     :exclude-members: Config

rdagent/app/benchmark/factor/analysis.py

@@ -2,7 +2,9 @@ import json
 import pickle
 from pathlib import Path
 
+import fire
 import matplotlib.pyplot as plt
+import numpy as np
 import pandas as pd
 import seaborn as sns
 
@@ -42,7 +44,24 @@ class BenchmarkAnalyzer:
             final_res[experiment] = processed_data.iloc[-1, :]
         return final_res
 
-    def reformat_succ_rate(self, display_df):
+    def reformat_index(self, display_df):
+        """
+        reform the results from
+
+        .. code-block:: python
+
+                              success rate
+            High_Beta_Factor           0.2
+
+        to
+
+        .. code-block:: python
+
+                                                    success rate
+            Category Difficulty Factor
+            量价       Hard       High_Beta_Factor           0.2
+
+        """
         new_idx = []
         display_df = display_df[display_df.index.isin(self.index_map.keys())]
         for idx in display_df.index:
@@ -76,11 +95,9 @@ class BenchmarkAnalyzer:
     def analyze_data(self, sum_df):
         index = [
             "FactorSingleColumnEvaluator",
-            "FactorOutputFormatEvaluator",
             "FactorRowCountEvaluator",
             "FactorIndexEvaluator",
-            "FactorMissingValuesEvaluator",
-            "FactorEqualValueCountEvaluator",
+            "FactorEqualValueRatioEvaluator",
             "FactorCorrelationEvaluator",
             "run factor error",
         ]
@@ -91,33 +108,35 @@ class BenchmarkAnalyzer:
         succ_rate = ~run_error
         succ_rate = succ_rate.mean(axis=0).to_frame("success rate")
 
-        succ_rate_f = self.reformat_succ_rate(succ_rate)
-        succ_rate_f
+        succ_rate_f = self.reformat_index(succ_rate)
 
-        sum_df_clean["FactorRowCountEvaluator"]
+        # if it rasis Error when running the evaluator, we will get NaN
+        # Running failures are reguarded to zero score.
+        format_issue = sum_df_clean[["FactorRowCountEvaluator", "FactorIndexEvaluator"]].apply(
+            lambda x: np.mean(x.fillna(0.0)), axis=1
+        )
+        format_succ_rate = format_issue.unstack().T.mean(axis=0).to_frame("success rate")
+        format_succ_rate_f = self.reformat_index(format_succ_rate)
 
-        format_issue = sum_df_clean["FactorRowCountEvaluator"] & sum_df_clean["FactorIndexEvaluator"]
-        eval_series = format_issue.unstack()
-        succ_rate = eval_series.T.fillna(False).astype(bool)  # false indicate failure
-        format_succ_rate = succ_rate.mean(axis=0).to_frame("success rate")
-        format_succ_rate_f = self.reformat_succ_rate(format_succ_rate)
-
-        corr = sum_df_clean["FactorCorrelationEvaluator"] * format_issue
+        corr = sum_df_clean["FactorCorrelationEvaluator"].fillna(0.0)
         corr = corr.unstack().T.mean(axis=0).to_frame("corr(only success)")
-        corr_res = self.reformat_succ_rate(corr)
-        corr_max = sum_df_clean["FactorCorrelationEvaluator"] * format_issue
+        corr_res = self.reformat_index(corr)
+        corr_max = sum_df_clean["FactorCorrelationEvaluator"]
 
         corr_max = corr_max.unstack().T.max(axis=0).to_frame("corr(only success)")
-        corr_max_res = self.reformat_succ_rate(corr_max)
+        corr_max_res = self.reformat_index(corr_max)
 
-        value_max = sum_df_clean["FactorMissingValuesEvaluator"] * format_issue
+        value_max = sum_df_clean["FactorEqualValueRatioEvaluator"]
         value_max = value_max.unstack().T.max(axis=0).to_frame("max_value")
-        value_max_res = self.reformat_succ_rate(value_max)
+        value_max_res = self.reformat_index(value_max)
 
         value_avg = (
-            (sum_df_clean["FactorMissingValuesEvaluator"] * format_issue).unstack().T.mean(axis=0).to_frame("avg_value")
+            (sum_df_clean["FactorEqualValueRatioEvaluator"] * format_issue)
+            .unstack()
+            .T.mean(axis=0)
+            .to_frame("avg_value")
         )
-        value_avg_res = self.reformat_succ_rate(value_avg)
+        value_avg_res = self.reformat_index(value_avg)
 
         result_all = pd.concat(
             {
@@ -159,20 +178,24 @@ class Plotter:
         plt.rc("figure", titlesize=font_size)
 
     @staticmethod
-    def plot_data(data, file_name):
+    def plot_data(data, file_name, title):
         plt.figure(figsize=(10, 6))
         sns.barplot(x="index", y="b", hue="a", data=data)
         plt.xlabel("Method")
         plt.ylabel("Value")
-        plt.title("Comparison of Different Methods")
+        plt.title(title)
         plt.savefig(file_name)
 
 
-if __name__ == "__main__":
+def main(
+    path="git_ignore_folder/eval_results/res_promptV220240724-060037.pkl",
+    round=1,
+    title="Comparison of Different Methods",
+):
     settings = BenchmarkSettings()
     benchmark = BenchmarkAnalyzer(settings)
     results = {
-        "1 round experiment": "git_ignore_folder/eval_results/res_promptV220240724-060037.pkl",
+        f"{round} round experiment": path,
     }
     final_results = benchmark.process_results(results)
     final_results_df = pd.DataFrame(final_results)
@@ -180,4 +203,8 @@ if __name__ == "__main__":
     Plotter.change_fs(20)
     plot_data = final_results_df.drop(["max. accuracy", "avg. accuracy"], axis=0).T
     plot_data = plot_data.reset_index().melt("index", var_name="a", value_name="b")
-    Plotter.plot_data(plot_data, "rdagent/app/quant_factor_benchmark/comparison_plot.png")
+    Plotter.plot_data(plot_data, "./comparison_plot.png", title)
+
+
+if __name__ == "__main__":
+    fire.Fire(main)

rdagent/app/cli.py

@@ -20,6 +20,7 @@ from rdagent.app.data_mining.model import main as med_model
 from rdagent.app.general_model.general_model import (
     extract_models_and_implement as general_model,
 )
+from rdagent.app.kaggle.loop import main as kaggle_main
 from rdagent.app.qlib_rd_loop.factor import main as fin_factor
 from rdagent.app.qlib_rd_loop.factor_from_report import main as fin_factor_report
 from rdagent.app.qlib_rd_loop.model import main as fin_model
@@ -51,5 +52,6 @@ def app():
             "general_model": general_model,
             "ui": ui,
             "collect_info": collect_info,
+            "kaggle": kaggle_main,
         }
     )

rdagent/app/data_mining/conf.py

@@ -5,7 +5,7 @@ from pydantic_settings import BaseSettings
 from rdagent.components.workflow.conf import BasePropSetting
 
 
-class PropSetting(BasePropSetting):
+class MedBasePropSetting(BasePropSetting):
     class Config:
         env_prefix = "DM_"
         """Use `DM_` as prefix for environment variables"""
@@ -46,4 +46,4 @@ class PropSetting(BasePropSetting):
     """Physionet account password"""
 
 
-PROP_SETTING = PropSetting()
+MED_PROP_SETTING = MedBasePropSetting()

rdagent/app/data_mining/model.py

@@ -1,6 +1,6 @@
 import fire
 
-from rdagent.app.data_mining.conf import PROP_SETTING
+from rdagent.app.data_mining.conf import MED_PROP_SETTING
 from rdagent.components.workflow.rd_loop import RDLoop
 from rdagent.core.exception import ModelEmptyError
 
@@ -21,7 +21,7 @@ def main(path=None, step_n=None):
 
     """
     if path is None:
-        model_loop = ModelRDLoop(PROP_SETTING)
+        model_loop = ModelRDLoop(MED_PROP_SETTING)
     else:
         model_loop = ModelRDLoop.load(path)
     model_loop.run(step_n=step_n)

rdagent/app/kaggle/conf.py

@@ -5,38 +5,68 @@ from pydantic_settings import BaseSettings
 from rdagent.components.workflow.conf import BasePropSetting
 
 
-class PropSetting(BasePropSetting):
+class KaggleBasePropSetting(BasePropSetting):
     class Config:
         env_prefix = "KG_"
         """Use `KG_` as prefix for environment variables"""
         protected_namespaces = ()
-        """Add 'model_' to the protected namespaces"""
+        """Do not allow overriding of these namespaces"""
 
     # 1) overriding the default
-    scen: str = "rdagent.scenarios.kaggle.experiment.model_experiment.KGModelScenario"
+    scen: str = "rdagent.scenarios.kaggle.experiment.scenario.KGScenario"
     """Scenario class for data mining model"""
 
-    hypothesis_gen: str = "rdagent.scenarios.kaggle.proposal.model_proposal.KGModelHypothesisGen"
+    knowledge_base: str = ""  # TODO enable this line to use the knowledge base
+    # knowledge_base: str = "rdagent.scenarios.kaggle.knowledge_management.graph.KGKnowledgeGraph"
+    """Knowledge base class"""
+
+    knowledge_base_path: str = "kg_graph.pkl"
+    """Knowledge base path"""
+
+    hypothesis_gen: str = "rdagent.scenarios.kaggle.proposal.proposal.KGHypothesisGen"
     """Hypothesis generation class"""
 
-    hypothesis2experiment: str = "rdagent.scenarios.kaggle.proposal.model_proposal.KGModelHypothesis2Experiment"
+    hypothesis2experiment: str = "rdagent.scenarios.kaggle.proposal.proposal.KGHypothesis2Experiment"
     """Hypothesis to experiment class"""
 
-    coder: str = "rdagent.scenarios.kaggle.developer.model_coder.KGModelCoSTEER"
-    """Coder class"""
+    feature_coder: str = "rdagent.scenarios.kaggle.developer.coder.KGFactorCoSTEER"
+    """Feature Coder class"""
 
-    runner: str = "rdagent.scenarios.kaggle.developer.model_runner.KGModelRunner"
-    """Runner class"""
+    model_feature_selection_coder: str = "rdagent.scenarios.kaggle.developer.coder.KGModelFeatureSelectionCoder"
+    """Model Feature Selection Coder class"""
 
-    summarizer: str = "rdagent.scenarios.kaggle.developer.feedback.KGModelHypothesisExperiment2Feedback"
+    model_coder: str = "rdagent.scenarios.kaggle.developer.coder.KGModelCoSTEER"
+    """Model Coder class"""
+
+    feature_runner: str = "rdagent.scenarios.kaggle.developer.runner.KGFactorRunner"
+    """Feature Runner class"""
+
+    model_runner: str = "rdagent.scenarios.kaggle.developer.runner.KGModelRunner"
+    """Model Runner class"""
+
+    summarizer: str = "rdagent.scenarios.kaggle.developer.feedback.KGHypothesisExperiment2Feedback"
     """Summarizer class"""
 
     evolving_n: int = 10
     """Number of evolutions"""
 
-    evolving_n: int = 10
-
     competition: str = ""
 
+    local_data_path: str = "/data/userdata/share/kaggle"
 
-PROP_SETTING = PropSetting()
+    domain_knowledge_path: str = "/data/userdata/share/kaggle/domain_knowledge"
+
+    rag_path: str = "git_ignore_folder/rag"
+
+    if_action_choosing_based_on_UCB: bool = False
+
+    if_using_graph_rag: bool = False
+
+    if_using_vector_rag: bool = False
+
+    auto_submit: bool = True
+
+    mini_case: bool = False
+
+
+KAGGLE_IMPLEMENT_SETTING = KaggleBasePropSetting()

rdagent/app/kaggle/loop.py

@@ -0,0 +1,142 @@
+import subprocess
+from collections import defaultdict
+from typing import Any
+
+import fire
+
+from rdagent.app.kaggle.conf import KAGGLE_IMPLEMENT_SETTING
+from rdagent.components.workflow.conf import BasePropSetting
+from rdagent.components.workflow.rd_loop import RDLoop
+from rdagent.core.developer import Developer
+from rdagent.core.exception import FactorEmptyError, ModelEmptyError
+from rdagent.core.proposal import (
+    Hypothesis2Experiment,
+    HypothesisExperiment2Feedback,
+    HypothesisGen,
+    Trace,
+)
+from rdagent.core.scenario import Scenario
+from rdagent.core.utils import import_class
+from rdagent.log import rdagent_logger as logger
+from rdagent.log.time import measure_time
+from rdagent.scenarios.kaggle.experiment.scenario import (
+    KG_ACTION_FEATURE_ENGINEERING,
+    KG_ACTION_FEATURE_PROCESSING,
+    KG_ACTION_MODEL_FEATURE_SELECTION,
+)
+from rdagent.scenarios.kaggle.experiment.utils import python_files_to_notebook
+from rdagent.scenarios.kaggle.kaggle_crawler import download_data
+from rdagent.scenarios.kaggle.proposal.proposal import KGTrace
+
+
+class KaggleRDLoop(RDLoop):
+    @measure_time
+    def __init__(self, PROP_SETTING: BasePropSetting):
+        with logger.tag("init"):
+            scen: Scenario = import_class(PROP_SETTING.scen)(PROP_SETTING.competition)
+            logger.log_object(scen, tag="scenario")
+            knowledge_base = (
+                import_class(PROP_SETTING.knowledge_base)(PROP_SETTING.knowledge_base_path, scen)
+                if PROP_SETTING.knowledge_base != ""
+                else None
+            )
+            logger.log_object(knowledge_base, tag="knowledge_base")
+            self.hypothesis_gen: HypothesisGen = import_class(PROP_SETTING.hypothesis_gen)(scen)
+            logger.log_object(self.hypothesis_gen, tag="hypothesis generator")
+            self.hypothesis2experiment: Hypothesis2Experiment = import_class(PROP_SETTING.hypothesis2experiment)()
+            logger.log_object(self.hypothesis2experiment, tag="hypothesis2experiment")
+            self.feature_coder: Developer = import_class(PROP_SETTING.feature_coder)(scen)
+            logger.log_object(self.feature_coder, tag="feature coder")
+            self.model_feature_selection_coder: Developer = import_class(PROP_SETTING.model_feature_selection_coder)(
+                scen
+            )
+            logger.log_object(self.model_feature_selection_coder, tag="model feature selection coder")
+            self.model_coder: Developer = import_class(PROP_SETTING.model_coder)(scen)
+            logger.log_object(self.model_coder, tag="model coder")
+            self.feature_runner: Developer = import_class(PROP_SETTING.feature_runner)(scen)
+            logger.log_object(self.feature_runner, tag="feature runner")
+            self.model_runner: Developer = import_class(PROP_SETTING.model_runner)(scen)
+            logger.log_object(self.model_runner, tag="model runner")
+            self.summarizer: HypothesisExperiment2Feedback = import_class(PROP_SETTING.summarizer)(scen)
+            logger.log_object(self.summarizer, tag="summarizer")
+            self.trace = KGTrace(scen=scen, knowledge_base=knowledge_base)
+            super(RDLoop, self).__init__()
+
+    @measure_time
+    def coding(self, prev_out: dict[str, Any]):
+        with logger.tag("d"):  # develop
+            if prev_out["propose"].action in [KG_ACTION_FEATURE_ENGINEERING, KG_ACTION_FEATURE_PROCESSING]:
+                exp = self.feature_coder.develop(prev_out["exp_gen"])
+            elif prev_out["propose"].action == KG_ACTION_MODEL_FEATURE_SELECTION:
+                exp = self.model_feature_selection_coder.develop(prev_out["exp_gen"])
+            else:
+                exp = self.model_coder.develop(prev_out["exp_gen"])
+            logger.log_object(exp.sub_workspace_list, tag="coder result")
+        return exp
+
+    @measure_time
+    def running(self, prev_out: dict[str, Any]):
+        with logger.tag("ef"):  # evaluate and feedback
+            if prev_out["propose"].action in [KG_ACTION_FEATURE_ENGINEERING, KG_ACTION_FEATURE_PROCESSING]:
+                exp = self.feature_runner.develop(prev_out["coding"])
+            else:
+                exp = self.model_runner.develop(prev_out["coding"])
+            logger.log_object(exp, tag="runner result")
+            if KAGGLE_IMPLEMENT_SETTING.competition in [
+                "optiver-realized-volatility-prediction",
+                "covid19-global-forecasting-week-1",
+            ]:
+                try:
+                    python_files_to_notebook(
+                        KAGGLE_IMPLEMENT_SETTING.competition, exp.experiment_workspace.workspace_path
+                    )
+                except Exception as e:
+                    logger.error(f"Merge python files to one file failed: {e}")
+            if KAGGLE_IMPLEMENT_SETTING.auto_submit:
+                csv_path = exp.experiment_workspace.workspace_path / "submission.csv"
+                try:
+                    subprocess.run(
+                        [
+                            "kaggle",
+                            "competitions",
+                            "submit",
+                            "-f",
+                            str(csv_path.absolute()),
+                            "-m",
+                            str(csv_path.parent.absolute()),
+                            KAGGLE_IMPLEMENT_SETTING.competition,
+                        ],
+                        check=True,
+                    )
+                except subprocess.CalledProcessError as e:
+                    logger.error(f"Auto submission failed: \n{e}")
+                except Exception as e:
+                    logger.error(f"Other exception when use kaggle api:\n{e}")
+
+        return exp
+
+    skip_loop_error = (ModelEmptyError, FactorEmptyError)
+
+
+def main(path=None, step_n=None, competition=None):
+    """
+    Auto R&D Evolving loop for models in a kaggle{} scenario.
+    You can continue running session by
+    .. code-block:: bash
+        dotenv run -- python rdagent/app/kaggle/loop.py [--competition titanic] $LOG_PATH/__session__/1/0_propose  --step_n 1   # `step_n` is a optional parameter
+        rdagent kaggle --competition playground-series-s4e8  # You are encouraged to use this one.
+    """
+    if competition:
+        KAGGLE_IMPLEMENT_SETTING.competition = competition
+        download_data(competition=competition, local_path=KAGGLE_IMPLEMENT_SETTING.local_data_path)
+    else:
+        logger.error("Please specify competition name.")
+    if path is None:
+        kaggle_loop = KaggleRDLoop(KAGGLE_IMPLEMENT_SETTING)
+    else:
+        kaggle_loop = KaggleRDLoop.load(path)
+    kaggle_loop.run(step_n=step_n)
+
+
+if __name__ == "__main__":
+    fire.Fire(main)

rdagent/app/kaggle/model.py

@@ -1,65 +0,0 @@
-from collections import defaultdict
-
-import fire
-
-from rdagent.app.kaggle.conf import PROP_SETTING
-from rdagent.components.workflow.conf import BasePropSetting
-from rdagent.components.workflow.rd_loop import RDLoop
-from rdagent.core.exception import ModelEmptyError
-from rdagent.core.proposal import (
-    Hypothesis2Experiment,
-    HypothesisExperiment2Feedback,
-    HypothesisGen,
-    Trace,
-)
-from rdagent.core.utils import import_class
-from rdagent.log import rdagent_logger as logger
-
-
-class ModelRDLoop(RDLoop):
-    def __init__(self, PROP_SETTING: BasePropSetting):
-        with logger.tag("init"):
-            scen: Scenario = import_class(PROP_SETTING.scen)(PROP_SETTING.competition)
-            logger.log_object(scen, tag="scenario")
-
-            self.hypothesis_gen: HypothesisGen = import_class(PROP_SETTING.hypothesis_gen)(scen)
-            logger.log_object(self.hypothesis_gen, tag="hypothesis generator")
-
-            self.hypothesis2experiment: Hypothesis2Experiment = import_class(PROP_SETTING.hypothesis2experiment)()
-            logger.log_object(self.hypothesis2experiment, tag="hypothesis2experiment")
-
-            self.coder: Developer = import_class(PROP_SETTING.coder)(scen)
-            logger.log_object(self.coder, tag="coder")
-            self.runner: Developer = import_class(PROP_SETTING.runner)(scen)
-            logger.log_object(self.runner, tag="runner")
-
-            self.summarizer: HypothesisExperiment2Feedback = import_class(PROP_SETTING.summarizer)(scen)
-            logger.log_object(self.summarizer, tag="summarizer")
-            self.trace = Trace(scen=scen)
-            super(RDLoop, self).__init__()
-
-    skip_loop_error = (ModelEmptyError,)
-
-
-def main(path=None, step_n=None, competition=None):
-    """
-    Auto R&D Evolving loop for models in a kaggle{} scenario.
-
-    You can continue running session by
-
-    .. code-block:: python
-
-        dotenv run -- python rdagent/app/kaggle/model.py [--competition titanic] $LOG_PATH/__session__/1/0_propose  --step_n 1   # `step_n` is a optional paramter
-
-    """
-    if competition:
-        PROP_SETTING.competition = competition
-    if path is None:
-        model_loop = ModelRDLoop(PROP_SETTING)
-    else:
-        model_loop = ModelRDLoop.load(path)
-    model_loop.run(step_n=step_n)
-
-
-if __name__ == "__main__":
-    fire.Fire(main)

rdagent/app/qlib_rd_loop/factor.py

@@ -10,11 +10,13 @@ from rdagent.app.qlib_rd_loop.conf import FACTOR_PROP_SETTING
 from rdagent.components.workflow.rd_loop import RDLoop
 from rdagent.core.exception import FactorEmptyError
 from rdagent.log import rdagent_logger as logger
+from rdagent.log.time import measure_time
 
 
 class FactorRDLoop(RDLoop):
     skip_loop_error = (FactorEmptyError,)
 
+    @measure_time
     def running(self, prev_out: dict[str, Any]):
         with logger.tag("ef"):  # evaluate and feedback
             exp = self.runner.develop(prev_out["coding"])

rdagent/app/qlib_rd_loop/factor_from_report.py

@@ -11,11 +11,10 @@ from rdagent.components.document_reader.document_reader import (
     extract_first_page_screenshot_from_pdf,
     load_and_process_pdfs_by_langchain,
 )
-from rdagent.components.workflow.rd_loop import RDLoop
-from rdagent.core.exception import FactorEmptyError
 from rdagent.core.prompts import Prompts
 from rdagent.core.proposal import Hypothesis
 from rdagent.log import rdagent_logger as logger
+from rdagent.log.time import measure_time
 from rdagent.oai.llm_utils import APIBackend
 from rdagent.scenarios.qlib.experiment.factor_experiment import QlibFactorExperiment
 from rdagent.scenarios.qlib.factor_experiment_loader.pdf_loader import (
@@ -103,6 +102,7 @@ def extract_hypothesis_and_exp_from_reports(report_file_path: str) -> Tuple[Qlib
 
 
 class FactorReportLoop(FactorRDLoop, metaclass=LoopMeta):
+    @measure_time
     def __init__(self, report_folder: str = None):
         super().__init__(PROP_SETTING=FACTOR_FROM_REPORT_PROP_SETTING)
         if report_folder is None:
@@ -118,6 +118,7 @@ class FactorReportLoop(FactorRDLoop, metaclass=LoopMeta):
         self.current_loop_exp = None
         self.steps = ["propose_hypo_exp", "propose", "exp_gen", "coding", "running", "feedback"]
 
+    @measure_time
     def propose_hypo_exp(self, prev_out: dict[str, Any]):
         with logger.tag("r"):
             while True:
@@ -139,9 +140,11 @@ class FactorReportLoop(FactorRDLoop, metaclass=LoopMeta):
                 self.current_loop_exp = exp
                 return None
 
+    @measure_time
     def propose(self, prev_out: dict[str, Any]):
         return self.current_loop_hypothesis
 
+    @measure_time
     def exp_gen(self, prev_out: dict[str, Any]):
         return self.current_loop_exp
 

rdagent/components/benchmark/eval_method.py

@@ -8,11 +8,9 @@ from tqdm import tqdm
 from rdagent.components.coder.factor_coder.config import FACTOR_IMPLEMENT_SETTINGS
 from rdagent.components.coder.factor_coder.CoSTEER.evaluators import (
     FactorCorrelationEvaluator,
-    FactorEqualValueCountEvaluator,
+    FactorEqualValueRatioEvaluator,
     FactorEvaluator,
     FactorIndexEvaluator,
-    FactorMissingValuesEvaluator,
-    FactorOutputFormatEvaluator,
     FactorRowCountEvaluator,
     FactorSingleColumnEvaluator,
 )
@@ -20,7 +18,7 @@ from rdagent.components.coder.factor_coder.factor import FactorFBWorkspace
 from rdagent.core.conf import RD_AGENT_SETTINGS
 from rdagent.core.developer import Developer
 from rdagent.core.exception import CoderError
-from rdagent.core.experiment import Task, Workspace
+from rdagent.core.experiment import Experiment, Task, Workspace
 from rdagent.core.scenario import Scenario
 from rdagent.core.utils import multiprocessing_wrapper
 
@@ -33,11 +31,34 @@ EVAL_RES = Dict[
 class TestCase:
     def __init__(
         self,
-        target_task: list[Task] = [],
-        ground_truth: list[Workspace] = [],
+        target_task: Task,
+        ground_truth: Workspace,
     ):
-        self.ground_truth = ground_truth
         self.target_task = target_task
+        self.ground_truth = ground_truth
+
+
+class TestCases:
+    def __init__(self, test_case_l: list[TestCase] = []):
+        # self.test_case_l = [TestCase(task, gt) for task, gt in zip(target_task, ground_truth)]
+        self.test_case_l = test_case_l
+
+    def __getitem__(self, item):
+        return self.test_case_l[item]
+
+    def __len__(self):
+        return len(self.test_case_l)
+
+    def get_exp(self):
+        return Experiment([case.target_task for case in self.test_case_l])
+
+    @property
+    def target_task(self):
+        return [case.target_task for case in self.test_case_l]
+
+    @property
+    def ground_truth(self):
+        return [case.ground_truth for case in self.test_case_l]
 
 
 class BaseEval:
@@ -48,13 +69,13 @@ class BaseEval:
     def __init__(
         self,
         evaluator_l: List[FactorEvaluator],
-        test_cases: List[TestCase],
+        test_cases: TestCases,
         generate_method: Developer,
         catch_eval_except: bool = True,
     ):
         """Parameters
         ----------
-        test_cases : List[TestCase]
+        test_cases : TestCases
             cases to be evaluated, ground truth are included in the test cases.
         evaluator_l : List[FactorEvaluator]
             A list of evaluators to evaluate the generated code.
@@ -102,6 +123,7 @@ class BaseEval:
         eval_res = []
         for ev in self.evaluator_l:
             try:
+                case_gen.raise_exception = True
                 eval_res.append((ev, ev.evaluate(implementation=case_gen, gt_implementation=case_gt)))
                 # if the corr ev is successfully evaluated and achieve the best performance, then break
             except CoderError as e:
@@ -118,7 +140,7 @@ class BaseEval:
 class FactorImplementEval(BaseEval):
     def __init__(
         self,
-        test_cases: TestCase,
+        test_cases: TestCases,
         method: Developer,
         *args,
         scen: Scenario,
@@ -127,26 +149,22 @@ class FactorImplementEval(BaseEval):
     ):
         online_evaluator_l = [
             FactorSingleColumnEvaluator(scen),
-            FactorOutputFormatEvaluator(scen),
             FactorRowCountEvaluator(scen),
             FactorIndexEvaluator(scen),
-            FactorMissingValuesEvaluator(scen),
-            FactorEqualValueCountEvaluator(scen),
+            FactorEqualValueRatioEvaluator(scen),
             FactorCorrelationEvaluator(hard_check=False, scen=scen),
         ]
         super().__init__(online_evaluator_l, test_cases, method, *args, **kwargs)
         self.test_round = test_round
 
-    def eval(self):
+    def develop(self):
         gen_factor_l_all_rounds = []
-        test_cases_all_rounds = []
-        res = defaultdict(list)
         for _ in tqdm(range(self.test_round), desc="Rounds of Eval"):
             print("\n========================================================")
             print(f"Eval {_}-th times...")
             print("========================================================\n")
             try:
-                gen_factor_l = self.generate_method.develop(self.test_cases.target_task)
+                gen_factor_l = self.generate_method.develop(self.test_cases.get_exp())
             except KeyboardInterrupt:
                 # TODO: Why still need to save result after KeyboardInterrupt?
                 print("Manually interrupted the evaluation. Saving existing results")
@@ -157,8 +175,14 @@ class FactorImplementEval(BaseEval):
                     "The number of cases to eval should be equal to the number of test cases.",
                 )
             gen_factor_l_all_rounds.extend(gen_factor_l.sub_workspace_list)
-            test_cases_all_rounds.extend(self.test_cases.ground_truth)
 
+        return gen_factor_l_all_rounds
+
+    def eval(self, gen_factor_l_all_rounds):
+        test_cases_all_rounds = []
+        res = defaultdict(list)
+        for _ in range(self.test_round):
+            test_cases_all_rounds.extend(self.test_cases.ground_truth)
         eval_res_list = multiprocessing_wrapper(
             [
                 (self.eval_case, (gt_case, gen_factor))

rdagent/components/coder/factor_coder/CoSTEER/__init__.py

@@ -88,7 +88,7 @@ class FactorCoSTEER(Developer[FactorExperiment]):
         self.rag = FactorGraphRAGStrategy(factor_knowledge_base)
 
         # init intermediate items
-        factor_experiment = FactorEvolvingItem(sub_tasks=exp.sub_tasks)
+        factor_experiment = FactorEvolvingItem.from_experiment(exp)
 
         self.evolve_agent = FactorRAGEvoAgent(
             max_loop=self.max_loop,

rdagent/components/coder/factor_coder/CoSTEER/evaluators.py

@@ -19,6 +19,7 @@ from rdagent.core.experiment import Task, Workspace
 from rdagent.core.prompts import Prompts
 from rdagent.core.utils import multiprocessing_wrapper
 from rdagent.log import rdagent_logger as logger
+from rdagent.oai.llm_conf import LLM_SETTINGS
 from rdagent.oai.llm_utils import APIBackend
 
 evaluate_prompts = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")
@@ -89,7 +90,13 @@ class FactorCodeEvaluator(FactorEvaluator):
         system_prompt = (
             Environment(undefined=StrictUndefined)
             .from_string(evaluate_prompts["evaluator_code_feedback_v1_system"])
-            .render(scenario=self.scen.get_scenario_all_desc() if self.scen is not None else "No scenario description.")
+            .render(
+                scenario=(
+                    self.scen.get_scenario_all_desc(target_task)
+                    if self.scen is not None
+                    else "No scenario description."
+                )
+            )
         )
 
         execution_feedback_to_render = execution_feedback
@@ -112,7 +119,7 @@ class FactorCodeEvaluator(FactorEvaluator):
                     user_prompt=user_prompt,
                     system_prompt=system_prompt,
                 )
-                > RD_AGENT_SETTINGS.chat_token_limit
+                > LLM_SETTINGS.chat_token_limit
             ):
                 execution_feedback_to_render = execution_feedback_to_render[len(execution_feedback_to_render) // 2 :]
             else:
@@ -126,6 +133,28 @@ class FactorCodeEvaluator(FactorEvaluator):
         return critic_response, None
 
 
+class FactorInfEvaluator(FactorEvaluator):
+    def evaluate(
+        self,
+        implementation: Workspace,
+        gt_implementation: Workspace,
+    ) -> Tuple[str, object]:
+        _, gen_df = self._get_df(gt_implementation, implementation)
+        if gen_df is None:
+            return (
+                "The source dataframe is None. Please check the implementation.",
+                False,
+            )
+        INF_count = gen_df.isin([float("inf"), -float("inf")]).sum().sum()
+        if INF_count == 0:
+            return "The source dataframe does not have any infinite values.", True
+        else:
+            return (
+                f"The source dataframe has {INF_count} infinite values. Please check the implementation.",
+                False,
+            )
+
+
 class FactorSingleColumnEvaluator(FactorEvaluator):
     def evaluate(
         self,
@@ -161,13 +190,19 @@ class FactorOutputFormatEvaluator(FactorEvaluator):
             )
         buffer = io.StringIO()
         gen_df.info(buf=buffer)
-        gen_df_info_str = buffer.getvalue()
+        gen_df_info_str = f"The use is currently working on a feature related task.\nThe output dataframe info is:\n{buffer.getvalue()}"
         system_prompt = (
             Environment(undefined=StrictUndefined)
             .from_string(
                 evaluate_prompts["evaluator_output_format_system"],
             )
-            .render(scenario=self.scen.get_scenario_all_desc() if self.scen is not None else "No scenario description.")
+            .render(
+                scenario=(
+                    self.scen.get_scenario_all_desc(implementation.target_task)
+                    if self.scen is not None
+                    else "No scenario description."
+                )
+            )
         )
 
         # TODO: with retry_context(retry_n=3, except_list=[KeyError]):
@@ -177,18 +212,12 @@ class FactorOutputFormatEvaluator(FactorEvaluator):
 
         while attempts < max_attempts:
             try:
-                resp = APIBackend().build_messages_and_create_chat_completion(
+                api = APIBackend() if attempts == 0 else APIBackend(use_chat_cache=False)
+                resp = api.build_messages_and_create_chat_completion(
                     user_prompt=gen_df_info_str, system_prompt=system_prompt, json_mode=True
                 )
                 resp_dict = json.loads(resp)
-
-                if isinstance(resp_dict["output_format_decision"], str) and resp_dict[
-                    "output_format_decision"
-                ].lower() in (
-                    "true",
-                    "false",
-                ):
-                    resp_dict["output_format_decision"] = bool(resp_dict["output_format_decision"])
+                resp_dict["output_format_decision"] = str(resp_dict["output_format_decision"]).lower() in ["true", "1"]
 
                 return (
                     resp_dict["output_format_feedback"],
@@ -225,11 +254,11 @@ class FactorDatetimeDailyEvaluator(FactorEvaluator):
             pd.to_datetime(gen_df.index.get_level_values("datetime"))
         except Exception:
             return (
-                "The source dataframe has a datetime index but it is not in the correct format (maybe a regular string or other objects). Please check the implementation.",
+                f"The source dataframe has a datetime index but it is not in the correct format (maybe a regular string or other objects). Please check the implementation.\n The head of the output dataframe is: \n{gen_df.head()}",
                 False,
             )
 
-        time_diff = gen_df.index.get_level_values("datetime").to_series().diff().dropna().unique()
+        time_diff = pd.to_datetime(gen_df.index.get_level_values("datetime")).to_series().diff().dropna().unique()
         if pd.Timedelta(minutes=1) in time_diff:
             return (
                 "The generated dataframe is not daily. The implementation is definitely wrong. Please check the implementation.",
@@ -250,13 +279,16 @@ class FactorRowCountEvaluator(FactorEvaluator):
                 "The source dataframe is None. Please check the implementation.",
                 False,
             )
-        if gen_df.shape[0] == gt_df.shape[0]:
-            return "Both dataframes have the same rows count.", True
-        else:
-            return (
-                f"The source dataframe and the ground truth dataframe have different rows count. The source dataframe has {gen_df.shape[0]} rows, while the ground truth dataframe has {gt_df.shape[0]} rows. Please check the implementation.",
-                False,
-            )
+        ratio = min(len(gen_df), len(gt_df)) / max(len(gen_df), len(gt_df))
+        return (
+            (
+                f"The ratio of rows count in the source dataframe to the ground truth dataframe is {ratio:.2f}. "
+                + "Please verify the implementation. "
+                if ratio <= 0.99
+                else ""
+            ),
+            ratio,
+        )
 
 
 class FactorIndexEvaluator(FactorEvaluator):
@@ -271,13 +303,17 @@ class FactorIndexEvaluator(FactorEvaluator):
                 "The source dataframe is None. Please check the implementation.",
                 False,
             )
-        if gen_df.index.equals(gt_df.index):
-            return "Both dataframes have the same index.", True
-        else:
-            return (
-                "The source dataframe and the ground truth dataframe have different index. Please check the implementation.",
-                False,
-            )
+        gen_index_set, gt_index_set = set(gen_df.index), set(gt_df.index)
+        similarity = len(gen_index_set.intersection(gt_index_set)) / len(gen_index_set.union(gt_index_set))
+        return (
+            (
+                f"The source dataframe and the ground truth dataframe have different index with a similarity of {similarity:.2%}. The similarity is calculated by the number of shared indices divided by the union indices. "
+                + "Please check the implementation."
+                if similarity <= 0.99
+                else ""
+            ),
+            similarity,
+        )
 
 
 class FactorMissingValuesEvaluator(FactorEvaluator):
@@ -301,7 +337,7 @@ class FactorMissingValuesEvaluator(FactorEvaluator):
             )
 
 
-class FactorEqualValueCountEvaluator(FactorEvaluator):
+class FactorEqualValueRatioEvaluator(FactorEvaluator):
     def evaluate(
         self,
         implementation: Workspace,
@@ -378,40 +414,51 @@ class FactorValueEvaluator(FactorEvaluator):
         self,
         implementation: Workspace,
         gt_implementation: Workspace,
+        version: int = 1,  # 1 for qlib factors and 2 for kaggle factors
         **kwargs,
     ) -> Tuple:
         conclusions = []
 
         # Initialize result variables
-        single_column_result = None
-        same_index_result = None
+        row_result = 0
+        index_result = 0
         output_format_result = None
         equal_value_ratio_result = 0
         high_correlation_result = False
+        row_result = None
 
-        # Check if both dataframe has only one columns
-        feedback_str, _ = FactorSingleColumnEvaluator(self.scen).evaluate(implementation, gt_implementation)
+        # Check if both dataframe has only one columns Mute this since factor task might generate more than one columns now
+        if version == 1:
+            feedback_str, _ = FactorSingleColumnEvaluator(self.scen).evaluate(implementation, gt_implementation)
+            conclusions.append(feedback_str)
+        elif version == 2:
+            input_shape = self.scen.input_shape
+            _, gen_df = self._get_df(gt_implementation, implementation)
+            if gen_df.shape[-1] > input_shape[-1]:
+                conclusions.append(
+                    "Output dataframe has more columns than input feature which is not acceptable in feature processing tasks. Please check the implementation to avoid generating too many columns. Consider this implementation as a failure."
+                )
+
+        feedback_str, inf_evaluate_res = FactorInfEvaluator(self.scen).evaluate(implementation, gt_implementation)
         conclusions.append(feedback_str)
 
         # Check if the index of the dataframe is ("datetime", "instrument")
         feedback_str, _ = FactorOutputFormatEvaluator(self.scen).evaluate(implementation, gt_implementation)
         conclusions.append(feedback_str)
-
-        feedback_str, daily_check_result = FactorDatetimeDailyEvaluator(self.scen).evaluate(
-            implementation, gt_implementation
-        )
-        conclusions.append(feedback_str)
-
-        # Check if both dataframe have the same rows count
-        if gt_implementation is not None:
-            feedback_str, single_column_result = FactorRowCountEvaluator(self.scen).evaluate(
+        if version == 1:
+            feedback_str, daily_check_result = FactorDatetimeDailyEvaluator(self.scen).evaluate(
                 implementation, gt_implementation
             )
             conclusions.append(feedback_str)
+        else:
+            daily_check_result = None
 
-            feedback_str, same_index_result = FactorIndexEvaluator(self.scen).evaluate(
-                implementation, gt_implementation
-            )
+        # Check dataframe format
+        if gt_implementation is not None:
+            feedback_str, row_result = FactorRowCountEvaluator(self.scen).evaluate(implementation, gt_implementation)
+            conclusions.append(feedback_str)
+
+            feedback_str, index_result = FactorIndexEvaluator(self.scen).evaluate(implementation, gt_implementation)
             conclusions.append(feedback_str)
 
             feedback_str, output_format_result = FactorMissingValuesEvaluator(self.scen).evaluate(
@@ -419,12 +466,12 @@ class FactorValueEvaluator(FactorEvaluator):
             )
             conclusions.append(feedback_str)
 
-            feedback_str, equal_value_ratio_result = FactorEqualValueCountEvaluator(self.scen).evaluate(
+            feedback_str, equal_value_ratio_result = FactorEqualValueRatioEvaluator(self.scen).evaluate(
                 implementation, gt_implementation
             )
             conclusions.append(feedback_str)
 
-            if same_index_result:
+            if index_result > 0.99:
                 feedback_str, high_correlation_result = FactorCorrelationEvaluator(
                     hard_check=True, scen=self.scen
                 ).evaluate(implementation, gt_implementation)
@@ -438,7 +485,13 @@ class FactorValueEvaluator(FactorEvaluator):
 
         if gt_implementation is not None and (equal_value_ratio_result > 0.99) or high_correlation_result:
             decision_from_value_check = True
-        elif single_column_result is False or output_format_result is False or daily_check_result is False:
+        elif (
+            row_result is not None
+            and row_result <= 0.99
+            or output_format_result is False
+            or daily_check_result is False
+            or inf_evaluate_res is False
+        ):
             decision_from_value_check = False
         else:
             decision_from_value_check = None
@@ -457,7 +510,13 @@ class FactorFinalDecisionEvaluator(Evaluator):
         system_prompt = (
             Environment(undefined=StrictUndefined)
             .from_string(evaluate_prompts["evaluator_final_decision_v1_system"])
-            .render(scenario=self.scen.get_scenario_all_desc() if self.scen is not None else "No scenario description.")
+            .render(
+                scenario=(
+                    self.scen.get_scenario_all_desc(target_task)
+                    if self.scen is not None
+                    else "No scenario description."
+                )
+            )
         )
         execution_feedback_to_render = execution_feedback
 
@@ -483,7 +542,7 @@ class FactorFinalDecisionEvaluator(Evaluator):
                     user_prompt=user_prompt,
                     system_prompt=system_prompt,
                 )
-                > RD_AGENT_SETTINGS.chat_token_limit
+                > LLM_SETTINGS.chat_token_limit
             ):
                 execution_feedback_to_render = execution_feedback_to_render[len(execution_feedback_to_render) // 2 :]
             else:
@@ -496,19 +555,19 @@ class FactorFinalDecisionEvaluator(Evaluator):
 
         while attempts < max_attempts:
             try:
+                api = APIBackend() if attempts == 0 else APIBackend(use_chat_cache=False)
                 final_evaluation_dict = json.loads(
-                    APIBackend().build_messages_and_create_chat_completion(
+                    api.build_messages_and_create_chat_completion(
                         user_prompt=user_prompt,
                         system_prompt=system_prompt,
                         json_mode=True,
+                        seed=attempts,  # in case of useless retrying when cache enabled.
                     ),
                 )
                 final_decision = final_evaluation_dict["final_decision"]
                 final_feedback = final_evaluation_dict["final_feedback"]
 
-                if isinstance(final_decision, str) and final_decision.lower() in ("true", "false"):
-                    final_decision = bool(final_decision)
-
+                final_decision = str(final_decision).lower() in ["true", "1"]
                 return final_decision, final_feedback
 
             except json.JSONDecodeError as e:
@@ -627,7 +686,9 @@ class FactorEvaluatorForCoder(FactorEvaluator):
                 (
                     factor_feedback.factor_value_feedback,
                     decision_from_value_check,
-                ) = self.value_evaluator.evaluate(implementation=implementation, gt_implementation=gt_implementation)
+                ) = self.value_evaluator.evaluate(
+                    implementation=implementation, gt_implementation=gt_implementation, version=target_task.version
+                )
 
             factor_feedback.final_decision_based_on_gt = gt_implementation is not None
 
@@ -637,8 +698,12 @@ class FactorEvaluatorForCoder(FactorEvaluator):
                 factor_feedback.final_decision = decision_from_value_check
                 factor_feedback.final_feedback = "Value evaluation passed, skip final decision evaluation."
             elif decision_from_value_check is not None and decision_from_value_check is False:
-                factor_feedback.code_feedback = (
-                    "Final decision is False because value evaluation gets a confident rejection to the result."
+                factor_feedback.code_feedback, _ = self.code_evaluator.evaluate(
+                    target_task=target_task,
+                    implementation=implementation,
+                    execution_feedback=factor_feedback.execution_feedback,
+                    factor_value_feedback=factor_feedback.factor_value_feedback,
+                    gt_implementation=gt_implementation,
                 )
                 factor_feedback.final_decision = decision_from_value_check
                 factor_feedback.final_feedback = "Value evaluation failed, skip final decision evaluation."
@@ -647,7 +712,7 @@ class FactorEvaluatorForCoder(FactorEvaluator):
                     target_task=target_task,
                     implementation=implementation,
                     execution_feedback=factor_feedback.execution_feedback,
-                    value_feedback=factor_feedback.factor_value_feedback,
+                    factor_value_feedback=factor_feedback.factor_value_feedback,
                     gt_implementation=gt_implementation,
                 )
                 (

rdagent/components/coder/factor_coder/CoSTEER/evolvable_subjects.py

@@ -28,3 +28,10 @@ class FactorEvolvingItem(FactorExperiment, EvolvableSubjects):
             )
         else:
             self.sub_gt_implementations = sub_gt_implementations
+
+    @classmethod
+    def from_experiment(cls, exp: FactorExperiment) -> "FactorExperiment":
+        ei = cls(sub_tasks=exp.sub_tasks)
+        ei.based_experiments = exp.based_experiments
+        ei.experiment_workspace = exp.experiment_workspace
+        return ei

rdagent/components/coder/factor_coder/CoSTEER/evolving_strategy.py

@@ -22,6 +22,7 @@ from rdagent.core.evolving_framework import EvolvingStrategy, QueriedKnowledge
 from rdagent.core.experiment import Workspace
 from rdagent.core.prompts import Prompts
 from rdagent.core.utils import multiprocessing_wrapper
+from rdagent.oai.llm_conf import LLM_SETTINGS
 from rdagent.oai.llm_utils import APIBackend
 
 if TYPE_CHECKING:
@@ -90,6 +91,7 @@ class MultiProcessEvolvingStrategy(EvolvingStrategy):
             ],
             n=RD_AGENT_SETTINGS.multi_proc_n,
         )
+        from rdagent.components.coder.factor_coder.factor import FactorFBWorkspace
 
         for index, target_index in enumerate(to_be_finished_task_index):
             if evo.sub_workspace_list[target_index] is None:
@@ -133,7 +135,7 @@ class FactorEvolvingStrategy(MultiProcessEvolvingStrategy):
                     implement_prompts["evolving_strategy_factor_implementation_v1_system"],
                 )
                 .render(
-                    scenario=self.scen.get_scenario_all_desc(),
+                    scenario=self.scen.get_scenario_all_desc(target_task),
                     queried_former_failed_knowledge=queried_former_failed_knowledge_to_render,
                 )
             )
@@ -159,7 +161,7 @@ class FactorEvolvingStrategy(MultiProcessEvolvingStrategy):
                     session.build_chat_completion_message_and_calculate_token(
                         user_prompt,
                     )
-                    < RD_AGENT_SETTINGS.chat_token_limit
+                    < LLM_SETTINGS.chat_token_limit
                 ):
                     break
                 elif len(queried_former_failed_knowledge_to_render) > 1:
@@ -215,18 +217,24 @@ class FactorEvolvingStrategyWithGraph(MultiProcessEvolvingStrategy):
             )  # A dict, {{error_type:[[error_imp_knowledge, success_imp_knowledge],...]},...}
 
             queried_former_failed_knowledge = (
-                queried_knowledge.former_traces[target_factor_task_information] if queried_knowledge is not None else []
+                queried_knowledge.former_traces[target_factor_task_information][0]
+                if queried_knowledge is not None
+                else []
             )
 
             queried_former_failed_knowledge_to_render = queried_former_failed_knowledge
 
+            latest_attempt_to_latest_successful_execution = queried_knowledge.former_traces[
+                target_factor_task_information
+            ][1]
+
             system_prompt = (
                 Environment(undefined=StrictUndefined)
                 .from_string(
                     implement_prompts["evolving_strategy_factor_implementation_v1_system"],
                 )
                 .render(
-                    scenario=self.scen.get_scenario_all_desc(),
+                    scenario=self.scen.get_scenario_all_desc(target_task),
                     queried_former_failed_knowledge=queried_former_failed_knowledge_to_render,
                 )
             )
@@ -250,7 +258,7 @@ class FactorEvolvingStrategyWithGraph(MultiProcessEvolvingStrategy):
                         Environment(undefined=StrictUndefined)
                         .from_string(implement_prompts["evolving_strategy_error_summary_v2_system"])
                         .render(
-                            scenario=self.scen.get_scenario_all_desc(),
+                            scenario=self.scen.get_scenario_all_desc(target_task),
                             factor_information_str=target_factor_task_information,
                             code_and_feedback=queried_former_failed_knowledge_to_render[
                                 -1
@@ -274,7 +282,7 @@ class FactorEvolvingStrategyWithGraph(MultiProcessEvolvingStrategy):
                         )
                         if (
                             session_summary.build_chat_completion_message_and_calculate_token(error_summary_user_prompt)
-                            < RD_AGENT_SETTINGS.chat_token_limit
+                            < LLM_SETTINGS.chat_token_limit
                         ):
                             break
                         elif len(queried_similar_error_knowledge_to_render) > 0:
@@ -295,6 +303,7 @@ class FactorEvolvingStrategyWithGraph(MultiProcessEvolvingStrategy):
                         queried_similar_error_knowledge=queried_similar_error_knowledge_to_render,
                         error_summary=error_summary,
                         error_summary_critics=error_summary_critics,
+                        latest_attempt_to_latest_successful_execution=latest_attempt_to_latest_successful_execution,
                     )
                     .strip("\n")
                 )
@@ -302,7 +311,7 @@ class FactorEvolvingStrategyWithGraph(MultiProcessEvolvingStrategy):
                     session.build_chat_completion_message_and_calculate_token(
                         user_prompt,
                     )
-                    < RD_AGENT_SETTINGS.chat_token_limit
+                    < LLM_SETTINGS.chat_token_limit
                 ):
                     break
                 elif len(queried_former_failed_knowledge_to_render) > 1:

rdagent/components/coder/factor_coder/CoSTEER/knowledge_management.py

@@ -21,9 +21,9 @@ from rdagent.components.knowledge_management.graph import (
 )
 from rdagent.core.evolving_framework import (
     EvolvableSubjects,
+    EvolvingKnowledgeBase,
     EvoStep,
     Knowledge,
-    KnowledgeBase,
     QueriedKnowledge,
     RAGStrategy,
 )
@@ -70,12 +70,13 @@ class FactorQueriedKnowledge(QueriedKnowledge):
         self.failed_task_info_set = failed_task_info_set
 
 
-class FactorKnowledgeBaseV1(KnowledgeBase):
-    def __init__(self) -> None:
+class FactorKnowledgeBaseV1(EvolvingKnowledgeBase):
+    def __init__(self, path: str | Path = None) -> None:
         self.implementation_trace: dict[str, FactorKnowledge] = dict()
         self.success_task_info_set: set[str] = set()
 
         self.task_to_embedding = dict()
+        super().__init__(path)
 
     def query(self) -> QueriedKnowledge | None:
         """
@@ -295,6 +296,7 @@ class FactorGraphRAGStrategy(RAGStrategy):
             evo,
             factor_implementation_queried_graph_knowledge,
             FACTOR_IMPLEMENT_SETTINGS.v2_query_former_trace_limit,
+            FACTOR_IMPLEMENT_SETTINGS.v2_add_fail_attempt_to_latest_successful_execution,
         )
         factor_implementation_queried_graph_knowledge = self.component_query(
             evo,
@@ -391,6 +393,7 @@ class FactorGraphRAGStrategy(RAGStrategy):
         evo: EvolvableSubjects,
         factor_implementation_queried_graph_knowledge: FactorQueriedGraphKnowledge,
         v2_query_former_trace_limit: int = 5,
+        v2_add_fail_attempt_to_latest_successful_execution: bool = False,
     ) -> Union[QueriedKnowledge, set]:
         """
         Query the former trace knowledge of the working trace, and find all the failed task information which tried more than fail_task_trial_limit times
@@ -428,11 +431,25 @@ class FactorGraphRAGStrategy(RAGStrategy):
                     else:
                         current_index += 1
 
-                factor_implementation_queried_graph_knowledge.former_traces[
-                    target_factor_task_information
-                ] = former_trace_knowledge[-v2_query_former_trace_limit:]
+                latest_attempt = None
+                if v2_add_fail_attempt_to_latest_successful_execution:
+                    # When the last successful execution is not the last one in the working trace, it means we have tried to correct it. We should tell the agent this fail trial to avoid endless loop in the future.
+                    if (
+                        len(former_trace_knowledge) > 0
+                        and len(self.knowledgebase.working_trace_knowledge[target_factor_task_information]) > 1
+                        and self.knowledgebase.working_trace_knowledge[target_factor_task_information].index(
+                            former_trace_knowledge[-1]
+                        )
+                        < len(self.knowledgebase.working_trace_knowledge[target_factor_task_information]) - 1
+                    ):
+                        latest_attempt = self.knowledgebase.working_trace_knowledge[target_factor_task_information][-1]
+
+                factor_implementation_queried_graph_knowledge.former_traces[target_factor_task_information] = (
+                    former_trace_knowledge[-v2_query_former_trace_limit:],
+                    latest_attempt,
+                )
             else:
-                factor_implementation_queried_graph_knowledge.former_traces[target_factor_task_information] = []
+                factor_implementation_queried_graph_knowledge.former_traces[target_factor_task_information] = ([], None)
 
         return factor_implementation_queried_graph_knowledge
 
@@ -564,7 +581,7 @@ class FactorGraphRAGStrategy(RAGStrategy):
                     if knowledge.feedback is not None and knowledge.feedback.final_decision_based_on_gt == False
                 ]
                 queried_from_gt_knowledge_count = max(
-                    min(v2_query_component_limit // 2, len(queried_from_gt_knowledge_list)),
+                    min((v2_query_component_limit // 2 + 1), len(queried_from_gt_knowledge_list)),
                     v2_query_component_limit - len(queried_without_gt_knowledge_list),
                 )
                 factor_implementation_queried_graph_knowledge.component_with_success_task[
@@ -606,7 +623,7 @@ class FactorGraphRAGStrategy(RAGStrategy):
                 ):
                     queried_last_trace = factor_implementation_queried_graph_knowledge.former_traces[
                         target_factor_task_information
-                    ][-1]
+                    ][0][-1]
                     target_index = self.knowledgebase.working_trace_knowledge[target_factor_task_information].index(
                         queried_last_trace,
                     )
@@ -711,12 +728,12 @@ class FactorGraphRAGStrategy(RAGStrategy):
         return factor_implementation_queried_graph_knowledge
 
 
-class FactorGraphKnowledgeBase(KnowledgeBase):
-    def __init__(self, init_component_list=None) -> None:
+class FactorGraphKnowledgeBase(EvolvingKnowledgeBase):
+    def __init__(self, init_component_list=None, path: str | Path = None) -> None:
         """
         Load knowledge, offer brief information of knowledge and common handle interfaces
         """
-        self.graph: UndirectedGraph = UndirectedGraph.load(Path.cwd() / "graph.pkl")
+        self.graph: UndirectedGraph = UndirectedGraph(Path.cwd() / "graph.pkl")
         logger.info(f"Knowledge Graph loaded, size={self.graph.size()}")
 
         if init_component_list:

rdagent/components/coder/factor_coder/CoSTEER/scheduler.py

@@ -7,10 +7,10 @@ from jinja2 import Environment, StrictUndefined
 from rdagent.components.coder.factor_coder.CoSTEER.evolvable_subjects import (
     FactorEvolvingItem,
 )
-from rdagent.core.conf import RD_AGENT_SETTINGS
 from rdagent.core.prompts import Prompts
 from rdagent.core.scenario import Scenario
 from rdagent.log import rdagent_logger as logger
+from rdagent.oai.llm_conf import LLM_SETTINGS
 from rdagent.oai.llm_utils import APIBackend
 
 scheduler_prompts = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")
@@ -40,7 +40,7 @@ def LLMSelect(
         # find corresponding former trace for each task
         target_factor_task_information = evo.sub_tasks[i].get_task_information()
         if target_factor_task_information in former_trace:
-            tasks.append((i, evo.sub_tasks[i], former_trace[target_factor_task_information]))
+            tasks.append((i, evo.sub_tasks[i], former_trace[target_factor_task_information][0]))
 
     system_prompt = (
         Environment(undefined=StrictUndefined)
@@ -68,7 +68,7 @@ def LLMSelect(
                 user_prompt=user_prompt,
                 system_prompt=system_prompt,
             )
-            < RD_AGENT_SETTINGS.chat_token_limit
+            < LLM_SETTINGS.chat_token_limit
         ):
             break
 

rdagent/components/coder/factor_coder/config.py

@@ -20,12 +20,6 @@ class FactorImplementSettings(BaseSettings):
     data_folder_debug: str = "git_ignore_folder/factor_implementation_source_data_debug"
     """Path to the folder containing partial financial data (for debugging)"""
 
-    cache_location: str = "git_ignore_folder/factor_implementation_execution_cache"
-    """Path to the cache location"""
-
-    enable_execution_cache: bool = True
-    """Indicates whether to enable the execution cache"""
-
     # TODO: the factor implement specific settings should not appear in this settings
     # Evolving should have a method specific settings
     # evolving related config
@@ -37,6 +31,7 @@ class FactorImplementSettings(BaseSettings):
     v2_query_component_limit: int = 1
     v2_query_error_limit: int = 1
     v2_query_former_trace_limit: int = 1
+    v2_add_fail_attempt_to_latest_successful_execution: bool = False
     v2_error_summary: bool = False
     v2_knowledge_sampler: float = 1.0
 

rdagent/components/coder/factor_coder/factor.py

@@ -9,9 +9,11 @@ from typing import Tuple, Union
 import pandas as pd
 from filelock import FileLock
 
+from rdagent.app.kaggle.conf import KAGGLE_IMPLEMENT_SETTING
 from rdagent.components.coder.factor_coder.config import FACTOR_IMPLEMENT_SETTINGS
 from rdagent.core.exception import CodeFormatError, CustomRuntimeError, NoOutputError
 from rdagent.core.experiment import Experiment, FBWorkspace, Task
+from rdagent.core.utils import cache_with_pickle
 from rdagent.log import rdagent_logger as logger
 from rdagent.oai.llm_utils import md5_hash
 
@@ -24,16 +26,21 @@ class FactorTask(Task):
         factor_name,
         factor_description,
         factor_formulation,
+        *args,
         variables: dict = {},
         resource: str = None,
         factor_implementation: bool = False,
+        **kwargs,
     ) -> None:
-        self.factor_name = factor_name
+        self.factor_name = (
+            factor_name  # TODO: remove it in the later version. Keep it only for pickle version compatibility
+        )
         self.factor_description = factor_description
         self.factor_formulation = factor_formulation
         self.variables = variables
         self.factor_resources = resource
         self.factor_implementation = factor_implementation
+        super().__init__(name=factor_name, *args, **kwargs)
 
     def get_task_information(self):
         return f"""factor_name: {self.factor_name}
@@ -65,7 +72,6 @@ class FactorFBWorkspace(FBWorkspace):
     """
 
     # TODO: (Xiao) think raising errors may get better information for processing
-    FB_FROM_CACHE = "The factor value has been executed and stored in the instance variable."
     FB_EXEC_SUCCESS = "Execution succeeded without error."
     FB_CODE_NOT_SET = "code is not set."
     FB_EXECUTION_SUCCEEDED = "Execution succeeded without error."
@@ -75,39 +81,38 @@ class FactorFBWorkspace(FBWorkspace):
     def __init__(
         self,
         *args,
-        executed_factor_value_dataframe=None,
-        raise_exception=False,
+        raise_exception: bool = False,
         **kwargs,
     ) -> None:
         super().__init__(*args, **kwargs)
-        self.executed_factor_value_dataframe = executed_factor_value_dataframe
         self.raise_exception = raise_exception
 
-    @staticmethod
-    def link_data_to_workspace(data_path: Path, workspace_path: Path):
-        data_path = Path(data_path)
-        workspace_path = Path(workspace_path)
-        for data_file_path in data_path.iterdir():
-            workspace_data_file_path = workspace_path / data_file_path.name
-            if workspace_data_file_path.exists():
-                workspace_data_file_path.unlink()
-            subprocess.run(
-                ["ln", "-s", data_file_path, workspace_data_file_path],
-                check=False,
-            )
+    def hash_func(self, data_type: str = "Debug") -> str:
+        return (
+            md5_hash(data_type + self.code_dict["factor.py"])
+            if ("factor.py" in self.code_dict and not self.raise_exception)
+            else None
+        )
 
-    def execute(self, store_result: bool = False, data_type: str = "Debug") -> Tuple[str, pd.DataFrame]:
+    @cache_with_pickle(hash_func)
+    def execute(self, data_type: str = "Debug") -> Tuple[str, pd.DataFrame]:
         """
         execute the implementation and get the factor value by the following steps:
         1. make the directory in workspace path
         2. write the code to the file in the workspace path
         3. link all the source data to the workspace path folder
-        4. execute the code
+        if call_factor_py is True:
+            4. execute the code
+        else:
+            4. generate a script from template to import the factor.py dump get the factor value to result.h5
         5. read the factor value from the output file in the workspace path folder
         returns the execution feedback as a string and the factor value as a pandas dataframe
 
-        parameters:
-        store_result: if True, store the factor value in the instance variable, this feature is to be used in the gt implementation to avoid multiple execution on the same gt implementation
+
+        Regarding the cache mechanism:
+        1. We will store the function's return value to ensure it behaves as expected.
+        - The cached information will include a tuple with the following: (execution_feedback, executed_factor_value_dataframe, Optional[Exception])
+
         """
         super().execute()
         if self.code_dict is None or "factor.py" not in self.code_dict:
@@ -116,40 +121,38 @@ class FactorFBWorkspace(FBWorkspace):
             else:
                 return self.FB_CODE_NOT_SET, None
         with FileLock(self.workspace_path / "execution.lock"):
-            if FACTOR_IMPLEMENT_SETTINGS.enable_execution_cache:
-                # NOTE: cache the result for the same code and same data type
-                target_file_name = md5_hash(data_type + self.code_dict["factor.py"])
-                cache_file_path = Path(FACTOR_IMPLEMENT_SETTINGS.cache_location) / f"{target_file_name}.pkl"
-                Path(FACTOR_IMPLEMENT_SETTINGS.cache_location).mkdir(exist_ok=True, parents=True)
-                if cache_file_path.exists() and not self.raise_exception:
-                    cached_res = pickle.load(open(cache_file_path, "rb"))
-                    if store_result and cached_res[1] is not None:
-                        self.executed_factor_value_dataframe = cached_res[1]
-                    return cached_res
-
-            if self.executed_factor_value_dataframe is not None:
-                return self.FB_FROM_CACHE, self.executed_factor_value_dataframe
-
-            source_data_path = (
-                Path(
-                    FACTOR_IMPLEMENT_SETTINGS.data_folder_debug,
+            if self.target_task.version == 1:
+                source_data_path = (
+                    Path(
+                        FACTOR_IMPLEMENT_SETTINGS.data_folder_debug,
+                    )
+                    if data_type == "Debug"  # FIXME: (yx) don't think we should use a debug tag for this.
+                    else Path(
+                        FACTOR_IMPLEMENT_SETTINGS.data_folder,
+                    )
                 )
-                if data_type == "Debug"
-                else Path(
-                    FACTOR_IMPLEMENT_SETTINGS.data_folder,
-                )
-            )
+            elif self.target_task.version == 2:
+                # TODO you can change the name of the data folder for a better understanding
+                source_data_path = Path(KAGGLE_IMPLEMENT_SETTING.local_data_path) / KAGGLE_IMPLEMENT_SETTING.competition
 
             source_data_path.mkdir(exist_ok=True, parents=True)
             code_path = self.workspace_path / f"factor.py"
 
-            self.link_data_to_workspace(source_data_path, self.workspace_path)
+            self.link_all_files_in_folder_to_workspace(source_data_path, self.workspace_path)
 
             execution_feedback = self.FB_EXECUTION_SUCCEEDED
             execution_success = False
+            execution_error = None
+
+            if self.target_task.version == 1:
+                execution_code_path = code_path
+            elif self.target_task.version == 2:
+                execution_code_path = self.workspace_path / f"{uuid.uuid4()}.py"
+                execution_code_path.write_text((Path(__file__).parent / "factor_execution_template.txt").read_text())
+
             try:
                 subprocess.check_output(
-                    f"{FACTOR_IMPLEMENT_SETTINGS.python_bin} {code_path}",
+                    f"{FACTOR_IMPLEMENT_SETTINGS.python_bin} {execution_code_path}",
                     shell=True,
                     cwd=self.workspace_path,
                     stderr=subprocess.STDOUT,
@@ -161,7 +164,7 @@ class FactorFBWorkspace(FBWorkspace):
 
                 execution_feedback = (
                     e.output.decode()
-                    .replace(str(code_path.parent.absolute()), r"/path/to")
+                    .replace(str(execution_code_path.parent.absolute()), r"/path/to")
                     .replace(str(site.getsitepackages()[0]), r"/path/to/site-packages")
                 )
                 if len(execution_feedback) > 2000:
@@ -170,10 +173,14 @@ class FactorFBWorkspace(FBWorkspace):
                     )
                 if self.raise_exception:
                     raise CustomRuntimeError(execution_feedback)
+                else:
+                    execution_error = CustomRuntimeError(execution_feedback)
             except subprocess.TimeoutExpired:
                 execution_feedback += f"Execution timeout error and the timeout is set to {FACTOR_IMPLEMENT_SETTINGS.file_based_execution_timeout} seconds."
                 if self.raise_exception:
                     raise CustomRuntimeError(execution_feedback)
+                else:
+                    execution_error = CustomRuntimeError(execution_feedback)
 
             workspace_output_file_path = self.workspace_path / "result.h5"
             if workspace_output_file_path.exists() and execution_success:
@@ -188,15 +195,9 @@ class FactorFBWorkspace(FBWorkspace):
                 executed_factor_value_dataframe = None
                 if self.raise_exception:
                     raise NoOutputError(execution_feedback)
+                else:
+                    execution_error = NoOutputError(execution_feedback)
 
-            if store_result and executed_factor_value_dataframe is not None:
-                self.executed_factor_value_dataframe = executed_factor_value_dataframe
-
-        if FACTOR_IMPLEMENT_SETTINGS.enable_execution_cache:
-            pickle.dump(
-                (execution_feedback, executed_factor_value_dataframe),
-                open(cache_file_path, "wb"),
-            )
         return execution_feedback, executed_factor_value_dataframe
 
     def __str__(self) -> str:
@@ -218,3 +219,4 @@ class FactorFBWorkspace(FBWorkspace):
 
 
 FactorExperiment = Experiment
+FeatureExperiment = Experiment

rdagent/components/coder/factor_coder/factor_execution_template.txt

@@ -0,0 +1,15 @@
+import os
+
+import numpy as np
+import pandas as pd
+from factor import feature_engineering_cls
+
+if os.path.exists("X_valid.pkl"):
+    valid_df = pd.read_pickle("X_valid.pkl").head(1000)
+else:
+    raise FileNotFoundError("No valid data found.")
+
+cls = feature_engineering_cls()
+cls.fit(valid_df)
+new_feat = cls.transform(valid_df)
+new_feat.to_hdf("result.h5", key="data", mode="w")

rdagent/components/coder/factor_coder/prompts.yaml

@@ -118,6 +118,13 @@ evolving_strategy_factor_implementation_v2_user: |-
   {{ similar_component_knowledge.implementation.code }}
   {% endfor %}
   {% endif %}
+  {% if latest_attempt_to_latest_successful_execution is not none %}
+  You have tried to correct your former failed code but still met some errors. Here is the latest attempt to the latest successful execution, try not to get the same error to your new code:
+  =====Your latest attempt=====
+  {{ latest_attempt_to_latest_successful_execution.implementation.code }}
+  =====Feedback to your latest attempt=====
+  {{ latest_attempt_to_latest_successful_execution.feedback }}
+  {% endif %}
 
 evolving_strategy_error_summary_v2_system: |-
   User is trying to implement some factors in the following scenario:
@@ -212,7 +219,7 @@ evaluator_final_decision_v1_system: |-
   The implementation final decision is considered in the following logic:
   1. If the value and the ground truth value are exactly the same under a small tolerance, the implementation is considered correct.
   2. If the value and the ground truth value have a high correlation on ic or rank ic, the implementation is considered correct.
-  3. If no ground truth value is not provided, the implementation is considered correct if the code execution is successful and the code feedback is align with the scenario and factor description.
+  3. If no ground truth value is provided, the implementation is considered correct if the code executes successfully (assuming the data provided is correct). Any exceptions, including those actively raised, are considered faults of the code. Additionally, the code feedback must align with the scenario and factor description.
 
   Please response the critic in the json format. Here is an example structure for the JSON output, please strictly follow the format:
   {

rdagent/components/coder/model_coder/CoSTEER/__init__.py

@@ -70,7 +70,7 @@ class ModelCoSTEER(Developer[ModelExperiment]):
         self.rag = ModelRAGStrategy(model_knowledge_base)
 
         # init intermediate items
-        model_experiment = ModelEvolvingItem(sub_tasks=exp.sub_tasks)
+        model_experiment = ModelEvolvingItem.from_experiment(exp)
 
         self.evolve_agent = ModelRAGEvoAgent(
             max_loop=self.max_loop,

rdagent/components/coder/model_coder/CoSTEER/evaluators.py

@@ -4,7 +4,6 @@ from pathlib import Path
 from typing import List, Tuple
 
 import numpy as np
-import torch
 from jinja2 import Environment, StrictUndefined
 
 from rdagent.components.coder.model_coder.conf import MODEL_IMPL_SETTINGS
@@ -19,12 +18,13 @@ from rdagent.core.experiment import Task, Workspace
 from rdagent.core.prompts import Prompts
 from rdagent.core.utils import multiprocessing_wrapper
 from rdagent.log import rdagent_logger as logger
+from rdagent.oai.llm_conf import LLM_SETTINGS
 from rdagent.oai.llm_utils import APIBackend
 
 evaluate_prompts = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")
 
 
-def shape_evaluator(prediction: torch.Tensor, target_shape: Tuple = None) -> Tuple[str, bool]:
+def shape_evaluator(prediction: np.ndarray, target_shape: Tuple = None) -> Tuple[str, bool]:
     if target_shape is None or prediction is None:
         return (
             "No output generated from the model. No shape evaluation conducted.",
@@ -41,18 +41,10 @@ def shape_evaluator(prediction: torch.Tensor, target_shape: Tuple = None) -> Tup
         )
 
 
-def reshape_tensor(original_tensor, target_shape):
-    new_tensor = torch.zeros(target_shape)
-    for i, dim in enumerate(original_tensor.shape):
-        new_tensor = new_tensor.narrow(i, 0, dim).copy_(original_tensor)
-
-    return new_tensor
-
-
 def value_evaluator(
-    prediction: torch.Tensor,
-    target: torch.Tensor,
-) -> Tuple[torch.Tensor, bool]:
+    prediction: np.ndarray,
+    target: np.ndarray,
+) -> Tuple[np.ndarray, bool]:
     if prediction is None:
         return "No output generated from the model. Skip value evaluation", False
     elif target is None:
@@ -62,7 +54,7 @@ def value_evaluator(
         )
     else:
         # Calculate the mean absolute difference
-        diff = torch.mean(torch.abs(target - prediction)).item()
+        diff = np.mean(np.abs(target - prediction))
         return (
             f"The value of the output is correct. The mean absolute difference is {diff}.",
             diff < 0.1,
@@ -89,7 +81,13 @@ class ModelCodeEvaluator(Evaluator):
         system_prompt = (
             Environment(undefined=StrictUndefined)
             .from_string(evaluate_prompts["evaluator_code_feedback"]["system"])
-            .render(scenario=self.scen.get_scenario_all_desc() if self.scen is not None else "No scenario description.")
+            .render(
+                scenario=(
+                    self.scen.get_scenario_all_desc(target_task)
+                    if self.scen is not None
+                    else "No scenario description."
+                )
+            )
         )
 
         execution_feedback_to_render = model_execution_feedback
@@ -112,7 +110,7 @@ class ModelCodeEvaluator(Evaluator):
                     user_prompt=user_prompt,
                     system_prompt=system_prompt,
                 )
-                > RD_AGENT_SETTINGS.chat_token_limit
+                > LLM_SETTINGS.chat_token_limit
             ):
                 execution_feedback_to_render = execution_feedback_to_render[len(execution_feedback_to_render) // 2 :]
             else:
@@ -145,7 +143,13 @@ class ModelFinalEvaluator(Evaluator):
         system_prompt = (
             Environment(undefined=StrictUndefined)
             .from_string(evaluate_prompts["evaluator_final_feedback"]["system"])
-            .render(scenario=self.scen.get_scenario_all_desc() if self.scen is not None else "No scenario description.")
+            .render(
+                scenario=(
+                    self.scen.get_scenario_all_desc(target_task)
+                    if self.scen is not None
+                    else "No scenario description."
+                )
+            )
         )
 
         execution_feedback_to_render = model_execution_feedback
@@ -168,7 +172,7 @@ class ModelFinalEvaluator(Evaluator):
                     user_prompt=user_prompt,
                     system_prompt=system_prompt,
                 )
-                > RD_AGENT_SETTINGS.chat_token_limit
+                > LLM_SETTINGS.chat_token_limit
             ):
                 execution_feedback_to_render = execution_feedback_to_render[len(execution_feedback_to_render) // 2 :]
             else:
@@ -260,7 +264,7 @@ class ModelCoderEvaluator(Evaluator):
         param_init_value = 0.6
 
         assert isinstance(implementation, ModelFBWorkspace)
-        model_execution_feedback, gen_tensor = implementation.execute(
+        model_execution_feedback, gen_np_array = implementation.execute(
             batch_size=batch_size,
             num_features=num_features,
             num_timesteps=num_timesteps,
@@ -269,7 +273,7 @@ class ModelCoderEvaluator(Evaluator):
         )
         if gt_implementation is not None:
             assert isinstance(gt_implementation, ModelFBWorkspace)
-            _, gt_tensor = gt_implementation.execute(
+            _, gt_np_array = gt_implementation.execute(
                 batch_size=batch_size,
                 num_features=num_features,
                 num_timesteps=num_timesteps,
@@ -277,14 +281,10 @@ class ModelCoderEvaluator(Evaluator):
                 param_init_value=param_init_value,
             )
         else:
-            gt_tensor = None
+            gt_np_array = None
 
-        if target_task.model_type == "XGBoost":
-            shape_feedback = "Not applicable for XGBoost models"
-            shape_decision = True
-        else:
-            shape_feedback, shape_decision = shape_evaluator(gen_tensor, (batch_size, 1))
-        value_feedback, value_decision = value_evaluator(gt_tensor, gen_tensor)
+        shape_feedback, shape_decision = shape_evaluator(gen_np_array, (batch_size, 1))
+        value_feedback, value_decision = value_evaluator(gen_np_array, gt_np_array)
         code_feedback, _ = ModelCodeEvaluator(scen=self.scen).evaluate(
             target_task=target_task,
             implementation=implementation,

rdagent/components/coder/model_coder/CoSTEER/evolvable_subjects.py

@@ -27,3 +27,10 @@ class ModelEvolvingItem(ModelExperiment, EvolvableSubjects):
             )
         else:
             self.sub_gt_implementations = sub_gt_implementations
+
+    @classmethod
+    def from_experiment(cls, exp: ModelExperiment) -> "ModelEvolvingItem":
+        ei = cls(sub_tasks=exp.sub_tasks)
+        ei.based_experiments = exp.based_experiments
+        ei.experiment_workspace = exp.experiment_workspace
+        return ei

rdagent/components/coder/model_coder/CoSTEER/evolving_strategy.py

@@ -11,12 +11,18 @@ from rdagent.components.coder.model_coder.CoSTEER.evolvable_subjects import (
 from rdagent.components.coder.model_coder.CoSTEER.knowledge_management import (
     ModelQueriedKnowledge,
 )
-from rdagent.components.coder.model_coder.model import ModelFBWorkspace, ModelTask
+from rdagent.components.coder.model_coder.model import (
+    ModelExperiment,
+    ModelFBWorkspace,
+    ModelTask,
+)
 from rdagent.core.conf import RD_AGENT_SETTINGS
 from rdagent.core.evolving_framework import EvolvingStrategy
 from rdagent.core.prompts import Prompts
 from rdagent.core.utils import multiprocessing_wrapper
+from rdagent.oai.llm_conf import LLM_SETTINGS
 from rdagent.oai.llm_utils import APIBackend
+from rdagent.scenarios.kaggle.experiment.kaggle_experiment import KG_MODEL_MAPPING
 
 coder_prompts = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")
 
@@ -26,8 +32,25 @@ class ModelCoderEvolvingStrategy(EvolvingStrategy):
         self,
         target_task: ModelTask,
         queried_knowledge: ModelQueriedKnowledge = None,
+        current_exp: ModelExperiment = None,  # Add this parameter
     ) -> str:
         model_information_str = target_task.get_task_information()
+        model_type = target_task.model_type
+
+        if len(current_exp.based_experiments) == 0:
+            current_code = None
+        else:
+            current_code = ""
+            sota_exp_code_dict = current_exp.based_experiments[-1].experiment_workspace.code_dict
+            if target_task.version == 2:
+                if model_type in KG_MODEL_MAPPING:
+                    current_code = sota_exp_code_dict.get(KG_MODEL_MAPPING[model_type], None)
+                elif "model.py" in sota_exp_code_dict:
+                    current_code = sota_exp_code_dict["model.py"]
+                else:
+                    current_code = None
+            elif target_task.version == 1:
+                current_code = sota_exp_code_dict.get("model.py", None)
 
         if queried_knowledge is not None and model_information_str in queried_knowledge.success_task_to_knowledge_dict:
             return queried_knowledge.success_task_to_knowledge_dict[model_information_str].implementation
@@ -55,6 +78,7 @@ class ModelCoderEvolvingStrategy(EvolvingStrategy):
                 .render(
                     scenario=self.scen.get_scenario_all_desc(),
                     queried_former_failed_knowledge=queried_former_failed_knowledge_to_render,
+                    current_code=current_code,
                 )
             )
 
@@ -77,7 +101,7 @@ class ModelCoderEvolvingStrategy(EvolvingStrategy):
                         user_prompt=user_prompt,
                         system_prompt=system_prompt,
                     )
-                    < RD_AGENT_SETTINGS.chat_token_limit
+                    < LLM_SETTINGS.chat_token_limit
                 ):
                     break
                 elif len(queried_former_failed_knowledge_to_render) > 1:
@@ -119,15 +143,14 @@ class ModelCoderEvolvingStrategy(EvolvingStrategy):
 
         result = multiprocessing_wrapper(
             [
-                (self.implement_one_model, (evo.sub_tasks[target_index], queried_knowledge))
+                (self.implement_one_model, (evo.sub_tasks[target_index], queried_knowledge, evo))
                 for target_index in to_be_finished_task_index
             ],
             n=RD_AGENT_SETTINGS.multi_proc_n,
         )
 
         for index, target_index in enumerate(to_be_finished_task_index):
-            if evo.sub_workspace_list[target_index] is None:
-                evo.sub_workspace_list[target_index] = ModelFBWorkspace(target_task=evo.sub_tasks[target_index])
+            evo.sub_workspace_list[target_index] = ModelFBWorkspace(target_task=evo.sub_tasks[target_index])
             evo.sub_workspace_list[target_index].inject_code(**{"model.py": result[index]})
 
         evo.corresponding_selection = to_be_finished_task_index

rdagent/components/coder/model_coder/CoSTEER/knowledge_management.py

@@ -1,11 +1,13 @@
+from pathlib import Path
+
 from rdagent.components.coder.model_coder.conf import MODEL_IMPL_SETTINGS
 from rdagent.components.coder.model_coder.CoSTEER.evaluators import ModelCoderFeedback
 from rdagent.components.coder.model_coder.model import ModelTask
 from rdagent.core.evolving_framework import (
     EvolvableSubjects,
+    EvolvingKnowledgeBase,
     EvoStep,
     Knowledge,
-    KnowledgeBase,
     QueriedKnowledge,
     RAGStrategy,
 )
@@ -49,13 +51,15 @@ class ModelQueriedKnowledge(QueriedKnowledge):
         self.working_task_to_similar_successful_knowledge_dict = dict()
 
 
-class ModelKnowledgeBase(KnowledgeBase):
-    def __init__(self) -> None:
+class ModelKnowledgeBase(EvolvingKnowledgeBase):
+    def __init__(self, path: str | Path = None) -> None:
         self.implementation_trace: dict[str, ModelKnowledge] = dict()
         self.success_task_info_set: set[str] = set()
 
         self.task_to_embedding = dict()
 
+        super().__init__(path)
+
     def query(self) -> QueriedKnowledge | None:
         """
         Query the knowledge base to get the queried knowledge. So far is handled in RAG strategy.

rdagent/components/coder/model_coder/conf.py

@@ -10,10 +10,6 @@ class ModelImplSettings(BaseSettings):
 
     coder_use_cache: bool = False
 
-    cache_location: str = str(
-        (Path().cwd() / "git_ignore_folder" / "model_implementation_execution_cache").absolute(),
-    )
-
     knowledge_base_path: Union[str, None] = None
     new_knowledge_base_path: Union[str, None] = None
 
@@ -23,7 +19,5 @@ class ModelImplSettings(BaseSettings):
     query_similar_success_limit: int = 5
     fail_task_trial_limit: int = 20
 
-    enable_execution_cache: bool = True  # whether to enable the execution cache
-
 
 MODEL_IMPL_SETTINGS = ModelImplSettings()

rdagent/components/coder/model_coder/model.py

@@ -1,20 +1,14 @@
-import json
 import pickle
 import site
 import traceback
-import uuid
 from pathlib import Path
-from typing import Any, Dict, Optional
-
-import numpy as np
-import torch
-import xgboost as xgb
+from typing import Dict, Optional
 
 from rdagent.components.coder.model_coder.conf import MODEL_IMPL_SETTINGS
-from rdagent.core.exception import CodeFormatError
 from rdagent.core.experiment import Experiment, FBWorkspace, Task
+from rdagent.core.utils import cache_with_pickle
 from rdagent.oai.llm_utils import md5_hash
-from rdagent.utils import get_module_by_module_path
+from rdagent.utils.env import KGDockerEnv, QTDockerEnv
 
 
 class ModelTask(Task):
@@ -22,29 +16,32 @@ class ModelTask(Task):
         self,
         name: str,
         description: str,
-        formulation: str,
         architecture: str,
-        variables: Dict[str, str],
+        *args,
         hyperparameters: Dict[str, str],
+        formulation: str = None,
+        variables: Dict[str, str] = None,
         model_type: Optional[str] = None,
+        **kwargs,
     ) -> None:
-        self.name: str = name
         self.description: str = description
         self.formulation: str = formulation
         self.architecture: str = architecture
         self.variables: str = variables
         self.hyperparameters: str = hyperparameters
         self.model_type: str = model_type  # Tabular for tabular model, TimesSeries for time series model, Graph for graph model, XGBoost for XGBoost model
+        super().__init__(name=name, *args, **kwargs)
 
     def get_task_information(self):
-        return f"""name: {self.name}
+        task_desc = f"""name: {self.name}
 description: {self.description}
-formulation: {self.formulation}
-architecture: {self.architecture}
-variables: {self.variables}
-hyperparameters: {self.hyperparameters}
-model_type: {self.model_type}
 """
+        task_desc += f"formulation: {self.formulation}\n" if self.formulation else ""
+        task_desc += f"architecture: {self.architecture}\n"
+        task_desc += f"variables: {self.variables}\n" if self.variables else ""
+        task_desc += f"hyperparameters: {self.hyperparameters}\n"
+        task_desc += f"model_type: {self.model_type}\n"
+        return task_desc
 
     @staticmethod
     def from_dict(dict):
@@ -66,14 +63,30 @@ class ModelFBWorkspace(FBWorkspace):
         - the `model.py` that contains a variable named `model_cls` which indicates the implemented model structure
             - `model_cls` is a instance of `torch.nn.Module`;
 
+    We support two ways of interface:
+        (version 1) for qlib we'll make a script to import the model in the implementation in file `model.py` after setting the cwd into the directory
+            - from model import model_cls
+            - initialize the model by initializing it `model_cls(input_dim=INPUT_DIM)`
+            - And then verify the model.
 
-    We'll import the model in the implementation in file `model.py` after setting the cwd into the directory
-    - from model import model_cls
-    - initialize the model by initializing it `model_cls(input_dim=INPUT_DIM)`
-    - And then verify the model.
-
+        (version 2) for kaggle we'll make a script to call the fit and predict function in the implementation in file `model.py` after setting the cwd into the directory
     """
 
+    def hash_func(
+        self,
+        batch_size: int = 8,
+        num_features: int = 10,
+        num_timesteps: int = 4,
+        num_edges: int = 20,
+        input_value: float = 1.0,
+        param_init_value: float = 1.0,
+    ) -> str:
+        target_file_name = f"{batch_size}_{num_features}_{num_timesteps}_{input_value}_{param_init_value}"
+        for code_file_name in sorted(list(self.code_dict.keys())):
+            target_file_name = f"{target_file_name}_{self.code_dict[code_file_name]}"
+        return md5_hash(target_file_name)
+
+    @cache_with_pickle(hash_func)
     def execute(
         self,
         batch_size: int = 8,
@@ -85,75 +98,38 @@ class ModelFBWorkspace(FBWorkspace):
     ):
         super().execute()
         try:
-            if MODEL_IMPL_SETTINGS.enable_execution_cache:
-                # NOTE: cache the result for the same code
-                target_file_name = md5_hash(
-                    f"{batch_size}_{num_features}_{num_timesteps}_{input_value}_{param_init_value}_{self.code_dict['model.py']}"
-                )
-                cache_file_path = Path(MODEL_IMPL_SETTINGS.cache_location) / f"{target_file_name}.pkl"
-                Path(MODEL_IMPL_SETTINGS.cache_location).mkdir(exist_ok=True, parents=True)
-                if cache_file_path.exists():
-                    return pickle.load(open(cache_file_path, "rb"))
-            mod = get_module_by_module_path(str(self.workspace_path / "model.py"))
+            qtde = QTDockerEnv() if self.target_task.version == 1 else KGDockerEnv()
+            qtde.prepare()
 
-            if self.target_task.model_type != "XGBoost":
-                model_cls = mod.model_cls
+            if self.target_task.version == 1:
+                dump_code = f"""
+MODEL_TYPE = "{self.target_task.model_type}"
+BATCH_SIZE = {batch_size}
+NUM_FEATURES = {num_features}
+NUM_TIMESTEPS = {num_timesteps}
+NUM_EDGES = {num_edges}
+INPUT_VALUE = {input_value}
+PARAM_INIT_VALUE = {param_init_value}
+{(Path(__file__).parent / 'model_execute_template_v1.txt').read_text()}
+"""
+            elif self.target_task.version == 2:
+                dump_code = (Path(__file__).parent / "model_execute_template_v2.txt").read_text()
 
-            if self.target_task.model_type == "XGBoost":
-                X_simulated = np.random.rand(100, num_features)  # 100 samples, `num_features` features each
-                y_simulated = np.random.randint(0, 2, 100)  # Binary target for example
-                params = mod.get_params()
-                num_round = mod.get_num_round()
-                dtrain = xgb.DMatrix(X_simulated, label=y_simulated)
-            elif self.target_task.model_type == "Tabular":
-                input_shape = (batch_size, num_features)
-                m = model_cls(num_features=input_shape[1])
-                data = torch.full(input_shape, input_value)
-            elif self.target_task.model_type == "TimeSeries":
-                input_shape = (batch_size, num_features, num_timesteps)
-                m = model_cls(num_features=input_shape[1], num_timesteps=input_shape[2])
-                data = torch.full(input_shape, input_value)
-            elif self.target_task.model_type == "Graph":
-                node_feature = torch.randn(batch_size, num_features)
-                edge_index = torch.randint(0, batch_size, (2, num_edges))
-                m = model_cls(num_features=num_features)
-                data = (node_feature, edge_index)
-            else:
-                raise ValueError(f"Unsupported model type: {self.target_task.model_type}")
-
-            if self.target_task.model_type == "XGBoost":
-                bst = xgb.train(params, dtrain, num_round)
-                y_pred = bst.predict(dtrain)
-                execution_model_output = y_pred
-                execution_feedback_str = "Execution successful, model trained and predictions made."
-            else:
-                # Initialize all parameters of `m` to `param_init_value`
-                for _, param in m.named_parameters():
-                    param.data.fill_(param_init_value)
-
-                # Execute the model
-                if self.target_task.model_type == "Graph":
-                    out = m(*data)
-                else:
-                    out = m(data)
-
-                execution_model_output = out.cpu().detach()
-                execution_feedback_str = f"Execution successful, output tensor shape: {execution_model_output.shape}"
-
-            if MODEL_IMPL_SETTINGS.enable_execution_cache:
-                pickle.dump(
-                    (execution_feedback_str, execution_model_output),
-                    open(cache_file_path, "wb"),
-                )
+            log, results = qtde.dump_python_code_run_and_get_results(
+                code=dump_code,
+                dump_file_names=["execution_feedback_str.pkl", "execution_model_output.pkl"],
+                local_path=str(self.workspace_path),
+                env={},
+                code_dump_file_py_name="model_test",
+            )
+            if results is None:
+                raise RuntimeError(f"Error in running the model code: {log}")
+            [execution_feedback_str, execution_model_output] = results
 
         except Exception as e:
             execution_feedback_str = f"Execution error: {e}\nTraceback: {traceback.format_exc()}"
             execution_model_output = None
 
-        code_path = self.workspace_path / f"model.py"
-        execution_feedback_str = execution_feedback_str.replace(str(code_path.parent.absolute()), r"/path/to").replace(
-            str(site.getsitepackages()[0]), r"/path/to/site-packages"
-        )
         if len(execution_feedback_str) > 2000:
             execution_feedback_str = (
                 execution_feedback_str[:1000] + "....hidden long error message...." + execution_feedback_str[-1000:]

rdagent/components/coder/model_coder/model_execute_template_v1.txt

@@ -0,0 +1,44 @@
+# MODEL_TYPE = "Tabular"
+# BATCH_SIZE = 32
+# NUM_FEATURES = 10
+# NUM_TIMESTEPS = 4
+# NUM_EDGES = 20
+# INPUT_VALUE = 1.0
+# PARAM_INIT_VALUE = 1.0
+
+import pickle
+
+import torch
+from model import model_cls
+
+if MODEL_TYPE == "Tabular":
+    input_shape = (BATCH_SIZE, NUM_FEATURES)
+    m = model_cls(num_features=input_shape[1])
+    data = torch.full(input_shape, INPUT_VALUE)
+elif MODEL_TYPE == "TimeSeries":
+    input_shape = (BATCH_SIZE, NUM_FEATURES, NUM_TIMESTEPS)
+    m = model_cls(num_features=input_shape[1], num_timesteps=input_shape[2])
+    data = torch.full(input_shape, INPUT_VALUE)
+elif MODEL_TYPE == "Graph":
+    node_feature = torch.randn(BATCH_SIZE, NUM_FEATURES)
+    edge_index = torch.randint(0, BATCH_SIZE, (2, NUM_EDGES))
+    m = model_cls(num_features=NUM_FEATURES)
+    data = (node_feature, edge_index)
+else:
+    raise ValueError(f"Unsupported model type: {MODEL_TYPE}")
+
+# Initialize all parameters of `m` to `param_init_value`
+for _, param in m.named_parameters():
+    param.data.fill_(PARAM_INIT_VALUE)
+
+# Execute the model
+if MODEL_TYPE == "Graph":
+    out = m(*data)
+else:
+    out = m(data)
+
+execution_model_output = out.cpu().detach().numpy()
+execution_feedback_str = f"Execution successful, output tensor shape: {execution_model_output.shape}"
+
+pickle.dump(execution_model_output, open("execution_model_output.pkl", "wb"))
+pickle.dump(execution_feedback_str, open("execution_feedback_str.pkl", "wb"))

rdagent/components/coder/model_coder/model_execute_template_v2.txt

@@ -0,0 +1,20 @@
+import os
+import pickle
+
+import numpy as np
+import pandas as pd
+import torch
+from model import fit, predict
+
+train_X = pd.DataFrame(np.random.randn(8, 30), columns=[f"{i}" for i in range(30)])
+train_y = pd.Series(np.random.randint(0, 2, 8))
+valid_X = pd.DataFrame(np.random.randn(8, 30), columns=[f"{i}" for i in range(30)])
+valid_y = pd.Series(np.random.randint(0, 2, 8))
+
+model = fit(train_X, train_y, valid_X, valid_y)
+execution_model_output = predict(model, valid_X).cpu().detach().numpy()
+
+execution_feedback_str = f"Execution successful, output numpy ndarray shape: {execution_model_output.shape}"
+
+pickle.dump(execution_model_output, open("execution_model_output.pkl", "wb"))
+pickle.dump(execution_feedback_str, open("execution_feedback_str.pkl", "wb"))

rdagent/components/coder/model_coder/prompts.yaml

@@ -52,6 +52,16 @@ evolving_strategy_model_coder:
 
         Your must write your code based on your former latest attempt below which consists of your former code and code feedback, you should read the former attempt carefully and must not modify the right part of your former code.
 
+        {% if current_code is not none %}
+        User has write some code before. You should write the new code based on this code. Here is the latest code:
+        ```python
+        {{ current_code }}
+        ```
+        Your code should be very similar to the former code which means your code should be ninety more percent same as the former code! You should not modify the right part of the code.
+        {% else %}
+        User has not write any code before. You should write the new code from scratch.
+        {% endif %}
+
         {% if queried_former_failed_knowledge|length != 0 %}
         --------------Your former latest attempt:---------------
         =====Code to the former implementation=====

rdagent/components/knowledge_management/graph.py

@@ -12,6 +12,7 @@ from rdagent.components.knowledge_management.vector_base import (
     VectorBase,
     cosine,
 )
+from rdagent.core.knowledge_base import KnowledgeBase
 from rdagent.oai.llm_utils import APIBackend
 
 Node = KnowledgeMetaData
@@ -47,14 +48,14 @@ class UndirectedNode(Node):
         )
 
 
-class Graph:
+class Graph(KnowledgeBase):
     """
     base Graph class for Knowledge Graph Search
     """
 
     def __init__(self, path: str | Path | None = None) -> None:
-        self.path = path
         self.nodes = {}
+        super().__init__(path=path)
 
     def size(self) -> int:
         return len(self.nodes)
@@ -77,22 +78,6 @@ class Graph:
                 return node
         return None
 
-    @classmethod
-    def load(cls: type[Graph], path: str | Path) -> Graph:
-        """use pickle as the default load method"""
-        path = path if isinstance(path, Path) else Path(path)
-        if not path.exists():
-            return cls(path=path)
-
-        with path.open("rb") as f:
-            return pickle.load(f)
-
-    def save(self, path: str | Path) -> None:
-        """use pickle as the default save method"""
-        Path.mkdir(path.parent, exist_ok=True)
-        with path.open("wb") as f:
-            pickle.dump(self, f)
-
     @staticmethod
     def batch_embedding(nodes: list[Node]) -> list[Node]:
         contents = [node.content for node in nodes]
@@ -119,8 +104,8 @@ class UndirectedGraph(Graph):
     """
 
     def __init__(self, path: str | Path | None = None) -> None:
-        super().__init__(path=path)
         self.vector_base: VectorBase = PDVectorBase()
+        super().__init__(path=path)
 
     def __str__(self) -> str:
         return f"UndirectedGraph(nodes={self.nodes})"
@@ -174,16 +159,6 @@ class UndirectedGraph(Graph):
 
             node.add_neighbor(neighbor)
 
-    @classmethod
-    def load(cls: type[UndirectedGraph], path: str | Path) -> UndirectedGraph:
-        """use pickle as the default load method"""
-        path = path if isinstance(path, Path) else Path(path)
-        if not path.exists():
-            return cls(path=path)
-
-        with path.open("rb") as f:
-            return pickle.load(f)
-
     def add_nodes(self, node: UndirectedNode, neighbors: list[UndirectedNode]) -> None:
         if not neighbors:
             self.add_node(node)

rdagent/components/knowledge_management/vector_base.py

@@ -5,6 +5,7 @@ from typing import List, Tuple, Union
 import pandas as pd
 from scipy.spatial.distance import cosine
 
+from rdagent.core.knowledge_base import KnowledgeBase
 from rdagent.log import rdagent_logger as logger
 from rdagent.oai.llm_utils import APIBackend
 
@@ -68,14 +69,11 @@ def contents_to_documents(contents: List[str], label: str = None) -> List[Docume
     return docs
 
 
-class VectorBase:
+class VectorBase(KnowledgeBase):
     """
     This class is used for handling vector storage and query
     """
 
-    def __init__(self, vector_df_path: Union[str, Path] = None, **kwargs):
-        pass
-
     def add(self, document: Union[Document, List[Document]]):
         """
         add new node to vector_df
@@ -104,28 +102,15 @@ class VectorBase:
         """
         pass
 
-    def load(self, **kwargs):
-        """load vector_df"""
-
-    def save(self, **kwargs):
-        """save vector_df"""
-
 
 class PDVectorBase(VectorBase):
     """
     Implement of VectorBase using Pandas
     """
 
-    def __init__(self, vector_df_path: Union[str, Path] = None):
-        super().__init__(vector_df_path)
-
-        if vector_df_path:
-            try:
-                self.vector_df = self.load(vector_df_path)
-            except FileNotFoundError:
-                self.vector_df = pd.DataFrame(columns=["id", "label", "content", "embedding"])
-        else:
-            self.vector_df = pd.DataFrame(columns=["id", "label", "content", "embedding"])
+    def __init__(self, path: Union[str, Path] = None):
+        self.vector_df = pd.DataFrame(columns=["id", "label", "content", "embedding"])
+        super().__init__(path)
 
     def shape(self):
         return self.vector_df.shape
@@ -196,10 +181,3 @@ class PDVectorBase(VectorBase):
         for _, similar_docs in most_similar_docs.iterrows():
             docs.append(Document().from_dict(similar_docs.to_dict()))
         return docs, searched_similarities.to_list()
-
-    def load(self, vector_df_path, **kwargs):
-        vector_df = pd.read_pickle(vector_df_path)
-        return vector_df
-
-    def save(self, vector_df_path, **kwargs):
-        self.vector_df.to_pickle(vector_df_path)

rdagent/components/proposal/model_proposal.py

@@ -39,7 +39,7 @@ class ModelHypothesisGen(HypothesisGen):
             Environment(undefined=StrictUndefined)
             .from_string(ModelHypothesisGen.prompts["hypothesis_gen"]["system_prompt"])
             .render(
-                targets="model",
+                targets="model tuning",
                 scenario=self.scen.get_scenario_all_desc(),
                 hypothesis_output_format=context_dict["hypothesis_output_format"],
                 hypothesis_specification=context_dict["hypothesis_specification"],
@@ -49,8 +49,7 @@ class ModelHypothesisGen(HypothesisGen):
             Environment(undefined=StrictUndefined)
             .from_string(ModelHypothesisGen.prompts["hypothesis_gen"]["user_prompt"])
             .render(
-                targets="model",
-                hypothesis_and_feedback=context_dict["hypothesis_and_feedback"],
+                targets="model tuning",
                 RAG=context_dict["RAG"],
             )
         )
@@ -82,7 +81,7 @@ class ModelHypothesis2Experiment(Hypothesis2Experiment[ModelExperiment]):
             Environment(undefined=StrictUndefined)
             .from_string(ModelHypothesis2Experiment.prompts["hypothesis2experiment"]["system_prompt"])
             .render(
-                targets="model",
+                targets="feature engineering and model building",
                 scenario=trace.scen.get_scenario_all_desc(),
                 experiment_output_format=context["experiment_output_format"],
             )
@@ -91,7 +90,7 @@ class ModelHypothesis2Experiment(Hypothesis2Experiment[ModelExperiment]):
             Environment(undefined=StrictUndefined)
             .from_string(ModelHypothesis2Experiment.prompts["hypothesis2experiment"]["user_prompt"])
             .render(
-                targets="model",
+                targets="feature engineering and model building",
                 target_hypothesis=context["target_hypothesis"],
                 hypothesis_and_feedback=context["hypothesis_and_feedback"],
                 target_list=context["target_list"],

rdagent/components/proposal/prompts.yaml

@@ -1,21 +1,25 @@
 hypothesis_gen:
   system_prompt: |-
-    The user is trying to generate new hypothesis on the {{targets}} in data-driven research and development.
-    The {{targets}} are used in a certain scenario, the scenario is as follows:
-    {{ scenario }}
-    The user has made several hypothesis on this scenario and did several evaluation on them. The user will provide this information to you. Check if a new hypothesis has already been proposed. If it is already generated and you agree with it, just use it. If you don't agree, generate a better one.
-    To help you generate new hypothesis, the user has prepared some additional information for you. You should use this information to help generate new {{targets}}.
-    Please generate the output following the format and specifications below:
+    The user is working on generating new hypotheses for the {{targets}} in a data-driven research and development process. 
+    The {{targets}} are used in the following scenario:
+    {{scenario}}
+    The user has already proposed several hypotheses and conducted evaluations on them. This information will be provided to you. Your task is to check whether a similar hypothesis has already been generated. 
+    If one exists and you agree with it, feel free to use it. If you disagree, please generate an improved version.
+    {% if hypothesis_specification %}
+    To assist you in formulating new hypotheses, the user has provided some additional information: {{hypothesis_specification}}.
+    **Important:** If the hypothesis_specification outlines the next steps you need to follow, ensure you adhere to those instructions.
+    {% endif %}
+    Please generate the output using the following format and specifications:
     {{ hypothesis_output_format }}
-    Here are the specifications: {{ hypothesis_specification }}
 
   user_prompt: |-
-    If it is not the first round, then the user has made several hypothesis on this scenario and did several evaluation on them.
-    The former hypothesis and the corresponding feedbacks are as follows (focus on the last one & the new hypothesis that it provides and reasoning to see if you agree):
-    {{ hypothesis_and_feedback }}
-    To help you generate new {{targets}}, we have prepared the following information for you:
-    {{ RAG }}
-    Please generate the new hypothesis based on the information above. Also generate the relevant keys for the reasoning and the distilled knowledge that follows. For those keys, in particular for knowledge, explain in the context of the specific scenario to build up domain knowledge in the specific field rather than genearl knowledge.
+    {% if RAG %}
+    To assist you in generating new {{targets}}, we have provided the following information: {{RAG}}.
+    **Note:** The provided RAG is for reference only. 
+    You must carefully assess whether the RAG aligns with the {{targets}}. 
+    If it does not, it should not be used. Exercise caution and make your own judgment.
+    {% endif %}
+    Also generate the relevant keys for the reasoning and the distilled knowledge that follows. For those keys, in particular for knowledge, explain in the context of the specific scenario to build up domain knowledge in the specific field rather than general knowledge.
 
 hypothesis2experiment:
   system_prompt: |-
@@ -36,8 +40,4 @@ hypothesis2experiment:
     {{ target_hypothesis }}
     The former hypothesis and the corresponding feedbacks are as follows:
     {{ hypothesis_and_feedback }}
-    The former proposed {{targets}} on similar hypothesis are as follows:
-    {{ target_list }}
-    To help you generate new {{targets}}, we have prepared the following information for you:
-    {{ RAG }}
     Please generate the new {{targets}} based on the information above.

rdagent/components/runner/__init__.py

@@ -1,8 +1,7 @@
 import pickle
 from pathlib import Path
-from typing import Tuple
+from typing import Any, Tuple
 
-from rdagent.components.runner.conf import RUNNER_SETTINGS
 from rdagent.core.developer import Developer
 from rdagent.core.experiment import ASpecificExp, Experiment
 from rdagent.oai.llm_utils import md5_hash
@@ -18,16 +17,8 @@ class CachedRunner(Developer[ASpecificExp]):
         task_info_str = "\n".join(task_info_list)
         return md5_hash(task_info_str)
 
-    def get_cache_result(self, exp: Experiment) -> Tuple[bool, object]:
-        task_info_key = self.get_cache_key(exp)
-        Path(RUNNER_SETTINGS.cache_path).mkdir(parents=True, exist_ok=True)
-        cache_path = Path(RUNNER_SETTINGS.cache_path) / f"{task_info_key}.pkl"
-        if cache_path.exists():
-            return True, pickle.load(open(cache_path, "rb"))
-        else:
-            return False, None
-
-    def dump_cache_result(self, exp: Experiment, result: object):
-        task_info_key = self.get_cache_key(exp)
-        cache_path = Path(RUNNER_SETTINGS.cache_path) / f"{task_info_key}.pkl"
-        pickle.dump(result, open(cache_path, "wb"))
+    def assign_cached_result(self, exp: Experiment, cached_res: Experiment) -> Experiment:
+        if exp.based_experiments and exp.based_experiments[-1].result is None:
+            exp.based_experiments[-1].result = cached_res.based_experiments[-1].result
+        exp.result = cached_res.result
+        return exp

rdagent/components/runner/conf.py

@@ -1,16 +0,0 @@
-from __future__ import annotations
-
-from pathlib import Path
-
-from pydantic_settings import BaseSettings
-
-
-class RunnerSettings(BaseSettings):
-    class Config:
-        env_prefix = "RUNNER_"  # Use MODEL_CODER_ as prefix for environment variables
-
-    cache_result: bool = True  # whether to cache the result of the docker execution
-    cache_path: str = str(Path.cwd() / "runner_cache/")  # the path to store the cache
-
-
-RUNNER_SETTINGS = RunnerSettings()

rdagent/components/workflow/conf.py

@@ -14,6 +14,8 @@ class BasePropSetting(BaseSettings):
     """
 
     scen: str = ""
+    knowledge_base: str = ""
+    knowledge_base_path: str = ""
     hypothesis_gen: str = ""
     hypothesis2experiment: str = ""
     coder: str = ""

rdagent/components/workflow/rd_loop.py

@@ -3,6 +3,7 @@ Model workflow with session control
 It is from `rdagent/app/qlib_rd_loop/model.py` and try to replace `rdagent/app/qlib_rd_loop/RDAgent.py`
 """
 
+import time
 from typing import Any
 
 from rdagent.components.workflow.conf import BasePropSetting
@@ -16,10 +17,12 @@ from rdagent.core.proposal import (
 from rdagent.core.scenario import Scenario
 from rdagent.core.utils import import_class
 from rdagent.log import rdagent_logger as logger
+from rdagent.log.time import measure_time
 from rdagent.utils.workflow import LoopBase, LoopMeta
 
 
 class RDLoop(LoopBase, metaclass=LoopMeta):
+    @measure_time
     def __init__(self, PROP_SETTING: BasePropSetting):
         with logger.tag("init"):
             scen: Scenario = import_class(PROP_SETTING.scen)()
@@ -41,30 +44,35 @@ class RDLoop(LoopBase, metaclass=LoopMeta):
             self.trace = Trace(scen=scen)
             super().__init__()
 
+    @measure_time
     def propose(self, prev_out: dict[str, Any]):
         with logger.tag("r"):  # research
             hypothesis = self.hypothesis_gen.gen(self.trace)
             logger.log_object(hypothesis, tag="hypothesis generation")
         return hypothesis
 
+    @measure_time
     def exp_gen(self, prev_out: dict[str, Any]):
         with logger.tag("r"):  # research
             exp = self.hypothesis2experiment.convert(prev_out["propose"], self.trace)
             logger.log_object(exp.sub_tasks, tag="experiment generation")
         return exp
 
+    @measure_time
     def coding(self, prev_out: dict[str, Any]):
         with logger.tag("d"):  # develop
             exp = self.coder.develop(prev_out["exp_gen"])
             logger.log_object(exp.sub_workspace_list, tag="coder result")
         return exp
 
+    @measure_time
     def running(self, prev_out: dict[str, Any]):
         with logger.tag("ef"):  # evaluate and feedback
             exp = self.runner.develop(prev_out["coding"])
             logger.log_object(exp, tag="runner result")
         return exp
 
+    @measure_time
     def feedback(self, prev_out: dict[str, Any]):
         feedback = self.summarizer.generate_feedback(prev_out["running"], prev_out["propose"], self.trace)
         with logger.tag("ef"):  # evaluate and feedback

rdagent/core/conf.py

@@ -14,83 +14,10 @@ class RDAgentSettings(BaseSettings):
     # Log configs
     # TODO: (xiao) think it can be a separate config.
     log_trace_path: str | None = None
-    log_llm_chat_content: bool = True
-
-    use_azure: bool = False
-    use_azure_token_provider: bool = False
-    managed_identity_client_id: str | None = None
-    max_retry: int = 10
-    retry_wait_seconds: int = 1
-    dump_chat_cache: bool = False
-    use_chat_cache: bool = False
-    dump_embedding_cache: bool = False
-    use_embedding_cache: bool = False
-    prompt_cache_path: str = str(Path.cwd() / "prompt_cache.db")
-    session_cache_folder_location: str = str(Path.cwd() / "session_cache_folder/")
-    max_past_message_include: int = 10
-
-    # Chat configs
-    openai_api_key: str = ""  # TODO: simplify the key design.
-    chat_openai_api_key: str = ""
-    chat_azure_api_base: str = ""
-    chat_azure_api_version: str = ""
-    chat_model: str = "gpt-4-turbo"
-    chat_max_tokens: int = 3000
-    chat_temperature: float = 0.5
-    chat_stream: bool = True
-    chat_seed: int | None = None
-    chat_frequency_penalty: float = 0.0
-    chat_presence_penalty: float = 0.0
-    chat_token_limit: int = (
-        100000  # 100000 is the maximum limit of gpt4, which might increase in the future version of gpt
-    )
-    default_system_prompt: str = "You are an AI assistant who helps to answer user's questions."
-
-    # Embedding configs
-    embedding_openai_api_key: str = ""
-    embedding_azure_api_base: str = ""
-    embedding_azure_api_version: str = ""
-    embedding_model: str = ""
-
-    # offline llama2 related config
-    use_llama2: bool = False
-    llama2_ckpt_dir: str = "Llama-2-7b-chat"
-    llama2_tokenizer_path: str = "Llama-2-7b-chat/tokenizer.model"
-    llams2_max_batch_size: int = 8
 
     # azure document intelligence configs
     azure_document_intelligence_key: str = ""
     azure_document_intelligence_endpoint: str = ""
-
-    # server served endpoints
-    use_gcr_endpoint: bool = False
-    gcr_endpoint_type: str = "llama2_70b"  # or "llama3_70b", "phi2", "phi3_4k", "phi3_128k"
-
-    llama2_70b_endpoint: str = ""
-    llama2_70b_endpoint_key: str = ""
-    llama2_70b_endpoint_deployment: str = ""
-
-    llama3_70b_endpoint: str = ""
-    llama3_70b_endpoint_key: str = ""
-    llama3_70b_endpoint_deployment: str = ""
-
-    phi2_endpoint: str = ""
-    phi2_endpoint_key: str = ""
-    phi2_endpoint_deployment: str = ""
-
-    phi3_4k_endpoint: str = ""
-    phi3_4k_endpoint_key: str = ""
-    phi3_4k_endpoint_deployment: str = ""
-
-    phi3_128k_endpoint: str = ""
-    phi3_128k_endpoint_key: str = ""
-    phi3_128k_endpoint_deployment: str = ""
-
-    gcr_endpoint_temperature: float = 0.7
-    gcr_endpoint_top_p: float = 0.9
-    gcr_endpoint_do_sample: bool = False
-    gcr_endpoint_max_token: int = 100
-
     # factor extraction conf
     max_input_duplicate_factor_group: int = 300
     max_output_duplicate_factor_group: int = 20
@@ -102,5 +29,15 @@ class RDAgentSettings(BaseSettings):
     # multi processing conf
     multi_proc_n: int = 1
 
+    # pickle cache conf
+    cache_with_pickle: bool = True  # whether to use pickle cache
+    pickle_cache_folder_path_str: str = str(
+        Path.cwd() / "pickle_cache/",
+    )  # the path of the folder to store the pickle cache
+    use_file_lock: bool = (
+        True  # when calling the function with same parameters, whether to use file lock to avoid
+        # executing the function multiple times
+    )
+
 
 RD_AGENT_SETTINGS = RDAgentSettings()

rdagent/core/evolving_agent.py

@@ -78,6 +78,8 @@ class RAGEvoAgent(EvoAgent):
             )
             # TODO: Due to design issues, we have chosen to ignore this mypy error.
             logger.log_object(evo.sub_workspace_list, tag="evolving code")  # type: ignore[attr-defined]
+            for sw in evo.sub_workspace_list:  # type: ignore[attr-defined]
+                logger.info(f"evolving code workspace: {sw}")
 
             # 4. Pack evolve results
             es = EvoStep(evo, queried_knowledge)

rdagent/core/evolving_framework.py

@@ -5,6 +5,8 @@ from abc import ABC, abstractmethod
 from dataclasses import dataclass
 from typing import TYPE_CHECKING, Any
 
+from rdagent.core.knowledge_base import KnowledgeBase
+
 if TYPE_CHECKING:
     from rdagent.core.evaluation import Feedback
     from rdagent.core.scenario import Scenario
@@ -18,7 +20,7 @@ class QueriedKnowledge:
     pass
 
 
-class KnowledgeBase(ABC):
+class EvolvingKnowledgeBase(KnowledgeBase):
     @abstractmethod
     def query(
         self,
@@ -78,7 +80,7 @@ class EvolvingStrategy(ABC):
 class RAGStrategy(ABC):
     """Retrieval Augmentation Generation Strategy"""
 
-    def __init__(self, knowledgebase: KnowledgeBase) -> None:
+    def __init__(self, knowledgebase: EvolvingKnowledgeBase) -> None:
         self.knowledgebase = knowledgebase
 
     @abstractmethod

rdagent/core/experiment.py

@@ -1,11 +1,14 @@
 from __future__ import annotations
 
+import os
+import platform
 import shutil
 import uuid
 from abc import ABC, abstractmethod
+from collections.abc import Sequence
 from copy import deepcopy
 from pathlib import Path
-from typing import Any, Generic, Sequence, TypeVar
+from typing import Any, Generic, TypeVar
 
 from rdagent.core.conf import RD_AGENT_SETTINGS
 
@@ -15,6 +18,15 @@ This file contains the all the class about organizing the task in RD-Agent.
 
 
 class Task(ABC):
+    def __init__(self, name: str, version: int = 1) -> None:
+        """
+        The version of the task, default is 1
+        Because qlib tasks execution and kaggle tasks execution are different, we need to distinguish them.
+        TODO: We may align them in the future.
+        """
+        self.version = version
+        self.name = name
+
     @abstractmethod
     def get_task_information(self) -> str:
         """
@@ -103,6 +115,19 @@ class FBWorkspace(Workspace):
         """
         self.workspace_path.mkdir(parents=True, exist_ok=True)
 
+    @staticmethod
+    def link_all_files_in_folder_to_workspace(data_path: Path, workspace_path: Path) -> None:
+        data_path = Path(data_path).absolute()  # in case of relative path that will be invalid when we change cwd.
+        workspace_path = Path(workspace_path)
+        for data_file_path in data_path.iterdir():
+            workspace_data_file_path = workspace_path / data_file_path.name
+            if workspace_data_file_path.exists():
+                workspace_data_file_path.unlink()
+            if platform.system() == "Linux":
+                os.symlink(data_file_path, workspace_data_file_path)
+            if platform.system() == "Windows":
+                os.link(data_file_path, workspace_data_file_path)
+
     def inject_code(self, **files: str) -> None:
         """
         Inject the code into the folder.
@@ -113,6 +138,9 @@ class FBWorkspace(Workspace):
         self.prepare()
         for k, v in files.items():
             self.code_dict[k] = v
+            target_file_path = self.workspace_path / k
+            if not target_file_path.parent.exists():
+                target_file_path.parent.mkdir(parents=True, exist_ok=True)
             with Path.open(self.workspace_path / k, "w") as f:
                 f.write(v)
 
@@ -129,9 +157,10 @@ class FBWorkspace(Workspace):
         """
         Load the workspace from the folder
         """
-        for file_path in folder_path.iterdir():
-            if file_path.suffix in {".py", ".yaml"}:
-                self.inject_code(**{file_path.name: file_path.read_text()})
+        for file_path in folder_path.rglob("*"):
+            if file_path.suffix in (".py", ".yaml", ".md"):
+                relative_path = file_path.relative_to(folder_path)
+                self.inject_code(**{str(relative_path): file_path.read_text()})
 
     def copy(self) -> FBWorkspace:
         """
@@ -143,7 +172,7 @@ class FBWorkspace(Workspace):
         """
         Clear the workspace
         """
-        shutil.rmtree(self.workspace_path)
+        shutil.rmtree(self.workspace_path, ignore_errors=True)
         self.code_dict = {}
 
     def execute(self) -> object | None:
@@ -154,20 +183,32 @@ class FBWorkspace(Workspace):
         self.inject_code(**self.code_dict)
         return None
 
+    def __str__(self) -> str:
+        return f"Workspace[{self.workspace_path=}" + (
+            "]" if self.target_task is None else f",{self.target_task.name=}]"
+        )
+
 
 ASpecificWSForExperiment = TypeVar("ASpecificWSForExperiment", bound=Workspace)
 ASpecificWSForSubTasks = TypeVar("ASpecificWSForSubTasks", bound=Workspace)
 
 
-class Experiment(ABC, Generic[ASpecificTask, ASpecificWSForExperiment, ASpecificWSForSubTasks]):
+class Experiment(
+    ABC,
+    Generic[ASpecificTask, ASpecificWSForExperiment, ASpecificWSForSubTasks],
+):
     """
     The experiment is a sequence of tasks and the implementations of the tasks after generated by the Developer.
     """
 
-    def __init__(self, sub_tasks: Sequence[ASpecificTask]) -> None:
+    def __init__(
+        self,
+        sub_tasks: Sequence[ASpecificTask],
+        based_experiments: Sequence[ASpecificWSForExperiment] = [],
+    ) -> None:
         self.sub_tasks = sub_tasks
         self.sub_workspace_list: list[ASpecificWSForSubTasks | None] = [None] * len(self.sub_tasks)
-        self.based_experiments: Sequence[ASpecificWSForExperiment] = []
+        self.based_experiments: Sequence[ASpecificWSForExperiment] = based_experiments
         self.result: object = None  # The result of the experiment, can be different types in different scenarios.
         self.experiment_workspace: ASpecificWSForExperiment | None = None
 

rdagent/core/knowledge_base.py

@@ -0,0 +1,27 @@
+from pathlib import Path
+
+import dill as pickle  # type: ignore[import-untyped]
+
+from rdagent.log import rdagent_logger as logger
+
+
+class KnowledgeBase:
+    def __init__(self, path: str | Path | None = None) -> None:
+        self.path = Path(path) if path else None
+        self.load()
+
+    def load(self) -> None:
+        if self.path is not None and self.path.exists():
+            with self.path.open("rb") as f:
+                loaded = pickle.load(f)
+                if isinstance(loaded, dict):
+                    self.__dict__.update(loaded)
+                else:
+                    self.__dict__.update(loaded.__dict__)
+
+    def dump(self) -> None:
+        if self.path is not None:
+            self.path.parent.mkdir(parents=True, exist_ok=True)
+            pickle.dump(self.__dict__, self.path.open("wb"))
+        else:
+            logger.warning("KnowledgeBase path is not set, dump failed.")

rdagent/core/prompts.py

@@ -1,12 +1,11 @@
-from pathlib import Path  # noqa: I001
-from typing import Dict
+from pathlib import Path
 
 import yaml
 
 from rdagent.core.utils import SingletonBaseClass
 
 
-class Prompts(SingletonBaseClass, Dict[str, str]):
+class Prompts(SingletonBaseClass, dict[str, str]):
     def __init__(self, file_path: Path) -> None:
         super().__init__()
         with file_path.open(encoding="utf8") as file:

rdagent/core/proposal.py

@@ -9,6 +9,7 @@ from typing import TYPE_CHECKING, Generic, TypeVar
 
 from rdagent.core.evaluation import Feedback
 from rdagent.core.experiment import ASpecificExp, Experiment
+from rdagent.core.knowledge_base import KnowledgeBase
 from rdagent.core.scenario import Scenario
 
 if TYPE_CHECKING:
@@ -83,12 +84,14 @@ Reason: {self.reason}"""
 
 
 ASpecificScen = TypeVar("ASpecificScen", bound=Scenario)
+ASpecificKB = TypeVar("ASpecificKB", bound=KnowledgeBase)
 
 
-class Trace(Generic[ASpecificScen]):
-    def __init__(self, scen: ASpecificScen) -> None:
+class Trace(Generic[ASpecificScen, ASpecificKB]):
+    def __init__(self, scen: ASpecificScen, knowledge_base: ASpecificKB | None = None) -> None:
         self.scen: ASpecificScen = scen
         self.hist: list[tuple[Hypothesis, Experiment, HypothesisFeedback]] = []
+        self.knowledge_base: ASpecificKB | None = knowledge_base
 
     def get_sota_hypothesis_and_experiment(self) -> tuple[Hypothesis | None, Experiment | None]:
         """Access the last experiment result, sub-task, and the corresponding hypothesis."""

rdagent/core/scenario.py

@@ -1,5 +1,7 @@
 from abc import ABC, abstractmethod
 
+from rdagent.core.experiment import Task
+
 
 class Scenario(ABC):
     @property
@@ -7,10 +9,21 @@ class Scenario(ABC):
     def background(self) -> str:
         """Background information"""
 
+    # TODO: We have to change all the sub classes to override get_source_data_desc instead of `source_data`
+    def get_source_data_desc(self, task: Task | None = None) -> str:  # noqa: ARG002
+        """
+        Source data description
+
+        The choice of data may vary based on the specific task at hand.
+        """
+        return ""
+
     @property
-    @abstractmethod
     def source_data(self) -> str:
-        """Source data description"""
+        """
+        A convenient shortcut for describing source data
+        """
+        return self.get_source_data_desc()
 
     @property
     @abstractmethod
@@ -33,8 +46,12 @@ class Scenario(ABC):
         """Rich style description to present"""
 
     @abstractmethod
-    def get_scenario_all_desc(self) -> str:
-        """Combine all the description together"""
+    def get_scenario_all_desc(self, task: Task | None = None) -> str:
+        """
+        Combine all descriptions together
+
+        The scenario description varies based on the task being performed.
+        """
 
     @property
     def experiment_setting(self) -> str | None:

rdagent/core/utils.py

@@ -1,14 +1,21 @@
 from __future__ import annotations
 
+import functools
 import importlib
 import json
 import multiprocessing as mp
 import pickle
+import random
 from collections.abc import Callable
+from pathlib import Path
 from typing import Any, ClassVar, NoReturn, cast
 
+from filelock import FileLock
 from fuzzywuzzy import fuzz  # type: ignore[import-untyped]
 
+from rdagent.core.conf import RD_AGENT_SETTINGS
+from rdagent.oai.llm_conf import LLM_SETTINGS
+
 
 class RDAgentException(Exception):  # noqa: N818
     pass
@@ -30,7 +37,7 @@ class SingletonBaseClass:
             raise RDAgentException(exception_message)
         class_name = [(-1, f"{cls.__module__}.{cls.__name__}")]
         args_l = [(i, args[i]) for i in args]
-        kwargs_l = list(sorted(kwargs.items()))
+        kwargs_l = sorted(kwargs.items())
         all_args = class_name + args_l + kwargs_l
         kwargs_hash = hash(tuple(all_args))
         if kwargs_hash not in cls._instance_dict:
@@ -81,11 +88,48 @@ def import_class(class_path: str) -> Any:
     return getattr(module, class_name)
 
 
+class CacheSeedGen:
+    """
+    It is a global seed generator to generate a sequence of seeds.
+    This will support the feature `use_auto_chat_cache_seed_gen` claim
+
+    NOTE:
+    - This seed is specifically for the cache and is different from a regular seed.
+    - If the cache is removed, setting the same seed will not produce the same QA trace.
+    """
+
+    def __init__(self) -> None:
+        self.set_seed(LLM_SETTINGS.init_chat_cache_seed)
+
+    def set_seed(self, seed: int) -> None:
+        random.seed(seed)
+
+    def get_next_seed(self) -> int:
+        """generate next random int"""
+        return random.randint(0, 10000)  # noqa: S311
+
+
+LLM_CACHE_SEED_GEN = CacheSeedGen()
+
+
+def _subprocess_wrapper(f: Callable, seed: int, args: list) -> Any:
+    """
+    It is a function wrapper. To ensure the subprocess has a fixed start seed.
+    """
+
+    LLM_CACHE_SEED_GEN.set_seed(seed)
+    return f(*args)
+
+
 def multiprocessing_wrapper(func_calls: list[tuple[Callable, tuple]], n: int) -> list:
     """It will use multiprocessing to call the functions in func_calls with the given parameters.
     The results equals to `return  [f(*args) for f, args in func_calls]`
     It will not call multiprocessing if `n=1`
 
+    NOTE:
+    We coooperate with chat_cache_seed feature
+    We ensure get the same seed trace even we have multiple number of seed
+
     Parameters
     ----------
     func_calls : List[Tuple[Callable, Tuple]]
@@ -100,6 +144,58 @@ def multiprocessing_wrapper(func_calls: list[tuple[Callable, tuple]], n: int) ->
     """
     if n == 1:
         return [f(*args) for f, args in func_calls]
-    with mp.Pool(processes=n) as pool:
-        results = [pool.apply_async(f, args) for f, args in func_calls]
+
+    with mp.Pool(processes=max(1, min(n, len(func_calls)))) as pool:
+        results = [
+            pool.apply_async(_subprocess_wrapper, args=(f, LLM_CACHE_SEED_GEN.get_next_seed(), args))
+            for f, args in func_calls
+        ]
         return [result.get() for result in results]
+
+
+def cache_with_pickle(hash_func: Callable, post_process_func: Callable | None = None) -> Callable:
+    """
+    This decorator will cache the return value of the function with pickle.
+    The cache key is generated by the hash_func. The hash function returns a string or None.
+    If it returns None, the cache will not be used. The cache will be stored in the folder
+    specified by RD_AGENT_SETTINGS.pickle_cache_folder_path_str with name hash_key.pkl.
+    The post_process_func will be called with the original arguments and the cached result
+    to give each caller a chance to process the cached result. The post_process_func should
+    return the final result.
+    """
+
+    def cache_decorator(func: Callable) -> Callable:
+        @functools.wraps(func)
+        def cache_wrapper(*args: Any, **kwargs: Any) -> Any:
+            if not RD_AGENT_SETTINGS.cache_with_pickle:
+                return func(*args, **kwargs)
+
+            target_folder = Path(RD_AGENT_SETTINGS.pickle_cache_folder_path_str) / f"{func.__module__}.{func.__name__}"
+            target_folder.mkdir(parents=True, exist_ok=True)
+            hash_key = hash_func(*args, **kwargs)
+
+            if hash_key is None:
+                return func(*args, **kwargs)
+
+            cache_file = target_folder / f"{hash_key}.pkl"
+            lock_file = target_folder / f"{hash_key}.lock"
+
+            if cache_file.exists():
+                with cache_file.open("rb") as f:
+                    cached_res = pickle.load(f)
+                return post_process_func(*args, cached_res=cached_res, **kwargs) if post_process_func else cached_res
+
+            if RD_AGENT_SETTINGS.use_file_lock:
+                with FileLock(lock_file):
+                    result = func(*args, **kwargs)
+            else:
+                result = func(*args, **kwargs)
+
+            with cache_file.open("wb") as f:
+                pickle.dump(result, f)
+
+            return result
+
+        return cache_wrapper
+
+    return cache_decorator

rdagent/log/storage.py

@@ -68,10 +68,10 @@ class FileStorage(Storage):
     def iter_msg(self, watch: bool = False) -> Generator[Message, None, None]:
         msg_l = []
         for file in self.path.glob("**/*.log"):
-            tag = ".".join(str(file.relative_to(self.path)).replace("/", ".").split(".")[:-3])
+            tag = ".".join(file.relative_to(self.path).as_posix().replace("/", ".").split(".")[:-3])
             pid = file.parent.name
 
-            with file.open("r") as f:
+            with file.open("r", encoding="utf-8") as f:
                 content = f.read()
 
             matches, next_matches = self.log_pattern.finditer(content), self.log_pattern.finditer(content)
@@ -100,7 +100,7 @@ class FileStorage(Storage):
                 msg_l.append(m)
 
         for file in self.path.glob("**/*.pkl"):
-            tag = ".".join(str(file.relative_to(self.path)).replace("/", ".").split(".")[:-3])
+            tag = ".".join(file.relative_to(self.path).as_posix().replace("/", ".").split(".")[:-3])
             pid = file.parent.name
 
             with file.open("rb") as f:

rdagent/log/time.py

@@ -0,0 +1,19 @@
+import time
+from functools import wraps
+
+from rdagent.log import rdagent_logger as logger
+
+
+def measure_time(method):
+    @wraps(method)
+    def timed(*args, **kwargs):
+        start_time = time.time()
+        result = method(*args, **kwargs)
+        end_time = time.time()
+        duration = end_time - start_time
+        method_name = method.__name__
+        # logger.log_object(f"{method_name} took {duration:.2f} sec")
+        logger.info(f"{method_name} took {duration:.2f} sec")
+        return result
+
+    return timed

rdagent/log/ui/app.py

@@ -27,6 +27,7 @@ from rdagent.log.storage import FileStorage
 from rdagent.log.ui.qlib_report_figure import report_figure
 from rdagent.scenarios.data_mining.experiment.model_experiment import DMModelScenario
 from rdagent.scenarios.general_model.scenario import GeneralModelScenario
+from rdagent.scenarios.kaggle.experiment.scenario import KGScenario
 from rdagent.scenarios.qlib.experiment.factor_experiment import QlibFactorScenario
 from rdagent.scenarios.qlib.experiment.factor_from_report_experiment import (
     QlibFactorFromReportScenario,
@@ -53,13 +54,15 @@ else:
     main_log_path = None
 
 
-SELECTED_METRICS = [
+QLIB_SELECTED_METRICS = [
     "IC",
     "1day.excess_return_without_cost.annualized_return",
     "1day.excess_return_without_cost.information_ratio",
     "1day.excess_return_without_cost.max_drawdown",
 ]
 
+SIMILAR_SCENARIOS = (QlibModelScenario, DMModelScenario, QlibFactorScenario, QlibFactorFromReportScenario, KGScenario)
+
 if "log_path" not in state:
     if main_log_path:
         state.log_path = next(main_log_path.iterdir()).relative_to(main_log_path)
@@ -85,6 +88,9 @@ if "current_tags" not in state:
 if "lround" not in state:
     state.lround = 0  # RD Loop Round
 
+if "times" not in state:
+    state.times = defaultdict(lambda: defaultdict(list))
+
 if "erounds" not in state:
     state.erounds = defaultdict(int)  # Evolving Rounds in each RD Loop
 
@@ -137,23 +143,27 @@ def get_msgs_until(end_func: Callable[[Message], bool] = lambda _: True):
                     if "model runner result" in tags or "factor runner result" in tags or "runner result" in tags:
                         # factor baseline exp metrics
                         if isinstance(state.scenario, QlibFactorScenario) and state.alpha158_metrics is None:
-                            sms = msg.content.based_experiments[0].result.loc[SELECTED_METRICS]
+                            sms = msg.content.based_experiments[0].result.loc[QLIB_SELECTED_METRICS]
                             sms.name = "alpha158"
                             state.alpha158_metrics = sms
 
                         # common metrics
                         if msg.content.result is None:
-                            state.metric_series.append(pd.Series([None], index=["AUROC"], name=f"Round {state.lround}"))
+                            if isinstance(state.scenario, DMModelScenario):
+                                state.metric_series.append(
+                                    pd.Series([None], index=["AUROC"], name=f"Round {state.lround}")
+                                )
                         else:
-                            if len(msg.content.result) < 4:
-                                ps = msg.content.result
-                                ps.index = ["AUROC"]
-                                ps.name = f"Round {state.lround}"
-                                state.metric_series.append(ps)
-                            else:
-                                sms = msg.content.result.loc[SELECTED_METRICS]
-                                sms.name = f"Round {state.lround}"
-                                state.metric_series.append(sms)
+                            sms = msg.content.result
+                            if isinstance(state.scenario, DMModelScenario):
+                                sms.index = ["AUROC"]
+                            elif isinstance(
+                                state.scenario, (QlibModelScenario, QlibFactorFromReportScenario, QlibFactorScenario)
+                            ):
+                                sms = sms.loc[QLIB_SELECTED_METRICS]
+
+                            sms.name = f"Round {state.lround}"
+                            state.metric_series.append(sms)
                     elif "hypothesis generation" in tags:
                         state.hypotheses[state.lround] = msg.content
                     elif "ef" in tags and "feedback" in tags:
@@ -179,6 +189,17 @@ def get_msgs_until(end_func: Callable[[Message], bool] = lambda _: True):
                             )
 
                     state.msgs[state.lround][msg.tag].append(msg)
+
+                    # Update Times
+                    if "init" in tags:
+                        state.times[state.lround]["init"].append(msg.timestamp)
+                    if "r" in tags:
+                        state.times[state.lround]["r"].append(msg.timestamp)
+                    if "d" in tags:
+                        state.times[state.lround]["d"].append(msg.timestamp)
+                    if "ef" in tags:
+                        state.times[state.lround]["ef"].append(msg.timestamp)
+
                     # Stop Getting Logs
                     if end_func(msg):
                         break
@@ -217,6 +238,7 @@ def refresh(same_trace: bool = False):
     state.last_msg = None
     state.current_tags = []
     state.alpha158_metrics = None
+    state.times = defaultdict(lambda: defaultdict(list))
 
 
 def evolving_feedback_window(wsf: FactorSingleFeedback | ModelCoderFeedback):
@@ -342,9 +364,7 @@ def metrics_window(df: pd.DataFrame, R: int, C: int, *, height: int = 300, color
 
 
 def summary_window():
-    if isinstance(
-        state.scenario, (QlibModelScenario, DMModelScenario, QlibFactorScenario, QlibFactorFromReportScenario)
-    ):
+    if isinstance(state.scenario, SIMILAR_SCENARIOS):
         st.header("Summary📊", divider="rainbow", anchor="_summary")
         if state.lround == 0:
             return
@@ -380,7 +400,6 @@ def summary_window():
                     st.table(df.iloc[0])
                 elif df.shape[0] > 1:
                     if df.shape[1] == 1:
-                        # suhan's scenario
                         fig = px.line(df, x=df.index, y=df.columns, markers=True)
                         fig.update_layout(xaxis_title="Loop Round", yaxis_title=None)
                         st.plotly_chart(fig)
@@ -439,9 +458,10 @@ def tasks_window(tasks: list[FactorTask | ModelTask]):
                 st.latex(ft.factor_formulation)
 
                 mks = "| Variable | Description |\n| --- | --- |\n"
-                for v, d in ft.variables.items():
-                    mks += f"| ${v}$ | {d} |\n"
-                st.markdown(mks)
+                if isinstance(ft.variables, dict):
+                    for v, d in ft.variables.items():
+                        mks += f"| ${v}$ | {d} |\n"
+                    st.markdown(mks)
 
     elif isinstance(tasks[0], ModelTask):
         st.markdown("**Model Tasks🚩**")
@@ -458,24 +478,17 @@ def tasks_window(tasks: list[FactorTask | ModelTask]):
                 st.latex(mt.formulation)
 
                 mks = "| Variable | Description |\n| --- | --- |\n"
-                for v, d in mt.variables.items():
-                    mks += f"| ${v}$ | {d} |\n"
-                st.markdown(mks)
+                if mt.variables:
+                    for v, d in mt.variables.items():
+                        mks += f"| ${v}$ | {d} |\n"
+                    st.markdown(mks)
 
 
 def research_window():
     with st.container(border=True):
-        title = (
-            "Research🔍"
-            if isinstance(
-                state.scenario, (QlibModelScenario, DMModelScenario, QlibFactorScenario, QlibFactorFromReportScenario)
-            )
-            else "Research🔍 (reader)"
-        )
+        title = "Research🔍" if isinstance(state.scenario, SIMILAR_SCENARIOS) else "Research🔍 (reader)"
         st.subheader(title, divider="blue", anchor="_research")
-        if isinstance(
-            state.scenario, (QlibModelScenario, DMModelScenario, QlibFactorScenario, QlibFactorFromReportScenario)
-        ):
+        if isinstance(state.scenario, SIMILAR_SCENARIOS):
             # pdf image
             if pim := state.msgs[round]["r.extract_factors_and_implement.load_pdf_screenshot"]:
                 for i in range(min(2, len(pim))):
@@ -505,19 +518,22 @@ def research_window():
             # loaded model exp
             with c2:
                 if mem := state.msgs[round]["d.load_experiment"]:
+                    # 'load_experiment' should in 'r' now, but old version trace may in 'd', so we need to check both
+                    # TODO: modify the way to get one message with a specific tag like 'load_experiment' in the future
+                    me: QlibModelExperiment = mem[0].content
+                    tasks_window(me.sub_tasks)
+                elif mem := state.msgs[round]["r.load_experiment"]:
                     me: QlibModelExperiment = mem[0].content
                     tasks_window(me.sub_tasks)
 
 
 def feedback_window():
-    if isinstance(
-        state.scenario, (QlibModelScenario, DMModelScenario, QlibFactorScenario, QlibFactorFromReportScenario)
-    ):
+    if isinstance(state.scenario, SIMILAR_SCENARIOS):
         with st.container(border=True):
             st.subheader("Feedback📝", divider="orange", anchor="_feedback")
 
             if state.lround > 0 and isinstance(
-                state.scenario, (QlibModelScenario, QlibFactorScenario, QlibFactorFromReportScenario)
+                state.scenario, (QlibModelScenario, QlibFactorScenario, QlibFactorFromReportScenario, KGScenario)
             ):
                 with st.expander("**Config⚙️**", expanded=True):
                     st.markdown(state.scenario.experiment_setting, unsafe_allow_html=True)
@@ -538,16 +554,22 @@ def feedback_window():
 - **Reason**: {h.reason}"""
                 )
 
+            if isinstance(state.scenario, KGScenario):
+                if fbe := state.msgs[round]["ef.runner result"]:
+                    submission_path = fbe[0].content.experiment_workspace.workspace_path / "submission.csv"
+                    st.markdown(
+                        f":green[**Exp Workspace**]: {str(fbe[0].content.experiment_workspace.workspace_path.absolute())}"
+                    )
+                    st.download_button(
+                        label="**Download** submission.csv",
+                        data=submission_path.read_bytes(),
+                        file_name="submission.csv",
+                        mime="text/csv",
+                    )
+
 
-@st.fragment
 def evolving_window():
-    title = (
-        "Development🛠️"
-        if isinstance(
-            state.scenario, (QlibModelScenario, DMModelScenario, QlibFactorScenario, QlibFactorFromReportScenario)
-        )
-        else "Development🛠️ (evolving coder)"
-    )
+    title = "Development🛠️" if isinstance(state.scenario, SIMILAR_SCENARIOS) else "Development🛠️ (evolving coder)"
     st.subheader(title, divider="green", anchor="_development")
 
     # Evolving Status
@@ -739,13 +761,23 @@ with st.container():
             st.markdown(state.scenario.rich_style_description + css, unsafe_allow_html=True)
 
 
+def show_times(round: int):
+    for k, v in state.times[round].items():
+        if len(v) > 1:
+            diff = v[-1] - v[0]
+        else:
+            diff = v[0] - v[0]
+        total_seconds = diff.seconds
+        seconds = total_seconds % 60
+        minutes = total_seconds // 60
+        st.markdown(f"**:blue[{k}]**: :red[**{minutes}**] minutes :orange[**{seconds}**] seconds")
+
+
 if state.scenario is not None:
     summary_window()
 
     # R&D Loops Window
-    if isinstance(
-        state.scenario, (QlibModelScenario, DMModelScenario, QlibFactorScenario, QlibFactorFromReportScenario)
-    ):
+    if isinstance(state.scenario, SIMILAR_SCENARIOS):
         st.header("R&D Loops♾️", divider="rainbow", anchor="_rdloops")
         if len(state.msgs) > 1:
             r_options = list(state.msgs.keys())
@@ -754,8 +786,12 @@ if state.scenario is not None:
             round = st.radio("**Loops**", horizontal=True, options=r_options, index=state.lround - 1)
         else:
             round = 1
+
+        show_times(round)
         rf_c, d_c = st.columns([2, 2])
     elif isinstance(state.scenario, GeneralModelScenario):
+        show_times(round)
+
         rf_c = st.container()
         d_c = st.container()
         round = 1

rdagent/log/ui/web.py

@@ -36,7 +36,7 @@ class WebView(View):
 
     def display(self, s: Storage, watch: bool = False):
         for msg in s.iter_msg():  # iterate overtime
-            # NOTE:  iter_msg will correctly seperate the information.
+            # NOTE:  iter_msg will correctly separate the information.
             # TODO: msg may support streaming mode.
             self.ui.consume_msg(msg)
 
@@ -291,10 +291,6 @@ class WorkspaceWindow(StWindow):
             self.container.markdown(f"`{k}`")
             self.container.code(v, language="python")
 
-        # executed_factor_value_dataframe
-        # if isinstance(ws, FactorFBWorkspace):
-        #     self.container.dataframe(ws.executed_factor_value_dataframe)
-
 
 class QlibFactorExpWindow(StWindow):
     def __init__(self, container: DeltaGenerator, show_task_info: bool = False):
@@ -622,9 +618,11 @@ class TraceWindow(StWindow):
             self.hypothesis_decisions[self.hypotheses[-1]] = msg.content.decision
             self.summary_c.markdown(
                 "\n".join(
-                    f"{id+1}. :green[{self.hypotheses[id].hypothesis}]\n\t>*{self.hypotheses[id].concise_reason}*"
-                    if d
-                    else f"{id+1}. {self.hypotheses[id].hypothesis}\n\t>*{self.hypotheses[id].concise_reason}*"
+                    (
+                        f"{id+1}. :green[{self.hypotheses[id].hypothesis}]\n\t>*{self.hypotheses[id].concise_reason}*"
+                        if d
+                        else f"{id+1}. {self.hypotheses[id].hypothesis}\n\t>*{self.hypotheses[id].concise_reason}*"
+                    )
                     for id, (h, d) in enumerate(self.hypothesis_decisions.items())
                 )
             )

rdagent/oai/llm_conf.py

@@ -0,0 +1,93 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+from pydantic_settings import BaseSettings
+
+
+class LLMSettings(BaseSettings):
+    log_llm_chat_content: bool = True
+
+    use_azure: bool = False
+    use_azure_token_provider: bool = False
+    managed_identity_client_id: str | None = None
+    max_retry: int = 10
+    retry_wait_seconds: int = 1
+    dump_chat_cache: bool = False
+    use_chat_cache: bool = False
+    dump_embedding_cache: bool = False
+    use_embedding_cache: bool = False
+    prompt_cache_path: str = str(Path.cwd() / "prompt_cache.db")
+    max_past_message_include: int = 10
+
+    # Behavior of returning answers to the same question when caching is enabled
+    use_auto_chat_cache_seed_gen: bool = False
+    """
+    `_create_chat_completion_inner_function` provdies a feature to pass in a seed to affect the cache hash key
+    We want to enable a auto seed generator to get different default seed for `_create_chat_completion_inner_function`
+    if seed is not given.
+    So the cache will only not miss you ask the same question on same round.
+    """
+    init_chat_cache_seed: int = 42
+
+    # Chat configs
+    openai_api_key: str = ""  # TODO: simplify the key design.
+    chat_openai_api_key: str = ""
+    chat_azure_api_base: str = ""
+    chat_azure_api_version: str = ""
+    chat_model: str = "gpt-4-turbo"
+    chat_max_tokens: int = 3000
+    chat_temperature: float = 0.5
+    chat_stream: bool = True
+    chat_seed: int | None = None
+    chat_frequency_penalty: float = 0.0
+    chat_presence_penalty: float = 0.0
+    chat_token_limit: int = (
+        100000  # 100000 is the maximum limit of gpt4, which might increase in the future version of gpt
+    )
+    default_system_prompt: str = "You are an AI assistant who helps to answer user's questions."
+
+    # Embedding configs
+    embedding_openai_api_key: str = ""
+    embedding_azure_api_base: str = ""
+    embedding_azure_api_version: str = ""
+    embedding_model: str = ""
+    embedding_max_str_num: int = 50
+
+    # offline llama2 related config
+    use_llama2: bool = False
+    llama2_ckpt_dir: str = "Llama-2-7b-chat"
+    llama2_tokenizer_path: str = "Llama-2-7b-chat/tokenizer.model"
+    llams2_max_batch_size: int = 8
+
+    # server served endpoints
+    use_gcr_endpoint: bool = False
+    gcr_endpoint_type: str = "llama2_70b"  # or "llama3_70b", "phi2", "phi3_4k", "phi3_128k"
+
+    llama2_70b_endpoint: str = ""
+    llama2_70b_endpoint_key: str = ""
+    llama2_70b_endpoint_deployment: str = ""
+
+    llama3_70b_endpoint: str = ""
+    llama3_70b_endpoint_key: str = ""
+    llama3_70b_endpoint_deployment: str = ""
+
+    phi2_endpoint: str = ""
+    phi2_endpoint_key: str = ""
+    phi2_endpoint_deployment: str = ""
+
+    phi3_4k_endpoint: str = ""
+    phi3_4k_endpoint_key: str = ""
+    phi3_4k_endpoint_deployment: str = ""
+
+    phi3_128k_endpoint: str = ""
+    phi3_128k_endpoint_key: str = ""
+    phi3_128k_endpoint_deployment: str = ""
+
+    gcr_endpoint_temperature: float = 0.7
+    gcr_endpoint_top_p: float = 0.9
+    gcr_endpoint_do_sample: bool = False
+    gcr_endpoint_max_token: int = 100
+
+
+LLM_SETTINGS = LLMSettings()

rdagent/oai/llm_utils.py

@@ -1,10 +1,9 @@
 from __future__ import annotations
 
-import datetime
 import hashlib
 import json
-import multiprocessing
 import os
+import random
 import re
 import sqlite3
 import ssl
@@ -13,15 +12,15 @@ import urllib.request
 import uuid
 from copy import deepcopy
 from pathlib import Path
-from typing import Any
+from typing import Any, Optional
 
 import numpy as np
 import tiktoken
 
-from rdagent.core.conf import RD_AGENT_SETTINGS
-from rdagent.core.utils import SingletonBaseClass
+from rdagent.core.utils import LLM_CACHE_SEED_GEN, SingletonBaseClass
 from rdagent.log import LogColors
 from rdagent.log import rdagent_logger as logger
+from rdagent.oai.llm_conf import LLM_SETTINGS
 
 DEFAULT_QLIB_DOT_PATH = Path("./")
 
@@ -90,7 +89,7 @@ class SQliteLazyCache(SingletonBaseClass):
         self.cache_location = cache_location
         db_file_exist = Path(cache_location).exists()
         # TODO: sqlite3 does not support multiprocessing.
-        self.conn = sqlite3.connect(cache_location)
+        self.conn = sqlite3.connect(cache_location, timeout=20)
         self.c = self.conn.cursor()
         if not db_file_exist:
             self.c.execute(
@@ -170,7 +169,7 @@ class SQliteLazyCache(SingletonBaseClass):
 class SessionChatHistoryCache(SingletonBaseClass):
     def __init__(self) -> None:
         """load all history conversation json file from self.session_cache_location"""
-        self.cache = SQliteLazyCache(cache_location=RD_AGENT_SETTINGS.prompt_cache_path)
+        self.cache = SQliteLazyCache(cache_location=LLM_SETTINGS.prompt_cache_path)
 
     def message_get(self, conversation_id: str) -> list[str]:
         return self.cache.message_get(conversation_id)
@@ -182,8 +181,7 @@ class SessionChatHistoryCache(SingletonBaseClass):
 class ChatSession:
     def __init__(self, api_backend: Any, conversation_id: str | None = None, system_prompt: str | None = None) -> None:
         self.conversation_id = str(uuid.uuid4()) if conversation_id is None else conversation_id
-        self.cfg = RD_AGENT_SETTINGS
-        self.system_prompt = system_prompt if system_prompt is not None else self.cfg.default_system_prompt
+        self.system_prompt = system_prompt if system_prompt is not None else LLM_SETTINGS.default_system_prompt
         self.api_backend = api_backend
 
     def build_chat_completion_message(self, user_prompt: str) -> list[dict[str, Any]]:
@@ -235,6 +233,15 @@ class ChatSession:
 
 
 class APIBackend:
+    """
+    This is a unified interface for different backends.
+
+    (xiao) thinks integrate all kinds of API in a single class is not a good design.
+    So we should split them into different classes in `oai/backends/` in the future.
+    """
+
+    # FIXME: (xiao) We should avoid using self.xxxx.
+    # Instead, we can use LLM_SETTINGS directly. If it's difficult to support different backend settings, we can split them into multiple BaseSettings.
     def __init__(  # noqa: C901, PLR0912, PLR0915
         self,
         *,
@@ -251,37 +258,36 @@ class APIBackend:
         use_embedding_cache: bool | None = None,
         dump_embedding_cache: bool | None = None,
     ) -> None:
-        self.cfg = RD_AGENT_SETTINGS
-        if self.cfg.use_llama2:
+        if LLM_SETTINGS.use_llama2:
             self.generator = Llama.build(
-                ckpt_dir=self.cfg.llama2_ckpt_dir,
-                tokenizer_path=self.cfg.llama2_tokenizer_path,
-                max_seq_len=self.cfg.max_tokens,
-                max_batch_size=self.cfg.llams2_max_batch_size,
+                ckpt_dir=LLM_SETTINGS.llama2_ckpt_dir,
+                tokenizer_path=LLM_SETTINGS.llama2_tokenizer_path,
+                max_seq_len=LLM_SETTINGS.max_tokens,
+                max_batch_size=LLM_SETTINGS.llams2_max_batch_size,
             )
             self.encoder = None
-        elif self.cfg.use_gcr_endpoint:
-            gcr_endpoint_type = self.cfg.gcr_endpoint_type
+        elif LLM_SETTINGS.use_gcr_endpoint:
+            gcr_endpoint_type = LLM_SETTINGS.gcr_endpoint_type
             if gcr_endpoint_type == "llama2_70b":
-                self.gcr_endpoint_key = self.cfg.llama2_70b_endpoint_key
-                self.gcr_endpoint_deployment = self.cfg.llama2_70b_endpoint_deployment
-                self.gcr_endpoint = self.cfg.llama2_70b_endpoint
+                self.gcr_endpoint_key = LLM_SETTINGS.llama2_70b_endpoint_key
+                self.gcr_endpoint_deployment = LLM_SETTINGS.llama2_70b_endpoint_deployment
+                self.gcr_endpoint = LLM_SETTINGS.llama2_70b_endpoint
             elif gcr_endpoint_type == "llama3_70b":
-                self.gcr_endpoint_key = self.cfg.llama3_70b_endpoint_key
-                self.gcr_endpoint_deployment = self.cfg.llama3_70b_endpoint_deployment
-                self.gcr_endpoint = self.cfg.llama3_70b_endpoint
+                self.gcr_endpoint_key = LLM_SETTINGS.llama3_70b_endpoint_key
+                self.gcr_endpoint_deployment = LLM_SETTINGS.llama3_70b_endpoint_deployment
+                self.gcr_endpoint = LLM_SETTINGS.llama3_70b_endpoint
             elif gcr_endpoint_type == "phi2":
-                self.gcr_endpoint_key = self.cfg.phi2_endpoint_key
-                self.gcr_endpoint_deployment = self.cfg.phi2_endpoint_deployment
-                self.gcr_endpoint = self.cfg.phi2_endpoint
+                self.gcr_endpoint_key = LLM_SETTINGS.phi2_endpoint_key
+                self.gcr_endpoint_deployment = LLM_SETTINGS.phi2_endpoint_deployment
+                self.gcr_endpoint = LLM_SETTINGS.phi2_endpoint
             elif gcr_endpoint_type == "phi3_4k":
-                self.gcr_endpoint_key = self.cfg.phi3_4k_endpoint_key
-                self.gcr_endpoint_deployment = self.cfg.phi3_4k_endpoint_deployment
-                self.gcr_endpoint = self.cfg.phi3_4k_endpoint
+                self.gcr_endpoint_key = LLM_SETTINGS.phi3_4k_endpoint_key
+                self.gcr_endpoint_deployment = LLM_SETTINGS.phi3_4k_endpoint_deployment
+                self.gcr_endpoint = LLM_SETTINGS.phi3_4k_endpoint
             elif gcr_endpoint_type == "phi3_128k":
-                self.gcr_endpoint_key = self.cfg.phi3_128k_endpoint_key
-                self.gcr_endpoint_deployment = self.cfg.phi3_128k_endpoint_deployment
-                self.gcr_endpoint = self.cfg.phi3_128k_endpoint
+                self.gcr_endpoint_key = LLM_SETTINGS.phi3_128k_endpoint_key
+                self.gcr_endpoint_deployment = LLM_SETTINGS.phi3_128k_endpoint_deployment
+                self.gcr_endpoint = LLM_SETTINGS.phi3_128k_endpoint
             else:
                 error_message = f"Invalid gcr_endpoint_type: {gcr_endpoint_type}"
                 raise ValueError(error_message)
@@ -290,46 +296,48 @@ class APIBackend:
                 "Authorization": ("Bearer " + self.gcr_endpoint_key),
                 "azureml-model-deployment": self.gcr_endpoint_deployment,
             }
-            self.gcr_endpoint_temperature = self.cfg.gcr_endpoint_temperature
-            self.gcr_endpoint_top_p = self.cfg.gcr_endpoint_top_p
-            self.gcr_endpoint_do_sample = self.cfg.gcr_endpoint_do_sample
-            self.gcr_endpoint_max_token = self.cfg.gcr_endpoint_max_token
+            self.gcr_endpoint_temperature = LLM_SETTINGS.gcr_endpoint_temperature
+            self.gcr_endpoint_top_p = LLM_SETTINGS.gcr_endpoint_top_p
+            self.gcr_endpoint_do_sample = LLM_SETTINGS.gcr_endpoint_do_sample
+            self.gcr_endpoint_max_token = LLM_SETTINGS.gcr_endpoint_max_token
             if not os.environ.get("PYTHONHTTPSVERIFY", "") and hasattr(ssl, "_create_unverified_context"):
                 ssl._create_default_https_context = ssl._create_unverified_context  # noqa: SLF001
             self.encoder = None
         else:
-            self.use_azure = self.cfg.use_azure
-            self.use_azure_token_provider = self.cfg.use_azure_token_provider
-            self.managed_identity_client_id = self.cfg.managed_identity_client_id
+            self.use_azure = LLM_SETTINGS.use_azure
+            self.use_azure_token_provider = LLM_SETTINGS.use_azure_token_provider
+            self.managed_identity_client_id = LLM_SETTINGS.managed_identity_client_id
 
             # Priority: chat_api_key/embedding_api_key > openai_api_key > os.environ.get("OPENAI_API_KEY")
             # TODO: Simplify the key design. Consider Pandatic's field alias & priority.
             self.chat_api_key = (
                 chat_api_key
-                or self.cfg.chat_openai_api_key
-                or self.cfg.openai_api_key
+                or LLM_SETTINGS.chat_openai_api_key
+                or LLM_SETTINGS.openai_api_key
                 or os.environ.get("OPENAI_API_KEY")
             )
             self.embedding_api_key = (
                 embedding_api_key
-                or self.cfg.embedding_openai_api_key
-                or self.cfg.openai_api_key
+                or LLM_SETTINGS.embedding_openai_api_key
+                or LLM_SETTINGS.openai_api_key
                 or os.environ.get("OPENAI_API_KEY")
             )
 
-            self.chat_model = self.cfg.chat_model if chat_model is None else chat_model
+            self.chat_model = LLM_SETTINGS.chat_model if chat_model is None else chat_model
             self.encoder = tiktoken.encoding_for_model(self.chat_model)
-            self.chat_api_base = self.cfg.chat_azure_api_base if chat_api_base is None else chat_api_base
-            self.chat_api_version = self.cfg.chat_azure_api_version if chat_api_version is None else chat_api_version
-            self.chat_stream = self.cfg.chat_stream
-            self.chat_seed = self.cfg.chat_seed
+            self.chat_api_base = LLM_SETTINGS.chat_azure_api_base if chat_api_base is None else chat_api_base
+            self.chat_api_version = (
+                LLM_SETTINGS.chat_azure_api_version if chat_api_version is None else chat_api_version
+            )
+            self.chat_stream = LLM_SETTINGS.chat_stream
+            self.chat_seed = LLM_SETTINGS.chat_seed
 
-            self.embedding_model = self.cfg.embedding_model if embedding_model is None else embedding_model
+            self.embedding_model = LLM_SETTINGS.embedding_model if embedding_model is None else embedding_model
             self.embedding_api_base = (
-                self.cfg.embedding_azure_api_base if embedding_api_base is None else embedding_api_base
+                LLM_SETTINGS.embedding_azure_api_base if embedding_api_base is None else embedding_api_base
             )
             self.embedding_api_version = (
-                self.cfg.embedding_azure_api_version if embedding_api_version is None else embedding_api_version
+                LLM_SETTINGS.embedding_azure_api_version if embedding_api_version is None else embedding_api_version
             )
 
             if self.use_azure:
@@ -367,20 +375,22 @@ class APIBackend:
                 self.chat_client = openai.OpenAI(api_key=self.chat_api_key)
                 self.embedding_client = openai.OpenAI(api_key=self.embedding_api_key)
 
-        self.dump_chat_cache = self.cfg.dump_chat_cache if dump_chat_cache is None else dump_chat_cache
-        self.use_chat_cache = self.cfg.use_chat_cache if use_chat_cache is None else use_chat_cache
+        self.dump_chat_cache = LLM_SETTINGS.dump_chat_cache if dump_chat_cache is None else dump_chat_cache
+        self.use_chat_cache = LLM_SETTINGS.use_chat_cache if use_chat_cache is None else use_chat_cache
         self.dump_embedding_cache = (
-            self.cfg.dump_embedding_cache if dump_embedding_cache is None else dump_embedding_cache
+            LLM_SETTINGS.dump_embedding_cache if dump_embedding_cache is None else dump_embedding_cache
+        )
+        self.use_embedding_cache = (
+            LLM_SETTINGS.use_embedding_cache if use_embedding_cache is None else use_embedding_cache
         )
-        self.use_embedding_cache = self.cfg.use_embedding_cache if use_embedding_cache is None else use_embedding_cache
         if self.dump_chat_cache or self.use_chat_cache or self.dump_embedding_cache or self.use_embedding_cache:
-            self.cache_file_location = self.cfg.prompt_cache_path
+            self.cache_file_location = LLM_SETTINGS.prompt_cache_path
             self.cache = SQliteLazyCache(cache_location=self.cache_file_location)
 
         # transfer the config to the class if the config is not supposed to change during the runtime
-        self.use_llama2 = self.cfg.use_llama2
-        self.use_gcr_endpoint = self.cfg.use_gcr_endpoint
-        self.retry_wait_seconds = self.cfg.retry_wait_seconds
+        self.use_llama2 = LLM_SETTINGS.use_llama2
+        self.use_gcr_endpoint = LLM_SETTINGS.use_gcr_endpoint
+        self.retry_wait_seconds = LLM_SETTINGS.retry_wait_seconds
 
     def build_chat_session(
         self,
@@ -401,7 +411,10 @@ class APIBackend:
         *,
         shrink_multiple_break: bool = False,
     ) -> list[dict]:
-        """build the messages to avoid implementing several redundant lines of code"""
+        """
+        build the messages to avoid implementing several redundant lines of code
+
+        """
         if former_messages is None:
             former_messages = []
         # shrink multiple break will recursively remove multiple breaks(more than 2)
@@ -411,14 +424,14 @@ class APIBackend:
             if system_prompt is not None:
                 while "\n\n\n" in system_prompt:
                     system_prompt = system_prompt.replace("\n\n\n", "\n\n")
-        system_prompt = self.cfg.default_system_prompt if system_prompt is None else system_prompt
+        system_prompt = LLM_SETTINGS.default_system_prompt if system_prompt is None else system_prompt
         messages = [
             {
                 "role": "system",
                 "content": system_prompt,
             },
         ]
-        messages.extend(former_messages[-1 * self.cfg.max_past_message_include :])
+        messages.extend(former_messages[-1 * LLM_SETTINGS.max_past_message_include :])
         messages.append(
             {
                 "role": "user",
@@ -440,7 +453,10 @@ class APIBackend:
         if former_messages is None:
             former_messages = []
         messages = self.build_messages(
-            user_prompt, system_prompt, former_messages, shrink_multiple_break=shrink_multiple_break
+            user_prompt,
+            system_prompt,
+            former_messages,
+            shrink_multiple_break=shrink_multiple_break,
         )
         return self._try_create_chat_completion_or_embedding(
             messages=messages,
@@ -489,7 +505,7 @@ class APIBackend:
         **kwargs: Any,
     ) -> Any:
         assert not (chat_completion and embedding), "chat_completion and embedding cannot be True at the same time"
-        max_retry = self.cfg.max_retry if self.cfg.max_retry is not None else max_retry
+        max_retry = LLM_SETTINGS.max_retry if LLM_SETTINGS.max_retry is not None else max_retry
         for i in range(max_retry):
             try:
                 if embedding:
@@ -528,21 +544,25 @@ class APIBackend:
             filtered_input_content_list = input_content_list
 
         if len(filtered_input_content_list) > 0:
-            if self.use_azure:
-                response = self.embedding_client.embeddings.create(
-                    model=self.embedding_model,
-                    input=filtered_input_content_list,
-                )
-            else:
-                response = self.embedding_client.embeddings.create(
-                    model=self.embedding_model,
-                    input=filtered_input_content_list,
-                )
-            for index, data in enumerate(response.data):
-                content_to_embedding_dict[filtered_input_content_list[index]] = data.embedding
+            for sliced_filtered_input_content_list in [
+                filtered_input_content_list[i : i + LLM_SETTINGS.embedding_max_str_num]
+                for i in range(0, len(filtered_input_content_list), LLM_SETTINGS.embedding_max_str_num)
+            ]:
+                if self.use_azure:
+                    response = self.embedding_client.embeddings.create(
+                        model=self.embedding_model,
+                        input=sliced_filtered_input_content_list,
+                    )
+                else:
+                    response = self.embedding_client.embeddings.create(
+                        model=self.embedding_model,
+                        input=sliced_filtered_input_content_list,
+                    )
+                for index, data in enumerate(response.data):
+                    content_to_embedding_dict[sliced_filtered_input_content_list[index]] = data.embedding
 
-            if self.dump_embedding_cache:
-                self.cache.embedding_set(content_to_embedding_dict)
+                if self.dump_embedding_cache:
+                    self.cache.embedding_set(content_to_embedding_dict)
         return [content_to_embedding_dict[content] for content in input_content_list]
 
     def _build_log_messages(self, messages: list[dict]) -> str:
@@ -567,30 +587,40 @@ class APIBackend:
         *,
         json_mode: bool = False,
         add_json_in_prompt: bool = False,
+        seed: Optional[int] = None,
     ) -> str:
+        """
+        seed : Optional[int]
+            When retrying with cache enabled, it will keep returning the same results.
+            To make retries useful, we need to enable a seed.
+            This seed is different from `self.chat_seed` for GPT. It is for the local cache mechanism enabled by RD-Agent locally.
+        """
+        if seed is None and LLM_SETTINGS.use_auto_chat_cache_seed_gen:
+            seed = LLM_CACHE_SEED_GEN.get_next_seed()
+
         # TODO: we can add this function back to avoid so much `self.cfg.log_llm_chat_content`
-        if self.cfg.log_llm_chat_content:
+        if LLM_SETTINGS.log_llm_chat_content:
             logger.info(self._build_log_messages(messages), tag="llm_messages")
         # TODO: fail to use loguru adaptor due to stream response
         input_content_json = json.dumps(messages)
         input_content_json = (
-            chat_cache_prefix + input_content_json
+            chat_cache_prefix + input_content_json + f"<seed={seed}/>"
         )  # FIXME this is a hack to make sure the cache represents the round index
         if self.use_chat_cache:
             cache_result = self.cache.chat_get(input_content_json)
             if cache_result is not None:
-                if self.cfg.log_llm_chat_content:
+                if LLM_SETTINGS.log_llm_chat_content:
                     logger.info(f"{LogColors.CYAN}Response:{cache_result}{LogColors.END}", tag="llm_messages")
                 return cache_result, None
 
         if temperature is None:
-            temperature = self.cfg.chat_temperature
+            temperature = LLM_SETTINGS.chat_temperature
         if max_tokens is None:
-            max_tokens = self.cfg.chat_max_tokens
+            max_tokens = LLM_SETTINGS.chat_max_tokens
         if frequency_penalty is None:
-            frequency_penalty = self.cfg.chat_frequency_penalty
+            frequency_penalty = LLM_SETTINGS.chat_frequency_penalty
         if presence_penalty is None:
-            presence_penalty = self.cfg.chat_presence_penalty
+            presence_penalty = LLM_SETTINGS.chat_presence_penalty
 
         finish_reason = None
         if self.use_llama2:
@@ -600,7 +630,7 @@ class APIBackend:
                 temperature=temperature,
             )
             resp = response[0]["generation"]["content"]
-            if self.cfg.log_llm_chat_content:
+            if LLM_SETTINGS.log_llm_chat_content:
                 logger.info(f"{LogColors.CYAN}Response:{resp}{LogColors.END}", tag="llm_messages")
         elif self.use_gcr_endpoint:
             body = str.encode(
@@ -622,7 +652,7 @@ class APIBackend:
             req = urllib.request.Request(self.gcr_endpoint, body, self.headers)  # noqa: S310
             response = urllib.request.urlopen(req)  # noqa: S310
             resp = json.loads(response.read().decode())["output"]
-            if self.cfg.log_llm_chat_content:
+            if LLM_SETTINGS.log_llm_chat_content:
                 logger.info(f"{LogColors.CYAN}Response:{resp}{LogColors.END}", tag="llm_messages")
         else:
             kwargs = dict(
@@ -647,7 +677,7 @@ class APIBackend:
             if self.chat_stream:
                 resp = ""
                 # TODO: with logger.config(stream=self.chat_stream): and add a `stream_start` flag to add timestamp for first message.
-                if self.cfg.log_llm_chat_content:
+                if LLM_SETTINGS.log_llm_chat_content:
                     logger.info(f"{LogColors.CYAN}Response:{LogColors.END}", tag="llm_messages")
 
                 for chunk in response:
@@ -656,19 +686,19 @@ class APIBackend:
                         if len(chunk.choices) > 0 and chunk.choices[0].delta.content is not None
                         else ""
                     )
-                    if self.cfg.log_llm_chat_content:
+                    if LLM_SETTINGS.log_llm_chat_content:
                         logger.info(LogColors.CYAN + content + LogColors.END, raw=True, tag="llm_messages")
                     resp += content
                     if len(chunk.choices) > 0 and chunk.choices[0].finish_reason is not None:
                         finish_reason = chunk.choices[0].finish_reason
 
-                if self.cfg.log_llm_chat_content:
+                if LLM_SETTINGS.log_llm_chat_content:
                     logger.info("\n", raw=True, tag="llm_messages")
 
             else:
                 resp = response.choices[0].message.content
                 finish_reason = response.choices[0].finish_reason
-                if self.cfg.log_llm_chat_content:
+                if LLM_SETTINGS.log_llm_chat_content:
                     logger.info(f"{LogColors.CYAN}Response:{resp}{LogColors.END}", tag="llm_messages")
             if json_mode:
                 json.loads(resp)
@@ -713,26 +743,6 @@ class APIBackend:
         return self.calculate_token_from_messages(messages)
 
 
-def calculate_embedding_process(str_list: list) -> list:
-    return APIBackend().create_embedding(str_list)
-
-
-def create_embedding_with_multiprocessing(str_list: list, slice_count: int = 50, nproc: int = 8) -> list:
-    embeddings = []
-
-    pool = multiprocessing.Pool(nproc)
-    result_list = [
-        pool.apply_async(calculate_embedding_process, (str_list[index : index + slice_count],))
-        for index in range(0, len(str_list), slice_count)
-    ]
-    pool.close()
-    pool.join()
-
-    for res in result_list:
-        embeddings.extend(res.get())
-    return embeddings
-
-
 def calculate_embedding_distance_between_str_list(
     source_str_list: list[str],
     target_str_list: list[str],
@@ -740,7 +750,8 @@ def calculate_embedding_distance_between_str_list(
     if not source_str_list or not target_str_list:
         return [[]]
 
-    embeddings = create_embedding_with_multiprocessing(source_str_list + target_str_list, slice_count=50, nproc=8)
+    embeddings = APIBackend().create_embedding(source_str_list + target_str_list)
+
     source_embeddings = embeddings[: len(source_str_list)]
     target_embeddings = embeddings[len(source_str_list) :]
 

rdagent/scenarios/data_mining/developer/model_runner.py

@@ -1,27 +1,12 @@
-import shutil
-import uuid
-from pathlib import Path
-
-import pandas as pd
-
-from rdagent.components.coder.model_coder.model import ModelExperiment, ModelFBWorkspace
 from rdagent.components.runner import CachedRunner
-from rdagent.components.runner.conf import RUNNER_SETTINGS
-from rdagent.core.developer import Developer
 from rdagent.core.exception import ModelEmptyError
-from rdagent.log import rdagent_logger as logger
+from rdagent.core.utils import cache_with_pickle
 from rdagent.scenarios.data_mining.experiment.model_experiment import DMModelExperiment
-from rdagent.utils.env import DMDockerEnv
 
 
 class DMModelRunner(CachedRunner[DMModelExperiment]):
+    @cache_with_pickle(CachedRunner.get_cache_key, CachedRunner.assign_cached_result)
     def develop(self, exp: DMModelExperiment) -> DMModelExperiment:
-        if RUNNER_SETTINGS.cache_result:
-            cache_hit, result = self.get_cache_result(exp)
-            if cache_hit:
-                exp.result = result
-                return exp
-
         if exp.sub_workspace_list[0].code_dict.get("model.py") is None:
             raise ModelEmptyError("model.py is empty")
         # to replace & inject code
@@ -32,7 +17,5 @@ class DMModelRunner(CachedRunner[DMModelExperiment]):
         result = exp.experiment_workspace.execute(run_env=env_to_use)
 
         exp.result = result
-        if RUNNER_SETTINGS.cache_result:
-            self.dump_cache_result(exp, result)
 
         return exp

rdagent/scenarios/data_mining/docker/Dockerfile

@@ -1,4 +1,4 @@
-FROM pytorch/pytorch:latest
+FROM pytorch/pytorch:2.2.1-cuda12.1-cudnn8-runtime
 # For GPU support, please choose the proper tag from https://hub.docker.com/r/pytorch/pytorch/tags
 
 RUN apt-get clean && apt-get update && apt-get install -y \  

rdagent/scenarios/data_mining/experiment/model_experiment.py

@@ -5,6 +5,7 @@ from rdagent.components.coder.model_coder.model import (
     ModelFBWorkspace,
     ModelTask,
 )
+from rdagent.core.experiment import Task
 from rdagent.core.prompts import Prompts
 from rdagent.core.scenario import Scenario
 from rdagent.scenarios.data_mining.experiment.workspace import DMFBWorkspace
@@ -62,7 +63,7 @@ The demo showcases the iterative process of hypothesis generation, knowledge con
  
 To demonstrate the dynamic evolution of models through the R&D loop, emphasizing how each iteration enhances the model performance and reliability. The performane is measured by the AUROC score (Area Under the Receiver Operating Characteristic), which is a commonly used metric for binary classification.   """
 
-    def get_scenario_all_desc(self) -> str:
+    def get_scenario_all_desc(self, task: Task | None = None) -> str:
         return f"""Background of the scenario:
 {self.background}
 The interface you should follow to write the runnable code:

rdagent/scenarios/data_mining/experiment/workspace.py

@@ -2,7 +2,7 @@ from pathlib import Path
 
 import pandas as pd
 
-from rdagent.app.data_mining.conf import PROP_SETTING
+from rdagent.app.data_mining.conf import MED_PROP_SETTING
 from rdagent.core.experiment import FBWorkspace
 from rdagent.log import rdagent_logger as logger
 from rdagent.utils.env import DMDockerEnv
@@ -15,7 +15,7 @@ class DMFBWorkspace(FBWorkspace):
 
     def execute(self, run_env: dict = {}, *args, **kwargs) -> str:
         qtde = DMDockerEnv()
-        qtde.prepare(PROP_SETTING.username, PROP_SETTING.password)
+        qtde.prepare(MED_PROP_SETTING.username, MED_PROP_SETTING.password)
 
         execute_log = qtde.run(
             local_path=str(self.workspace_path),

rdagent/scenarios/data_mining/proposal/model_proposal.py

@@ -35,14 +35,18 @@ class DMModelHypothesisGen(ModelHypothesisGen):
         super().__init__(scen)
 
     def prepare_context(self, trace: Trace) -> Tuple[dict, bool]:
-        hypothesis_feedback = (
-            Environment(undefined=StrictUndefined)
-            .from_string(prompt_dict["hypothesis_and_feedback"])
-            .render(trace=trace)
+        hypothesis_and_feedback = (
+            (
+                Environment(undefined=StrictUndefined)
+                .from_string(prompt_dict["hypothesis_and_feedback"])
+                .render(trace=trace)
+            )
+            if len(trace.hist) > 0
+            else "No previous hypothesis and feedback available since it's the first round."
         )
         context_dict = {
-            "hypothesis_and_feedback": hypothesis_feedback,
-            "RAG": "",
+            "hypothesis_and_feedback": hypothesis_and_feedback,
+            "RAG": None,
             "hypothesis_output_format": prompt_dict["hypothesis_output_format"],
             "hypothesis_specification": prompt_dict["model_hypothesis_specification"],
         }
@@ -67,9 +71,13 @@ class DMModelHypothesis2Experiment(ModelHypothesis2Experiment):
         experiment_output_format = prompt_dict["model_experiment_output_format"]
 
         hypothesis_and_feedback = (
-            Environment(undefined=StrictUndefined)
-            .from_string(prompt_dict["hypothesis_and_feedback"])
-            .render(trace=trace)
+            (
+                Environment(undefined=StrictUndefined)
+                .from_string(prompt_dict["hypothesis_and_feedback"])
+                .render(trace=trace)
+            )
+            if len(trace.hist) > 0
+            else "No previous hypothesis and feedback available since it's the first round."
         )
 
         experiment_list: List[ModelExperiment] = [t[1] for t in trace.hist]
@@ -84,7 +92,7 @@ class DMModelHypothesis2Experiment(ModelHypothesis2Experiment):
             "hypothesis_and_feedback": hypothesis_and_feedback,
             "experiment_output_format": experiment_output_format,
             "target_list": model_list,
-            "RAG": ...,
+            "RAG": None,
         }, True
 
     def convert_response(self, response: str, trace: Trace) -> ModelExperiment:
@@ -98,7 +106,15 @@ class DMModelHypothesis2Experiment(ModelHypothesis2Experiment):
             hyperparameters = response_dict[model_name]["hyperparameters"]
             model_type = response_dict[model_name]["model_type"]
             tasks.append(
-                ModelTask(model_name, description, formulation, architecture, variables, hyperparameters, model_type)
+                ModelTask(
+                    name=model_name,
+                    description=description,
+                    formulation=formulation,
+                    architecture=architecture,
+                    variables=variables,
+                    hyperparameters=hyperparameters,
+                    model_type=model_type,
+                )
             )
         exp = DMModelExperiment(tasks)
         exp.based_experiments = [t[1] for t in trace.hist if t[2]]

rdagent/scenarios/general_model/scenario.py

@@ -1,6 +1,7 @@
 from copy import deepcopy
 from pathlib import Path
 
+from rdagent.core.experiment import Task
 from rdagent.core.prompts import Prompts
 from rdagent.core.scenario import Scenario
 
@@ -40,7 +41,7 @@ class GeneralModelScenario(Scenario):
     def rich_style_description(self) -> str:
         return self._rich_style_description
 
-    def get_scenario_all_desc(self) -> str:
+    def get_scenario_all_desc(self, task: Task | None = None) -> str:
         return f"""Background of the scenario:
 {self.background}
 The interface you should follow to write the runnable code:

rdagent/scenarios/kaggle/developer/coder.py

@@ -0,0 +1,72 @@
+import json
+from pathlib import Path
+
+from jinja2 import Environment, StrictUndefined
+
+from rdagent.components.coder.factor_coder.CoSTEER import FactorCoSTEER
+from rdagent.components.coder.model_coder.CoSTEER import ModelCoSTEER
+from rdagent.core.developer import Developer
+from rdagent.core.prompts import Prompts
+from rdagent.oai.llm_utils import APIBackend
+from rdagent.scenarios.kaggle.experiment.kaggle_experiment import (
+    KG_SELECT_MAPPING,
+    KGModelExperiment,
+)
+
+KGModelCoSTEER = ModelCoSTEER
+KGFactorCoSTEER = FactorCoSTEER
+
+prompt_dict = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")
+
+DEFAULT_SELECTION_CODE = """
+import pandas as pd
+def select(X: pd.DataFrame) -> pd.DataFrame:
+    \"""
+    Select relevant features. To be used in fit & predict function.
+    \"""
+    if X.columns.nlevels == 1:
+        return X
+    {% if feature_index_list is not none %}
+    X = X.loc[:, X.columns.levels[0][{{feature_index_list}}].tolist()]
+    {% endif %}
+    X.columns = ["_".join(str(i) for i in col).strip() for col in X.columns.values]
+    return X
+"""
+
+
+class KGModelFeatureSelectionCoder(Developer[KGModelExperiment]):
+    def develop(self, exp: KGModelExperiment) -> KGModelExperiment:
+        target_model_type = exp.sub_tasks[0].model_type
+        assert target_model_type in KG_SELECT_MAPPING
+        if len(exp.experiment_workspace.data_description) == 1:
+            code = (
+                Environment(undefined=StrictUndefined)
+                .from_string(DEFAULT_SELECTION_CODE)
+                .render(feature_index_list=None)
+            )
+        else:
+            system_prompt = (
+                Environment(undefined=StrictUndefined)
+                .from_string(prompt_dict["model_feature_selection"]["system"])
+                .render(scenario=self.scen.get_scenario_all_desc(), model_type=exp.sub_tasks[0].model_type)
+            )
+            user_prompt = (
+                Environment(undefined=StrictUndefined)
+                .from_string(prompt_dict["model_feature_selection"]["user"])
+                .render(feature_groups=[desc[0] for desc in exp.experiment_workspace.data_description])
+            )
+
+            chosen_index = json.loads(
+                APIBackend().build_messages_and_create_chat_completion(
+                    user_prompt=user_prompt, system_prompt=system_prompt, json_mode=True
+                )
+            ).get("Selected Group Index", [i + 1 for i in range(len(exp.experiment_workspace.data_description))])
+            chosen_index_to_list_index = [i - 1 for i in chosen_index]
+
+            code = (
+                Environment(undefined=StrictUndefined)
+                .from_string(DEFAULT_SELECTION_CODE)
+                .render(feature_index_list=chosen_index_to_list_index)
+            )
+        exp.experiment_workspace.inject_code(**{KG_SELECT_MAPPING[target_model_type]: code})
+        return exp

rdagent/scenarios/kaggle/developer/feedback.py

@@ -1,6 +1,7 @@
 import json
 from pathlib import Path
 
+import pandas as pd
 from jinja2 import Environment, StrictUndefined
 
 from rdagent.core.experiment import Experiment
@@ -15,54 +16,174 @@ from rdagent.log import rdagent_logger as logger
 from rdagent.oai.llm_utils import APIBackend
 from rdagent.utils import convert2bool
 
-feedback_prompts = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")
+prompt_dict = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")
 DIRNAME = Path(__file__).absolute().resolve().parent
 
 
-class KGModelHypothesisExperiment2Feedback(HypothesisExperiment2Feedback):
-    """Generated feedbacks on the hypothesis from **Executed** Implementations of different tasks & their comparisons with previous performances"""
+class KGHypothesisExperiment2Feedback(HypothesisExperiment2Feedback):
+    def process_results(self, current_result, sota_result):
+        # Convert the results to dataframes
+        current_df = pd.DataFrame(current_result)
+        sota_df = pd.DataFrame(sota_result)
+
+        # Combine the dataframes on the Metric index
+        combined_df = pd.concat([current_df, sota_df], axis=1)
+        combined_df.columns = ["current_df", "sota_df"]
+
+        # combined_df["the largest"] = combined_df.apply(
+        #     lambda row: "sota_df"
+        #     if row["sota_df"] > row["current_df"]
+        #     else ("Equal" if row["sota_df"] == row["current_df"] else "current_df"),
+        #     axis=1,
+        # )
+
+        # Add a note about metric direction
+        evaluation_direction = "higher" if self.scen.evaluation_metric_direction else "lower"
+        evaluation_description = f"Direction of improvement (higher/lower is better) should be judged per metric. Here '{evaluation_direction}' is better for the metrics."
+        combined_df["Note"] = evaluation_description
+
+        return combined_df, evaluation_description
 
     def generate_feedback(self, exp: Experiment, hypothesis: Hypothesis, trace: Trace) -> HypothesisFeedback:
         """
         The `ti` should be executed and the results should be included, as well as the comparison between previous results (done by LLM).
         For example: `mlflow` of Qlib will be included.
         """
-
+        """
+        Generate feedback for the given experiment and hypothesis.
+        Args:
+            exp: The experiment to generate feedback for.
+            hypothesis: The hypothesis to generate feedback for.
+            trace: The trace of the experiment.
+        Returns:
+            Any: The feedback generated for the given experiment and hypothesis.
+        """
         logger.info("Generating feedback...")
-        # Define the system prompt for hypothesis feedback
-        system_prompt = feedback_prompts["model_feedback_generation"]["system"]
+        hypothesis_text = hypothesis.hypothesis
+        current_result = exp.result
+        tasks_factors = []
+        if exp.sub_tasks:
+            tasks_factors = []
+            for task in exp.sub_tasks:
+                try:
+                    task_info = task.get_task_information_and_implementation_result()
+                    tasks_factors.append(task_info)
+                except AttributeError:
+                    print(f"Warning: Task {task} does not have get_task_information_and_implementation_result method")
 
-        # Define the user prompt for hypothesis feedback
-        context = trace.scen
-        SOTA_hypothesis, SOTA_experiment = trace.get_sota_hypothesis_and_experiment()
+        evaluation_description = None
+        # Check if there are any based experiments
+        if exp.based_experiments:
+            sota_result = exp.based_experiments[-1].result
+            # Process the results to filter important metrics
+            combined_result, evaluation_description = self.process_results(current_result, sota_result)
+        else:
+            # If there are no based experiments, we'll only use the current result
+            combined_result, evaluation_description = self.process_results(
+                current_result, current_result
+            )  # Compare with itself
+            print("Warning: No previous experiments to compare against. Using current result as baseline.")
 
-        user_prompt = (
+        available_features = {
+            task_info: feature_shape for task_info, feature_shape in exp.experiment_workspace.data_description
+        }
+        model_code = exp.experiment_workspace.model_description
+
+        # Generate the user prompt based on the action type
+        if hypothesis.action == "Model tuning":
+            prompt_key = "model_tuning_feedback_generation"
+        elif hypothesis.action == "Model feature selection":
+            prompt_key = "feature_selection_feedback_generation"
+        else:
+            prompt_key = "factor_feedback_generation"
+
+        # Generate the system prompt
+        sys_prompt = (
             Environment(undefined=StrictUndefined)
-            .from_string(feedback_prompts["model_feedback_generation"]["user"])
-            .render(
-                context=context,
-                last_hypothesis=SOTA_hypothesis,
-                last_task=SOTA_experiment.sub_tasks[0].get_task_information() if SOTA_hypothesis else None,
-                last_code=SOTA_experiment.sub_workspace_list[0].code_dict.get("model.py") if SOTA_hypothesis else None,
-                last_result=SOTA_experiment.result if SOTA_hypothesis else None,
-                hypothesis=hypothesis,
-                exp=exp,
-            )
+            .from_string(prompt_dict[prompt_key]["system"])
+            .render(scenario=self.scen.get_scenario_all_desc())
         )
 
-        # Call the APIBackend to generate the response for hypothesis feedback
-        response_hypothesis = APIBackend().build_messages_and_create_chat_completion(
-            user_prompt=user_prompt,
-            system_prompt=system_prompt,
+        last_task_and_code = None
+        if trace.hist:
+            last_task_and_code = (
+                trace.hist[-1][1].experiment_workspace.data_description
+                if trace.hist[-1][0].action == "Feature engineering" or trace.hist[-1][0].action == "Feature processing"
+                else trace.hist[-1][1].experiment_workspace.model_description
+            )
+
+        # Prepare render dictionary
+        render_dict = {
+            "last_hypothesis": trace.hist[-1][0] if trace.hist else None,
+            "last_task_and_code": last_task_and_code,
+            "last_result": trace.hist[-1][1].result if trace.hist else None,
+            "sota_task_and_code": (
+                exp.based_experiments[-1].experiment_workspace.data_description if exp.based_experiments else None
+            ),
+            "sota_result": exp.based_experiments[-1].result if exp.based_experiments else None,
+            "hypothesis": hypothesis,
+            "exp": exp,
+            "model_code": model_code,  # This turn
+            "available_features": available_features,  # This turn
+            "combined_result": combined_result,  # This turn and sota
+            "hypothesis_text": hypothesis_text,  # This turn
+            "task_details": tasks_factors,  # This turn
+            "evaluation_description": evaluation_description,
+        }
+
+        usr_prompt = (
+            Environment(undefined=StrictUndefined).from_string(prompt_dict[prompt_key]["user"]).render(**render_dict)
+        )
+
+        response = APIBackend().build_messages_and_create_chat_completion(
+            user_prompt=usr_prompt,
+            system_prompt=sys_prompt,
             json_mode=True,
         )
 
-        # Parse the JSON response to extract the feedback
-        response_json_hypothesis = json.loads(response_hypothesis)
+        response_json = json.loads(response)
+
+        observations = response_json.get("Observations", "No observations provided")
+        hypothesis_evaluation = response_json.get("Feedback for Hypothesis", "No feedback provided")
+        new_hypothesis = response_json.get("New Hypothesis", "No new hypothesis provided")
+        reason = response_json.get("Reasoning", "No reasoning provided")
+        decision = convert2bool(response_json.get("Replace Best Result", "no"))
+        leaderboard = self.scen.leaderboard
+        current_score = current_result.iloc[0]
+        sorted_scores = sorted(leaderboard, reverse=True)
+        import bisect
+
+        if self.scen.evaluation_metric_direction:
+            insert_position = bisect.bisect_right([-score for score in sorted_scores], -current_score)
+        else:
+            insert_position = bisect.bisect_left(sorted_scores, current_score, lo=0, hi=len(sorted_scores))
+        percentile_ranking = (insert_position) / (len(sorted_scores)) * 100
+
+        experiment_feedback = {
+            "current_competition": self.scen.get_competition_full_desc(),
+            "hypothesis_text": hypothesis_text,
+            "current_result": current_result,
+            "model_code": model_code,
+            "available_features": available_features,
+            "observations": observations,
+            "hypothesis_evaluation": hypothesis_evaluation,
+            "reason": reason,
+            "percentile_ranking": percentile_ranking,
+        }
+
+        if self.scen.if_using_vector_rag:
+            self.scen.vector_base.add_experience_to_vector_base(experiment_feedback)
+            self.scen.vector_base.dump()
+        elif self.scen.if_using_graph_rag:
+            trace.knowledge_base.add_document(experiment_feedback, self.scen)
+
+        if self.scen.if_action_choosing_based_on_UCB:
+            self.scen.action_counts[hypothesis.action] += 1
+
         return HypothesisFeedback(
-            observations=response_json_hypothesis.get("Observations", "No observations provided"),
-            hypothesis_evaluation=response_json_hypothesis.get("Feedback for Hypothesis", "No feedback provided"),
-            new_hypothesis=response_json_hypothesis.get("New Hypothesis", "No new hypothesis provided"),
-            reason=response_json_hypothesis.get("Reasoning", "No reasoning provided"),
-            decision=convert2bool(response_json_hypothesis.get("Decision", "false")),
+            observations=observations,
+            hypothesis_evaluation=hypothesis_evaluation,
+            new_hypothesis=new_hypothesis,
+            reason=reason,
+            decision=decision,
         )

rdagent/scenarios/kaggle/developer/model_coder.py

@@ -1,3 +0,0 @@
-from rdagent.components.coder.model_coder.CoSTEER import ModelCoSTEER
-
-KGModelCoSTEER = ModelCoSTEER

rdagent/scenarios/kaggle/developer/model_runner.py

@@ -1,38 +0,0 @@
-import shutil
-import uuid
-from pathlib import Path
-
-import pandas as pd
-
-from rdagent.components.coder.model_coder.model import ModelExperiment, ModelFBWorkspace
-from rdagent.components.runner import CachedRunner
-from rdagent.components.runner.conf import RUNNER_SETTINGS
-from rdagent.core.developer import Developer
-from rdagent.core.exception import ModelEmptyError
-from rdagent.log import rdagent_logger as logger
-from rdagent.scenarios.kaggle.experiment.model_experiment import KGModelExperiment
-from rdagent.utils.env import KGDockerEnv
-
-
-class KGModelRunner(CachedRunner[KGModelExperiment]):
-    def develop(self, exp: KGModelExperiment) -> KGModelExperiment:
-        if RUNNER_SETTINGS.cache_result:
-            cache_hit, result = self.get_cache_result(exp)
-            if cache_hit:
-                exp.result = result
-                return exp
-
-        if exp.sub_workspace_list[0].code_dict.get("model.py") is None:
-            raise ModelEmptyError("model.py is empty")
-        # to replace & inject code
-        exp.experiment_workspace.inject_code(**{"model.py": exp.sub_workspace_list[0].code_dict["model.py"]})
-
-        env_to_use = {"PYTHONPATH": "./"}
-
-        result = exp.experiment_workspace.execute(run_env=env_to_use)
-
-        exp.result = result
-        if RUNNER_SETTINGS.cache_result:
-            self.dump_cache_result(exp, result)
-
-        return exp

rdagent/scenarios/kaggle/developer/runner.py

@@ -0,0 +1,112 @@
+import json
+import pickle
+import shutil
+from pathlib import Path
+
+from rdagent.components.runner import CachedRunner
+from rdagent.core.exception import CoderError, FactorEmptyError, ModelEmptyError
+from rdagent.core.experiment import ASpecificExp
+from rdagent.core.prompts import Prompts
+from rdagent.core.utils import cache_with_pickle
+from rdagent.oai.llm_utils import md5_hash
+from rdagent.scenarios.kaggle.experiment.kaggle_experiment import (
+    KGFactorExperiment,
+    KGModelExperiment,
+)
+
+prompt_dict = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")
+
+
+class KGCachedRunner(CachedRunner[ASpecificExp]):
+    def get_cache_key(self, exp: ASpecificExp) -> str:
+        codes = []
+        for f in sorted((exp.experiment_workspace.workspace_path / "feature").glob("*.py"), key=lambda x: x.name):
+            codes.append(f.read_text())
+        for f in sorted((exp.experiment_workspace.workspace_path / "model").glob("*.py"), key=lambda x: x.name):
+            codes.append(f.read_text())
+        codes = "\n".join(codes)
+        return md5_hash(codes)
+
+    @cache_with_pickle(get_cache_key, CachedRunner.assign_cached_result)
+    def init_develop(self, exp: KGFactorExperiment | KGModelExperiment) -> KGFactorExperiment | KGModelExperiment:
+        """
+        For the initial development, the experiment serves as a benchmark for feature engineering.
+        """
+
+        env_to_use = {"PYTHONPATH": "./"}
+
+        result = exp.experiment_workspace.execute(run_env=env_to_use)
+
+        exp.result = result
+
+        return exp
+
+
+class KGModelRunner(KGCachedRunner[KGModelExperiment]):
+    @cache_with_pickle(KGCachedRunner.get_cache_key, KGCachedRunner.assign_cached_result)
+    def develop(self, exp: KGModelExperiment) -> KGModelExperiment:
+        if exp.based_experiments and exp.based_experiments[-1].result is None:
+            exp.based_experiments[-1] = self.init_develop(exp.based_experiments[-1])
+
+        sub_ws = exp.sub_workspace_list[0]
+        if sub_ws is not None:
+            # TODO: There's a possibility of generating a hybrid model (lightgbm + xgboost), which results in having two items in the model_type list.
+            model_type = sub_ws.target_task.model_type
+
+            if sub_ws.code_dict == {}:
+                raise ModelEmptyError("No model is implemented.")
+            else:
+                model_file_name = f"model/model_{model_type.lower()}.py"
+                exp.experiment_workspace.inject_code(**{model_file_name: sub_ws.code_dict["model.py"]})
+
+        env_to_use = {"PYTHONPATH": "./"}
+
+        result = exp.experiment_workspace.execute(run_env=env_to_use)
+
+        if result is None:
+            raise CoderError("No result is returned from the experiment workspace")
+
+        exp.result = result
+
+        return exp
+
+
+class KGFactorRunner(KGCachedRunner[KGFactorExperiment]):
+    def develop(self, exp: KGFactorExperiment) -> KGFactorExperiment:
+        current_feature_file_count = len(list(exp.experiment_workspace.workspace_path.glob("feature/feature*.py")))
+        implemented_factor_count = 0
+        for sub_ws in exp.sub_workspace_list:
+            if sub_ws.code_dict == {}:
+                continue
+            implemented_factor_count += 1
+            target_feature_file_name = f"feature/feature_{current_feature_file_count:05d}.py"
+            exp.experiment_workspace.inject_code(**{target_feature_file_name: sub_ws.code_dict["factor.py"]})
+            feature_shape = sub_ws.execute()[1].shape[-1]
+            exp.experiment_workspace.data_description.append((sub_ws.target_task.get_task_information(), feature_shape))
+            current_feature_file_count += 1
+        if implemented_factor_count == 0:
+            raise FactorEmptyError("No factor is implemented")
+
+        # initial template result
+        if exp.based_experiments and exp.based_experiments[-1].result is None:
+            exp.based_experiments[-1] = self.init_develop(exp.based_experiments[-1])
+
+        if RUNNER_SETTINGS.cache_result:
+            cache_hit, result = self.get_cache_result(exp)
+            if cache_hit:
+                exp.result = result
+                return exp
+
+        env_to_use = {"PYTHONPATH": "./"}
+
+        result = exp.experiment_workspace.execute(run_env=env_to_use)
+
+        if result is None:
+            raise CoderError("No result is returned from the experiment workspace")
+
+        exp.result = result
+
+        if RUNNER_SETTINGS.cache_result:
+            self.dump_cache_result(exp, result)
+
+        return exp

rdagent/scenarios/kaggle/docker/Dockerfile

@@ -1,4 +1,4 @@
-FROM pytorch/pytorch:latest
+FROM pytorch/pytorch:2.2.1-cuda12.1-cudnn8-runtime
 # For GPU support, please choose the proper tag from https://hub.docker.com/r/pytorch/pytorch/tags
 
 RUN apt-get clean && apt-get update && apt-get install -y \  
@@ -11,7 +11,7 @@ RUN apt-get clean && apt-get update && apt-get install -y \
 WORKDIR /workspace
 
 RUN python -m pip install numpy
-RUN python -m pip install --upgrade cython
+# RUN python -m pip install --upgrade cython
 # RUN python -m pip install -e .
 
 RUN python -m pip install pandas
@@ -23,3 +23,6 @@ RUN pip install scikit-learn
 RUN pip install catboost
 RUN pip install xgboost
 RUN pip install sparse
+RUN pip install lightgbm 
+RUN pip install pyarrow
+RUN pip install fastparquet

rdagent/scenarios/kaggle/experiment/covid19-global-forecasting-week-1_template/fea_share_preprocess.py

@@ -0,0 +1,64 @@
+import os
+
+import numpy as np
+import pandas as pd
+from sklearn.model_selection import train_test_split
+from sklearn.preprocessing import LabelEncoder
+
+
+def prepreprocess():
+    # Load the data
+    train = pd.read_csv("/kaggle/input/train.csv")
+    test = pd.read_csv("/kaggle/input/test.csv")
+
+    # Combine train and test for preprocessing
+    all_data = pd.concat([train, test], sort=False)
+
+    # Convert date to datetime
+    all_data["Date"] = pd.to_datetime(all_data["Date"])
+
+    # Create new features
+    all_data["Day"] = all_data["Date"].dt.day
+    all_data["Month"] = all_data["Date"].dt.month
+    all_data["Year"] = all_data["Date"].dt.year
+
+    # Encode categorical variables
+    le = LabelEncoder()
+    all_data["Country/Region"] = le.fit_transform(all_data["Country/Region"])
+    all_data["Province/State"] = le.fit_transform(all_data["Province/State"].fillna("None"))
+
+    # Split back into train and test
+    train = all_data[all_data["ForecastId"].isna()]
+    test = all_data[all_data["ForecastId"].notna()]
+
+    # Prepare features and targets
+    features = ["Country/Region", "Province/State", "Day", "Month", "Year"]
+    X = train[features]
+    y = train[["ConfirmedCases", "Fatalities"]]
+
+    # Split into train and validation sets
+    X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=0.2, random_state=42)
+
+    return X_train, X_valid, y_train, y_valid, test[features], test["ForecastId"]
+
+
+def preprocess_script():
+    if os.path.exists("/kaggle/input/X_train.pkl"):
+        X_train = pd.read_pickle("/kaggle/input/X_train.pkl")
+        X_valid = pd.read_pickle("/kaggle/input/X_valid.pkl")
+        y_train = pd.read_pickle("/kaggle/input/y_train.pkl")
+        y_valid = pd.read_pickle("/kaggle/input/y_valid.pkl")
+        X_test = pd.read_pickle("/kaggle/input/X_test.pkl")
+        forecast_ids = pd.read_pickle("/kaggle/input/forecast_ids.pkl")
+    else:
+        X_train, X_valid, y_train, y_valid, X_test, forecast_ids = prepreprocess()
+
+        # Save preprocessed data
+        X_train.to_pickle("/kaggle/input/X_train.pkl")
+        X_valid.to_pickle("/kaggle/input/X_valid.pkl")
+        y_train.to_pickle("/kaggle/input/y_train.pkl")
+        y_valid.to_pickle("/kaggle/input/y_valid.pkl")
+        X_test.to_pickle("/kaggle/input/X_test.pkl")
+        forecast_ids.to_pickle("/kaggle/input/forecast_ids.pkl")
+
+    return X_train, X_valid, y_train, y_valid, X_test, forecast_ids

rdagent/scenarios/kaggle/experiment/covid19-global-forecasting-week-1_template/feature/feature.py

@@ -0,0 +1,23 @@
+import pandas as pd
+
+"""
+Here is the feature engineering code for each task, with a class that has a fit and transform method.
+Remember
+"""
+
+
+class IdentityFeature:
+    def fit(self, train_df: pd.DataFrame):
+        """
+        Fit the feature engineering model to the training data.
+        """
+        pass
+
+    def transform(self, X: pd.DataFrame):
+        """
+        Transform the input data.
+        """
+        return X
+
+
+feature_engineering_cls = IdentityFeature

rdagent/scenarios/kaggle/experiment/covid19-global-forecasting-week-1_template/model/model_xgboost.py

@@ -0,0 +1,33 @@
+import pandas as pd
+import xgboost as xgb
+
+
+def fit(X_train: pd.DataFrame, y_train: pd.DataFrame, X_valid: pd.DataFrame, y_valid: pd.DataFrame):
+    """Define and train the model for both ConfirmedCases and Fatalities."""
+    models = {}
+    for target in ["ConfirmedCases", "Fatalities"]:
+        dtrain = xgb.DMatrix(X_train, label=y_train[target])
+        dvalid = xgb.DMatrix(X_valid, label=y_valid[target])
+
+        params = {
+            "objective": "reg:squarederror",
+            "eval_metric": "rmse",
+            "nthread": -1,
+            "tree_method": "gpu_hist",
+            "device": "cuda",
+        }
+        num_round = 1000
+
+        evallist = [(dtrain, "train"), (dvalid, "eval")]
+        models[target] = xgb.train(params, dtrain, num_round, evallist, early_stopping_rounds=50)
+
+    return models
+
+
+def predict(models, X):
+    """Make predictions for both ConfirmedCases and Fatalities."""
+    dtest = xgb.DMatrix(X)
+    predictions = {}
+    for target, model in models.items():
+        predictions[target] = model.predict(dtest)
+    return pd.DataFrame(predictions)

rdagent/scenarios/kaggle/experiment/covid19-global-forecasting-week-1_template/model/select_xgboost.py

@@ -0,0 +1,12 @@
+import pandas as pd
+
+
+def select(X: pd.DataFrame) -> pd.DataFrame:
+    """
+    Select relevant features. To be used in fit & predict function.
+    """
+    # For now, we assume all features are relevant. This can be expanded to feature selection logic.
+    if X.columns.nlevels == 1:
+        return X
+    X.columns = ["_".join(str(i) for i in col).strip() for col in X.columns.values]
+    return X

rdagent/scenarios/kaggle/experiment/covid19-global-forecasting-week-1_template/train.py

@@ -0,0 +1,119 @@
+import importlib.util
+import random
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+from fea_share_preprocess import preprocess_script
+from sklearn.metrics import mean_squared_log_error
+
+# Set random seed for reproducibility
+SEED = 42
+random.seed(SEED)
+np.random.seed(SEED)
+DIRNAME = Path(__file__).absolute().resolve().parent
+
+
+def compute_rmsle(y_true, y_pred):
+    """Compute Root Mean Squared Logarithmic Error for regression."""
+    return np.sqrt(mean_squared_log_error(y_true, y_pred))
+
+
+def import_module_from_path(module_name, module_path):
+    spec = importlib.util.spec_from_file_location(module_name, module_path)
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+# 1) Preprocess the data
+X_train, X_valid, y_train, y_valid, X_test, forecast_ids = preprocess_script()
+
+# 2) Auto feature engineering
+X_train_l, X_valid_l = [], []
+X_test_l = []
+
+for f in DIRNAME.glob("feature/feat*.py"):
+    cls = import_module_from_path(f.stem, f).feature_engineering_cls()
+    cls.fit(X_train)
+    X_train_f = cls.transform(X_train)
+    X_valid_f = cls.transform(X_valid)
+    X_test_f = cls.transform(X_test)
+
+    if X_train_f.shape[-1] == X_valid_f.shape[-1] and X_train_f.shape[-1] == X_test_f.shape[-1]:
+        X_train_l.append(X_train_f)
+        X_valid_l.append(X_valid_f)
+        X_test_l.append(X_test_f)
+
+X_train = pd.concat(X_train_l, axis=1, keys=[f"feature_{i}" for i in range(len(X_train_l))])
+X_valid = pd.concat(X_valid_l, axis=1, keys=[f"feature_{i}" for i in range(len(X_valid_l))])
+X_test = pd.concat(X_test_l, axis=1, keys=[f"feature_{i}" for i in range(len(X_test_l))])
+
+print(X_train.shape, X_valid.shape, X_test.shape)
+
+# Handle inf and -inf values
+X_train.replace([np.inf, -np.inf], np.nan, inplace=True)
+X_valid.replace([np.inf, -np.inf], np.nan, inplace=True)
+X_test.replace([np.inf, -np.inf], np.nan, inplace=True)
+
+from sklearn.impute import SimpleImputer
+
+imputer = SimpleImputer(strategy="mean")
+
+X_train = pd.DataFrame(imputer.fit_transform(X_train), columns=X_train.columns)
+X_valid = pd.DataFrame(imputer.transform(X_valid), columns=X_valid.columns)
+X_test = pd.DataFrame(imputer.transform(X_test), columns=X_test.columns)
+
+# Remove duplicate columns
+X_train = X_train.loc[:, ~X_train.columns.duplicated()]
+X_valid = X_valid.loc[:, ~X_valid.columns.duplicated()]
+X_test = X_test.loc[:, ~X_test.columns.duplicated()]
+
+# 3) Train the model
+model_l = []  # list[tuple[model, predict_func,]]
+for f in DIRNAME.glob("model/model*.py"):
+    select_python_path = f.with_name(f.stem.replace("model", "select") + f.suffix)
+    select_m = import_module_from_path(select_python_path.stem, select_python_path)
+    X_train_selected = select_m.select(X_train.copy())
+    X_valid_selected = select_m.select(X_valid.copy())
+
+    m = import_module_from_path(f.stem, f)
+    model_l.append((m.fit(X_train_selected, y_train, X_valid_selected, y_valid), m.predict, select_m))
+
+
+# 4) Evaluate the model on the validation set
+metrics_all = []
+for model, predict_func, select_m in model_l:
+    X_valid_selected = select_m.select(X_valid.copy())
+    y_valid_pred = predict_func(model, X_valid_selected)
+
+    # Add a small positive value to avoid negative or zero values
+    epsilon = 1e-8
+    y_valid_cases = np.maximum(y_valid["ConfirmedCases"], epsilon)
+    y_pred_cases = np.maximum(y_valid_pred["ConfirmedCases"], epsilon)
+
+    rmsle_cases = compute_rmsle(y_valid_cases, y_pred_cases)
+    rmsle_fatalities = compute_rmsle(
+        np.maximum(y_valid["Fatalities"], epsilon), np.maximum(y_valid_pred["Fatalities"], epsilon)
+    )
+    rmsle_avg = (rmsle_cases + rmsle_fatalities) / 2
+    print(f"Average RMSLE on valid set: {rmsle_avg}")
+    metrics_all.append(rmsle_avg)
+
+# 5) Save the validation accuracy
+min_index = np.argmin(metrics_all)
+pd.Series(data=[metrics_all[min_index]], index=["RMSLE"]).to_csv("submission_score.csv")
+
+# 6) Make predictions on the test set and save them
+X_test_selected = model_l[min_index][2].select(X_test.copy())
+y_test_pred = model_l[min_index][1](model_l[min_index][0], X_test_selected)
+
+# 7) Submit predictions for the test set
+submission_result = pd.DataFrame(
+    {
+        "ForecastId": forecast_ids,
+        "ConfirmedCases": y_test_pred["ConfirmedCases"],
+        "Fatalities": y_test_pred["Fatalities"],
+    }
+)
+submission_result.to_csv("submission.csv", index=False)

rdagent/scenarios/kaggle/experiment/digit-recognizer_template/fea_share_preprocess.py

@@ -0,0 +1,66 @@
+import os
+
+import numpy as np
+import pandas as pd
+from sklearn.impute import SimpleImputer
+from sklearn.model_selection import train_test_split
+
+
+def prepreprocess():
+    """
+    This method loads the data, drops the unnecessary columns, and splits it into train and validation sets.
+    """
+    # Load and preprocess the data
+    data_df = pd.read_csv("/kaggle/input/train.csv")
+    # data_df = data_df.drop(["ImageId"], axis=1)
+
+    X = data_df.drop(["label"], axis=1)
+    y = data_df["label"]
+
+    # Split the data into training and validation sets
+    X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=0.20, random_state=42)
+
+    return X_train, X_valid, y_train, y_valid
+
+
+def preprocess_script():
+    """
+    This method applies the preprocessing steps to the training, validation, and test datasets.
+    """
+    if os.path.exists("/kaggle/input/X_train.pkl"):
+        X_train = pd.read_pickle("/kaggle/input/X_train.pkl")
+        X_valid = pd.read_pickle("/kaggle/input/X_valid.pkl")
+        y_train = pd.read_pickle("/kaggle/input/y_train.pkl")
+        y_valid = pd.read_pickle("/kaggle/input/y_valid.pkl")
+        X_test = pd.read_pickle("/kaggle/input/X_test.pkl")
+        others = pd.read_pickle("/kaggle/input/others.pkl")
+
+        return X_train, X_valid, y_train, y_valid, X_test, *others
+
+    X_train, X_valid, y_train, y_valid = prepreprocess()
+
+    # Load and preprocess the test data
+    submission_df = pd.read_csv("/kaggle/input/test.csv")
+    # ids = submission_df["ImageId"]
+    X_test = submission_df
+
+    X_train = X_train / 255
+    X_valid = X_valid / 255
+    X_test = X_test / 255
+
+    return X_train, X_valid, y_train, y_valid, X_test
+
+
+def clean_and_impute_data(X_train, X_valid, X_test):
+    """
+    Handles inf and -inf values by replacing them with NaN,
+    then imputes missing values using the mean strategy.
+    Also removes duplicate columns.
+    """
+    # Impute missing values
+    imputer = SimpleImputer(strategy="mean")
+    X_train = pd.DataFrame(imputer.fit_transform(X_train), columns=X_train.columns)
+    X_valid = pd.DataFrame(imputer.transform(X_valid), columns=X_valid.columns)
+    X_test = pd.DataFrame(imputer.transform(X_test), columns=X_test.columns)
+
+    return X_train, X_valid, X_test

rdagent/scenarios/kaggle/experiment/digit-recognizer_template/feature/feature.py

@@ -0,0 +1,23 @@
+import pandas as pd
+
+"""
+Here is the feature engineering code for each task, with a class that has a fit and transform method.
+Remember
+"""
+
+
+class IdentityFeature:
+    def fit(self, train_df: pd.DataFrame):
+        """
+        Fit the feature engineering model to the training data.
+        """
+        pass
+
+    def transform(self, X: pd.DataFrame):
+        """
+        Transform the input data.
+        """
+        return X
+
+
+feature_engineering_cls = IdentityFeature

rdagent/scenarios/kaggle/experiment/digit-recognizer_template/model/model_nn.py

@@ -0,0 +1,86 @@
+import pandas as pd
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import torch.optim as optim
+from torch.utils.data import DataLoader, TensorDataset
+
+
+# Define the neural network model with Batch Normalization
+class NeuralNetwork(nn.Module):
+    def __init__(self, input_channels, num_classes):
+        super(NeuralNetwork, self).__init__()
+        self.conv1 = nn.Conv2d(in_channels=input_channels, out_channels=30, kernel_size=(3, 3), stride=2)
+        self.dropout1 = nn.Dropout(0.5)
+        self.conv2 = nn.Conv2d(in_channels=30, out_channels=30, kernel_size=(3, 3), stride=2)
+        self.dropout2 = nn.Dropout(0.5)
+        self.flatten = nn.Flatten()
+        self.fc1 = nn.Linear(30 * 6 * 6, 128)  # Adjust based on your input size
+        self.fc2 = nn.Linear(128, num_classes)
+
+    def forward(self, x):
+        x = F.relu(self.conv1(x))
+        x = self.dropout1(x)
+        x = F.relu(self.conv2(x))
+        x = self.dropout2(x)
+        x = self.flatten(x)
+        x = F.relu(self.fc1(x))
+        x = F.softmax(self.fc2(x), dim=1)
+        return x
+
+
+def fit(X_train: pd.DataFrame, y_train: pd.DataFrame, X_valid: pd.DataFrame, y_valid: pd.DataFrame):
+    # Convert data to PyTorch tensors and reshape it for convolutional layers
+    X_train_tensor = (
+        torch.tensor(X_train.values, dtype=torch.float32).view(-1, 1, 28, 28).to(device)
+    )  # Reshape and move to GPU
+    y_train_tensor = torch.tensor(y_train.values, dtype=torch.long).to(device)
+    X_valid_tensor = torch.tensor(X_valid.values, dtype=torch.float32).view(-1, 1, 28, 28).to(device)
+    y_valid_tensor = torch.tensor(y_valid.values, dtype=torch.long).to(device)
+
+    # Create datasets and dataloaders
+    train_dataset = TensorDataset(X_train_tensor, y_train_tensor)
+    valid_dataset = TensorDataset(X_valid_tensor, y_valid_tensor)
+    train_loader = DataLoader(train_dataset, batch_size=128, shuffle=True)
+    valid_loader = DataLoader(valid_dataset, batch_size=128, shuffle=False)
+
+    # Initialize the model, loss function and optimizer
+    model = NeuralNetwork(input_channels=1, num_classes=len(set(y_train))).to(device)
+    criterion = nn.CrossEntropyLoss().to(device)
+    optimizer = optim.Adam(model.parameters(), lr=0.0005)
+
+    # Train the model
+    num_epochs = 400
+    for epoch in range(num_epochs):
+        model.train()
+        for X_batch, y_batch in train_loader:
+            optimizer.zero_grad()
+            outputs = model(X_batch)
+            loss = criterion(outputs, y_batch)
+            loss.backward()
+            optimizer.step()
+
+        # Validate the model
+        model.eval()
+        valid_loss = 0
+        correct = 0
+        with torch.no_grad():
+            for X_batch, y_batch in valid_loader:
+                outputs = model(X_batch)
+                valid_loss += criterion(outputs, y_batch).item()
+                _, predicted = torch.max(outputs, 1)
+                correct += (predicted == y_batch).sum().item()
+
+        accuracy = correct / len(valid_loader.dataset)
+        print(f"Epoch {epoch+1}/{num_epochs}, Validation Accuracy: {accuracy:.4f}")
+
+    return model
+
+
+def predict(model, X):
+    X_tensor = torch.tensor(X.values, dtype=torch.float32).view(-1, 1, 28, 28).to(device)
+    model.eval()
+    with torch.no_grad():
+        outputs = model(X_tensor)
+        _, predicted = torch.max(outputs, 1)
+    return predicted.cpu().numpy().reshape(-1, 1)

rdagent/scenarios/kaggle/experiment/digit-recognizer_template/model/model_xgboost.py

@@ -0,0 +1,35 @@
+"""
+motivation  of the model
+"""
+
+import pandas as pd
+import xgboost as xgb
+
+
+def fit(X_train, y_train, X_valid, y_valid):
+    """Define and train the model. Merge feature_select"""
+    dtrain = xgb.DMatrix(X_train, label=y_train)
+    dvalid = xgb.DMatrix(X_valid, label=y_valid)
+
+    params = {
+        "objective": "multi:softmax",
+        "eval_metric": "mlogloss",
+        "num_class": 10,
+        "nthread": -1,
+        "tree_method": "gpu_hist",
+        "device": "cuda",
+    }
+    num_round = 100
+
+    evallist = [(dtrain, "train"), (dvalid, "eval")]
+    model = xgb.train(params, dtrain, num_round, evallist, early_stopping_rounds=10)
+
+    return model
+
+
+def predict(model, X):
+    """
+    Keep feature select's consistency.
+    """
+    dtest = xgb.DMatrix(X)
+    return model.predict(dtest).astype(int)

rdagent/scenarios/kaggle/experiment/digit-recognizer_template/model/select_lightgbm.py

@@ -0,0 +1,12 @@
+import pandas as pd
+
+
+def select(X: pd.DataFrame) -> pd.DataFrame:
+    """
+    Select relevant features. To be used in fit & predict function.
+    """
+    # For now, we assume all features are relevant. This can be expanded to feature selection logic.
+    if X.columns.nlevels == 1:
+        return X
+    X.columns = ["_".join(str(i) for i in col).strip() for col in X.columns.values]
+    return X

rdagent/scenarios/kaggle/experiment/digit-recognizer_template/model/select_nn.py

@@ -0,0 +1,12 @@
+import pandas as pd
+
+
+def select(X: pd.DataFrame) -> pd.DataFrame:
+    """
+    Select relevant features. To be used in fit & predict function.
+    """
+    # For now, we assume all features are relevant. This can be expanded to feature selection logic.
+    if X.columns.nlevels == 1:
+        return X
+    X.columns = ["_".join(str(i) for i in col).strip() for col in X.columns.values]
+    return X

rdagent/scenarios/kaggle/experiment/digit-recognizer_template/model/select_randomforest.py

@@ -0,0 +1,12 @@
+import pandas as pd
+
+
+def select(X: pd.DataFrame) -> pd.DataFrame:
+    """
+    Select relevant features. To be used in fit & predict function.
+    """
+    # For now, we assume all features are relevant. This can be expanded to feature selection logic.
+    if X.columns.nlevels == 1:
+        return X
+    X.columns = ["_".join(str(i) for i in col).strip() for col in X.columns.values]
+    return X

rdagent/scenarios/kaggle/experiment/digit-recognizer_template/model/select_xgboost.py

@@ -0,0 +1,12 @@
+import pandas as pd
+
+
+def select(X: pd.DataFrame) -> pd.DataFrame:
+    """
+    Select relevant features. To be used in fit & predict function.
+    """
+    # For now, we assume all features are relevant. This can be expanded to feature selection logic.
+    if X.columns.nlevels == 1:
+        return X
+    X.columns = ["_".join(str(i) for i in col).strip() for col in X.columns.values]
+    return X

rdagent/scenarios/kaggle/experiment/digit-recognizer_template/train.py

@@ -0,0 +1,88 @@
+import importlib.util
+import random
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+from fea_share_preprocess import clean_and_impute_data, preprocess_script
+from sklearn.metrics import accuracy_score
+
+# Set random seed for reproducibility
+SEED = 42
+random.seed(SEED)
+np.random.seed(SEED)
+DIRNAME = Path(__file__).absolute().resolve().parent
+
+
+def compute_metrics_for_classification(y_true, y_pred):
+    """Compute accuracy for classification."""
+    return accuracy_score(y_true, y_pred)
+
+
+def import_module_from_path(module_name, module_path):
+    spec = importlib.util.spec_from_file_location(module_name, module_path)
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+# 1) Preprocess the data
+X_train, X_valid, y_train, y_valid, X_test = preprocess_script()
+
+# 2) Auto feature engineering
+X_train_l, X_valid_l = [], []
+X_test_l = []
+
+for f in DIRNAME.glob("feature/feat*.py"):
+    cls = import_module_from_path(f.stem, f).feature_engineering_cls()
+    cls.fit(X_train)
+    X_train_f = cls.transform(X_train)
+    X_valid_f = cls.transform(X_valid)
+    X_test_f = cls.transform(X_test)
+
+    if X_train_f.shape[-1] == X_valid_f.shape[-1] and X_train_f.shape[-1] == X_test_f.shape[-1]:
+        X_train_l.append(X_train_f)
+        X_valid_l.append(X_valid_f)
+        X_test_l.append(X_test_f)
+
+X_train = pd.concat(X_train_l, axis=1, keys=[f"feature_{i}" for i in range(len(X_train_l))])
+X_valid = pd.concat(X_valid_l, axis=1, keys=[f"feature_{i}" for i in range(len(X_valid_l))])
+X_test = pd.concat(X_test_l, axis=1, keys=[f"feature_{i}" for i in range(len(X_test_l))])
+
+print(X_train.shape, X_valid.shape, X_test.shape)
+
+# Handle inf and -inf values
+X_train, X_valid, X_test = clean_and_impute_data(X_train, X_valid, X_test)
+
+
+model_l = []  # list[tuple[model, predict_func]]
+for f in DIRNAME.glob("model/model*.py"):
+    select_python_path = f.with_name(f.stem.replace("model", "select") + f.suffix)
+    select_m = import_module_from_path(select_python_path.stem, select_python_path)
+    X_train_selected = select_m.select(X_train.copy())
+    X_valid_selected = select_m.select(X_valid.copy())
+
+    m = import_module_from_path(f.stem, f)
+    model_l.append((m.fit(X_train_selected, y_train, X_valid_selected, y_valid), m.predict, select_m))
+
+# 4) Evaluate the model on the validation set
+metrics_all = []
+for model, predict_func, select_m in model_l:
+    X_valid_selected = select_m.select(X_valid.copy())
+    y_valid_pred = predict_func(model, X_valid_selected)
+    accuracy = accuracy_score(y_valid, y_valid_pred)
+    print(f"final accuracy on valid set: {accuracy}")
+    metrics_all.append(accuracy)
+
+# 5) Save the validation accuracy
+max_index = np.argmax(metrics_all)
+pd.Series(data=[metrics_all[max_index]], index=["multi-class accuracy"]).to_csv("submission_score.csv")
+
+# 6) Submit predictions for the test
+ids = range(1, len(X_test) + 1)
+
+# TODO: fix selection
+print(X_valid_selected.columns)
+y_test_pred = model_l[max_index][1](model_l[max_index][0], model_l[max_index][2].select(X_test)).flatten()
+submission_result = pd.DataFrame({"ImageId": ids, "Label": y_test_pred})
+submission_result.to_csv("submission.csv", index=False)

rdagent/scenarios/kaggle/experiment/feedback-prize-english-language-learning_template/fea_share_preprocess.py

@@ -0,0 +1,43 @@
+import os
+import re
+
+import numpy as np  # linear algebra
+import pandas as pd  # data processing, CSV file I/O (e.g. pd.read_csv)
+from sklearn.model_selection import train_test_split
+
+
+def preprocess_script():
+    """
+    This method applies the preprocessing steps to the training, validation, and test datasets.
+    """
+    if os.path.exists("/kaggle/input/X_train.pkl"):
+        X_train = pd.read_pickle("/kaggle/input/X_train.pkl")
+        X_valid = pd.read_pickle("/kaggle/input/X_valid.pkl")
+        y_train = pd.read_pickle("/kaggle/input/y_train.pkl")
+        y_valid = pd.read_pickle("/kaggle/input/y_valid.pkl")
+        X_test = pd.read_pickle("/kaggle/input/X_test.pkl")
+        others = pd.read_pickle("/kaggle/input/others.pkl")
+
+        return X_train, X_valid, y_train, y_valid, X_test, *others
+
+    def data_cleaner(text):
+        text = text.strip()
+        text = re.sub(r"\n", "", text)
+        text = text.lower()
+        return text
+
+    # train
+    train = pd.read_csv("/kaggle/input/train.csv")
+    test = pd.read_csv("/kaggle/input/test.csv")
+
+    train["full_text"] = train["full_text"].apply(data_cleaner)
+    test["full_text"] = test["full_text"].apply(data_cleaner)
+
+    y_train = train[["cohesion", "syntax", "vocabulary", "phraseology", "grammar", "conventions"]]
+
+    X_train = train[["full_text"]]
+    X_test = test[["full_text"]]
+
+    X_train, X_valid, y_train, y_valid = train_test_split(X_train, y_train, test_size=0.2, random_state=42)
+
+    return X_train, X_valid, y_train, y_valid, X_test

rdagent/scenarios/kaggle/experiment/feedback-prize-english-language-learning_template/feature/feature.py

@@ -0,0 +1,27 @@
+import pandas as pd
+from sklearn.feature_extraction.text import TfidfVectorizer
+
+"""
+Here is the feature engineering code for each task, with a class that has a fit and transform method.
+Remember
+"""
+
+
+class IdentityFeature:
+    def fit(self, train_df: pd.DataFrame):
+        """
+        Fit the feature engineering model to the training data.
+        """
+        self.vectorizer = TfidfVectorizer()
+        self.vectorizer.fit(train_df["full_text"])
+
+    def transform(self, X: pd.DataFrame):
+        """
+        Transform the input data.
+        """
+        X = self.vectorizer.transform(X["full_text"])
+        X = pd.DataFrame.sparse.from_spmatrix(X)
+        return X
+
+
+feature_engineering_cls = IdentityFeature

rdagent/scenarios/kaggle/experiment/feedback-prize-english-language-learning_template/model/model_randomforest.py

@@ -0,0 +1,26 @@
+import numpy as np
+import pandas as pd
+from sklearn.ensemble import RandomForestRegressor
+
+
+def fit(X_train: pd.DataFrame, y_train: pd.Series, X_valid: pd.DataFrame, y_valid: pd.Series):
+    """
+    Define and train the Random Forest model. Merge feature selection into the pipeline.
+    """
+    # Initialize the Random Forest model
+    model = RandomForestRegressor(n_estimators=100, random_state=32, n_jobs=-1)
+
+    # Fit the model
+    model.fit(X_train, y_train)
+
+    return model
+
+
+def predict(model, X):
+    """
+    Keep feature selection's consistency and make predictions.
+    """
+    # Predict using the trained model
+    y_pred = model.predict(X)
+
+    return y_pred

rdagent/scenarios/kaggle/experiment/feedback-prize-english-language-learning_template/model/model_xgboost.py

@@ -0,0 +1,37 @@
+"""
+motivation  of the model
+"""
+
+import pandas as pd
+import xgboost as xgb
+from sklearn.multioutput import MultiOutputRegressor
+
+
+def is_sparse_df(df: pd.DataFrame) -> bool:
+    # 检查 DataFrame 中的每一列是否为稀疏类型
+    return any(isinstance(dtype, pd.SparseDtype) for dtype in df.dtypes)
+
+
+def fit(X_train: pd.DataFrame, y_train: pd.DataFrame, X_valid: pd.DataFrame, y_valid: pd.DataFrame):
+    """Define and train the model. Merge feature_select"""
+    xgb_estimator = xgb.XGBRegressor(
+        n_estimators=500, random_state=0, objective="reg:squarederror", tree_method="hist", device="cuda"
+    )
+
+    model = MultiOutputRegressor(xgb_estimator, n_jobs=-1)
+
+    if is_sparse_df(X_train):
+        X_train = X_train.sparse.to_coo()
+
+    model.fit(X_train, y_train)
+    return model
+
+
+def predict(model, X_test):
+    """
+    Keep feature select's consistency.
+    """
+    if is_sparse_df(X_test):
+        X_test = X_test.sparse.to_coo()
+    y_pred = model.predict(X_test)
+    return y_pred

rdagent/scenarios/kaggle/experiment/feedback-prize-english-language-learning_template/model/select_randomforest.py

@@ -0,0 +1,12 @@
+import pandas as pd
+
+
+def select(X: pd.DataFrame) -> pd.DataFrame:
+    """
+    Select relevant features. To be used in fit & predict function.
+    """
+    # For now, we assume all features are relevant. This can be expanded to feature selection logic.
+    if X.columns.nlevels == 1:
+        return X
+    X.columns = ["_".join(str(i) for i in col).strip() for col in X.columns.values]
+    return X

rdagent/scenarios/kaggle/experiment/feedback-prize-english-language-learning_template/model/select_xgboost.py

@@ -0,0 +1,12 @@
+import pandas as pd
+
+
+def select(X: pd.DataFrame) -> pd.DataFrame:
+    """
+    Select relevant features. To be used in fit & predict function.
+    """
+    # For now, we assume all features are relevant. This can be expanded to feature selection logic.
+    if X.columns.nlevels == 1:
+        return X
+    X.columns = ["_".join(str(i) for i in col).strip() for col in X.columns.values]
+    return X

rdagent/scenarios/kaggle/experiment/feedback-prize-english-language-learning_template/train.py

@@ -0,0 +1,85 @@
+import importlib.util
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+from fea_share_preprocess import preprocess_script
+
+DIRNAME = Path(__file__).absolute().resolve().parent
+
+
+def import_module_from_path(module_name, module_path):
+    spec = importlib.util.spec_from_file_location(module_name, module_path)
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+def MCRMSE(y_true, y_pred):
+    return np.mean(np.sqrt(np.mean((y_true - y_pred) ** 2, axis=0)))
+
+
+# 1) Preprocess the data
+X_train, X_valid, y_train, y_valid, X_test = preprocess_script()
+
+# 2) Auto feature engineering
+X_train_l, X_valid_l = [], []
+X_test_l = []
+
+for f in DIRNAME.glob("feature/feat*.py"):
+    cls = import_module_from_path(f.stem, f).feature_engineering_cls()
+    print(X_train.head())
+    cls.fit(X_train)
+    X_train_f = cls.transform(X_train)
+    X_valid_f = cls.transform(X_valid)
+    X_test_f = cls.transform(X_test)
+
+    if X_train_f.shape[-1] == X_valid_f.shape[-1] and X_train_f.shape[-1] == X_test_f.shape[-1]:
+        X_train_l.append(X_train_f)
+        X_valid_l.append(X_valid_f)
+        X_test_l.append(X_test_f)
+
+X_train = pd.concat(X_train_l, axis=1, keys=[f"feature_{i}" for i in range(len(X_train_l))])
+X_valid = pd.concat(X_valid_l, axis=1, keys=[f"feature_{i}" for i in range(len(X_valid_l))])
+X_test = pd.concat(X_test_l, axis=1, keys=[f"feature_{i}" for i in range(len(X_test_l))])
+
+print(X_train.shape, X_valid.shape, X_test.shape)
+
+# 3) Train the model
+model_l = []  # list[tuple[model, predict_func,]]
+for f in DIRNAME.glob("model/model*.py"):
+    select_python_path = f.with_name(f.stem.replace("model", "select") + f.suffix)
+    select_m = import_module_from_path(select_python_path.stem, select_python_path)
+    X_train_selected = select_m.select(X_train.copy())
+    X_valid_selected = select_m.select(X_valid.copy())
+
+    m = import_module_from_path(f.stem, f)
+    model_l.append((m.fit(X_train_selected, y_train, X_valid_selected, y_valid), m.predict))
+
+# 4) Evaluate the model on the validation set
+metrics_all = []
+for model, predict_func in model_l:
+    X_valid_selected = select_m.select(X_valid.copy())
+    y_valid_pred = predict_func(model, X_valid_selected)
+    metrics = MCRMSE(y_valid, y_valid_pred)
+    print(f"MCRMSE on valid set: {metrics}")
+    metrics_all.append(metrics)
+
+# 5) Save the validation accuracy
+min_index = np.argmin(metrics_all)
+pd.Series(data=[metrics_all[min_index]], index=["MCRMSE"]).to_csv("submission_score.csv")
+
+# 6) Make predictions on the test set and save them
+X_test_selected = select_m.select(X_test.copy())
+y_test_pred = model_l[min_index][1](model_l[min_index][0], X_test_selected)
+
+# 7) Submit predictions for the test set
+submission_result = pd.read_csv("/kaggle/input/sample_submission.csv")
+submission_result["cohesion"] = y_test_pred[:, 0]
+submission_result["syntax"] = y_test_pred[:, 1]
+submission_result["vocabulary"] = y_test_pred[:, 2]
+submission_result["phraseology"] = y_test_pred[:, 3]
+submission_result["grammar"] = y_test_pred[:, 4]
+submission_result["conventions"] = y_test_pred[:, 5]
+
+submission_result.to_csv("submission.csv", index=False)

rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/fea_share_preprocess.py

@@ -0,0 +1,72 @@
+import os
+
+import numpy as np
+import pandas as pd
+from sklearn.impute import SimpleImputer
+from sklearn.model_selection import train_test_split
+
+
+def prepreprocess():
+    """
+    This method loads the data, drops the unnecessary columns, and splits it into train and validation sets.
+    """
+    # Load and preprocess the data
+    data_df = pd.read_csv("/kaggle/input/train.csv")
+    data_df = data_df.drop(["Id"], axis=1)
+
+    X = data_df.drop(["Cover_Type"], axis=1)
+    y = data_df["Cover_Type"] - 1
+
+    # Split the data into training and validation sets
+    X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=0.20, random_state=42)
+
+    return X_train, X_valid, y_train, y_valid
+
+
+def preprocess_script():
+    """
+    This method applies the preprocessing steps to the training, validation, and test datasets.
+    """
+    if os.path.exists("/kaggle/input/X_train.pkl"):
+        X_train = pd.read_pickle("/kaggle/input/X_train.pkl")
+        X_valid = pd.read_pickle("/kaggle/input/X_valid.pkl")
+        y_train = pd.read_pickle("/kaggle/input/y_train.pkl")
+        y_valid = pd.read_pickle("/kaggle/input/y_valid.pkl")
+        X_test = pd.read_pickle("/kaggle/input/X_test.pkl")
+        others = pd.read_pickle("/kaggle/input/others.pkl")
+
+        return X_train, X_valid, y_train, y_valid, X_test, *others
+
+    X_train, X_valid, y_train, y_valid = prepreprocess()
+
+    # Load and preprocess the test data
+    submission_df = pd.read_csv("/kaggle/input/test.csv")
+    ids = submission_df["Id"]
+    X_test = submission_df.drop(["Id"], axis=1)
+
+    return X_train, X_valid, y_train, y_valid, X_test, ids
+
+
+def clean_and_impute_data(X_train, X_valid, X_test):
+    """
+    Handles inf and -inf values by replacing them with NaN,
+    then imputes missing values using the mean strategy.
+    Also removes duplicate columns.
+    """
+    # Replace inf and -inf with NaN
+    X_train.replace([np.inf, -np.inf], np.nan, inplace=True)
+    X_valid.replace([np.inf, -np.inf], np.nan, inplace=True)
+    X_test.replace([np.inf, -np.inf], np.nan, inplace=True)
+
+    # Impute missing values
+    imputer = SimpleImputer(strategy="mean")
+    X_train = pd.DataFrame(imputer.fit_transform(X_train), columns=X_train.columns)
+    X_valid = pd.DataFrame(imputer.transform(X_valid), columns=X_valid.columns)
+    X_test = pd.DataFrame(imputer.transform(X_test), columns=X_test.columns)
+
+    # Remove duplicate columns
+    X_train = X_train.loc[:, ~X_train.columns.duplicated()]
+    X_valid = X_valid.loc[:, ~X_valid.columns.duplicated()]
+    X_test = X_test.loc[:, ~X_test.columns.duplicated()]
+
+    return X_train, X_valid, X_test

rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/feature/feature.py

@@ -0,0 +1,23 @@
+import pandas as pd
+
+"""
+Here is the feature engineering code for each task, with a class that has a fit and transform method.
+Remember
+"""
+
+
+class IdentityFeature:
+    def fit(self, train_df: pd.DataFrame):
+        """
+        Fit the feature engineering model to the training data.
+        """
+        pass
+
+    def transform(self, X: pd.DataFrame):
+        """
+        Transform the input data.
+        """
+        return X
+
+
+feature_engineering_cls = IdentityFeature