chore: release 0.7.0

Release-As: 0.7.0

.devcontainer/Dockerfile

@@ -0,0 +1,10 @@
+# 1. Pull down your Azure Container Registry image
+FROM rdagentappregistry.azurecr.io/rd-agent-mle:20250623
+
+# 2. (Optional) install any additional tools you need
+#    e.g. git, bash-completion, etc.
+# RUN apt update && \
+#     apt install -y git bash-completion && \
+#     rm -rf /var/lib/apt/lists/*
+RUN apt update && \
+    apt install -y git bash-completion

.devcontainer/README.md

@@ -0,0 +1,30 @@
+# Introduction
+
+!!!!!This dev container is not for public development!!!!!!
+!!!!!Please don't use it if you are just a public open-source user.!!!!!!
+
+# Steps to run the dev container (for internal use only)
+
+Prerequisites(this is the reason why this dev container is not for public use):
+
+- Make sure you have the `rdagentappregistry.azurecr.io/rd-agent-mle:20250623` image locally & DevContainer is installed in your IDE
+- The kaggle dataset is located at `/home/shared/RD-Agent/kaggle`
+
+1. Open the project and select "Open In DevContainer"
+2. Set up your Kaggle Key (do not share this; other internal URLs are hardcoded in the config files)
+
+```bash
+export KAGGLE_USERNAME=
+export KAGGLE_KEY=
+```
+
+3. Run: python rdagent/app/data_science/loop.py --competition nomad2018-predict-transparent-conductors
+
+
+# Additional Notes
+- Please install and use this Dev Container in VS Code.
+- You **must open VS Code remotely and enter the `RD-Agent` directory before running the DevContainer configuration (`.devcontainer/devcontainer.json`)**. Otherwise, the workspace and path mappings will not work as expected.
+- To open the DevContainer correctly in VS Code:
+  1. Remotely connect to the machine and open the `RD-Agent` folder in VS Code.
+  2. Press `Ctrl+Shift+P` (or `Cmd+Shift+P` on Mac), type and select **"Dev Containers: Reopen in Container"**.
+

.devcontainer/devcontainer.json

@@ -0,0 +1,29 @@
+{
+  "name": "rd-agent-mle DevContainer",
+  "build": {
+    "dockerfile": "Dockerfile",
+    "context": ".."
+  },
+  "workspaceFolder": "/workspace/RD-Agent",
+  "workspaceMount": "source=${localWorkspaceFolder},target=/workspace/RD-Agent,type=bind,consistency=cached",
+  "remoteUser": "root",
+  "settings": {
+    "terminal.integrated.shell.linux": "/bin/bash"
+  },
+  "mounts": [
+    "source=/home/shared/RD-Agent/kaggle,target=/home/shared/RD-Agent/kaggle,type=bind,consistency=cached,readonly"
+  ],
+  "extensions": [
+    "ms-python.python",
+    "ms-python.vscode-pylance",
+    "ms-toolsai.jupyter"
+  ],
+  "runArgs": [
+    "--init",
+    "--shm-size=1g",
+    "--env-file", "${localWorkspaceFolder}/.devcontainer/env",
+    "--network=host",
+    "--gpus=all"
+  ],
+  "postCreateCommand": "make dev"
+}

.devcontainer/env

@@ -0,0 +1,47 @@
+# Global configs:
+
+MAX_RETRY=12000
+RETRY_WAIT_SECONDS=5
+TIMEOUT_FAIL_LIMIT=100
+
+# litellm
+# CHAT_MODEL=gpt-4o
+# CHAT_TEMPERATURE=0.7
+
+CHAT_STREAM=False
+CHAT_TEMPERATURE=1
+CHAT_MODEL=o1-preview
+SYSTEM_PROMPT_ROLE=user
+
+BACKEND=rdagent.oai.backend.LiteLLMAPIBackend
+OPENAI_API_KEY=sk-1234
+OPENAI_API_BASE=http://ep14.213428.xyz:38881
+
+
+# amc chat model configs:
+EMBEDDING_MODEL=text-embedding-ada-002
+
+# Cache Setting (Optional):
+DUMP_CHAT_CACHE=True
+USE_CHAT_CACHE=False
+DUMP_EMBEDDING_CACHE=True
+USE_EMBEDDING_CACHE=False
+LOG_LLM_CHAT_CONTENT=True
+
+DS_LOCAL_DATA_PATH=/home/shared/RD-Agent/kaggle
+
+DS_IF_USING_MLE_DATA=True
+
+
+PICKLE_CACHE_FOLDER_PATH_STR=./log/pickle_cache
+CACHE_WITH_PICKLE=False
+ENABLE_CACHE=False
+PROMPT_CACHE_PATH=./log/prompt_cache.db
+
+DS_CODER_COSTEER_ENV_TYPE=conda
+DS_PROPOSAL_VERSION=v2
+
+DS_CODER_ON_WHOLE_PIPELINE=True
+COSTEER_V2_QUERY_FORMER_TRACE_LIMIT=3
+
+# export PYTHONPATH=.  # this is for running researcher branch;

.env.example

@@ -7,25 +7,53 @@ For more information about configuration options, please refer to the documentat
 
 """
 
+# ==========================================
 # Global configs:
-USE_AZURE=False
-CHAT_USE_AZURE_TOKEN_PROVIDER=False
-EMBEDDING_USE_AZURE_TOKEN_PROVIDER=False
 MAX_RETRY=10
 RETRY_WAIT_SECONDS=20
+# ==========================================
 
-# LLM API Setting:
-OPENAI_API_KEY=<your_api_key>
-CHAT_MODEL=gpt-4-turbo
-CHAT_MAX_TOKENS=3000
-CHAT_TEMPERATURE=0.7
+
+# ==========================================
+# Backend Configuration
+# ==========================================
+# BACKEND=rdagent.oai.backend.LiteLLMAPIBackend
+# ==========================================
+
+# ==========================================
+# Backend Configuration (choose one)
+# ==========================================
+
+# 1. Set universal API key
+# CHAT_MODEL="gpt-4o"
+# EMBEDDING_MODEL="text-embedding-3-small"
+# OPENAI_API_BASE="https://your-endpoint.com/v1"
+# OPENAI_API_KEY="sk-your-api-key-here"
+
+# 2. Set separate API KEY
+# Chat configuration
+OPENAI_API_KEY="sk-chat-key"
+OPENAI_API_BASE="https://xxx-litellm.com/v1"
+CHAT_MODEL='gpt-4o'
+
+# Embedding configuration (using other service)
+# Use siliconflow as example, pay attention to the litellm_proxy prefix
+LITELLM_PROXY_API_KEY="sk-embedding-service-key"
+LITELLM_PROXY_API_BASE="https://api.siliconflow.cn/v1"
+EMBEDDING_MODEL="litellm_proxy/BAAI/bge-large-en-v1.5"
+# ==========================================
+
+# ==========================================
+# Other Configuration 
+# ==========================================
 # CHAT_AZURE_API_BASE=<for_Azure_user>
 # CHAT_AZURE_API_VERSION=<for_Azure_user>
 
-EMBEDDING_MODEL=text-embedding-3-small
 # EMBEDDING_AZURE_API_BASE=<for_Azure_user>
 # EMBEDDING_AZURE_API_VERSION=<for_Azure_user>
 
 # Cache Setting (Optional):
-
-# Senario Configs:
+# USE_CHAT_CACHE=True
+# USE_EMBEDDING_CACHE=True
+# Senario Configs:
+# ==========================================

.gitignore

@@ -112,6 +112,7 @@ celerybeat.pid
 
 # Environments
 .env*
+*.env
 .venv
 ^env/
 venv/
@@ -173,3 +174,7 @@ mlruns/
 /*.sh
 .aider*
 rdagent/app/benchmark/factor/example.json
+
+# UI Server resources
+videos/
+static/

CHANGELOG.md

@@ -1,5 +1,143 @@
 # Changelog
 
+## [0.7.0](https://github.com/microsoft/RD-Agent/compare/v0.6.1...v0.7.0) (2025-07-08)
+
+
+### Features
+
+* add code change summary ([#1000](https://github.com/microsoft/RD-Agent/issues/1000)) ([937ec26](https://github.com/microsoft/RD-Agent/commit/937ec263b215928633822c4d76ad4e47442c8198))
+* add hide_base_name option and update data folder prompts ([#1004](https://github.com/microsoft/RD-Agent/issues/1004)) ([2f61fa8](https://github.com/microsoft/RD-Agent/commit/2f61fa8cd90c91ad29f320ce9ea6c49f49ac9111))
+* added running time statistics for the DS scenario experiment ([#1007](https://github.com/microsoft/RD-Agent/issues/1007)) ([030abd8](https://github.com/microsoft/RD-Agent/commit/030abd87191377641a678c80852f5ecad84e7a6e))
+* merge code summary and support more traces ([#1025](https://github.com/microsoft/RD-Agent/issues/1025)) ([48201e7](https://github.com/microsoft/RD-Agent/commit/48201e79b55ff5a98dad51702a7d0ac6b1ddc9eb))
+* show first evo round codes diff ([#1009](https://github.com/microsoft/RD-Agent/issues/1009)) ([4844622](https://github.com/microsoft/RD-Agent/commit/4844622e5fd28d7cbaabd9d7888f8204c60b76b3))
+* try coder on whole data ([#1017](https://github.com/microsoft/RD-Agent/issues/1017)) ([4973e05](https://github.com/microsoft/RD-Agent/commit/4973e0532248c6172eec3bb70dffda052af2d14f))
+
+
+### Bug Fixes
+
+* fix a minor bug in DS eval ([#1012](https://github.com/microsoft/RD-Agent/issues/1012)) ([5a520e9](https://github.com/microsoft/RD-Agent/commit/5a520e9d44899d44fddc0f2e5571596223161b71))
+* fix some bugs in quant scen ([#1026](https://github.com/microsoft/RD-Agent/issues/1026)) ([7b34d41](https://github.com/microsoft/RD-Agent/commit/7b34d418642d1c0c2986db9ecf6a5d9bc22cc3da))
+* support experimental support for Deepseek models and update docs about configuration ([#1024](https://github.com/microsoft/RD-Agent/issues/1024)) ([35cfc19](https://github.com/microsoft/RD-Agent/commit/35cfc193f9b35d786aeb7585334427ad358c982f))
+
+## [0.6.1](https://github.com/microsoft/RD-Agent/compare/v0.6.0...v0.6.1) (2025-06-28)
+
+
+### Bug Fixes
+
+* fix mount ([#1001](https://github.com/microsoft/RD-Agent/issues/1001)) ([4ae2f13](https://github.com/microsoft/RD-Agent/commit/4ae2f1303dfcbaea53d459be7c8e85bf85ce5f4f))
+* handle the bug of wrong dag_parant index ([#996](https://github.com/microsoft/RD-Agent/issues/996)) ([bda12ff](https://github.com/microsoft/RD-Agent/commit/bda12ffecf9ae116e0d04eece0c6a1b61413d916))
+* improve log folder sorting and selection UX ([#993](https://github.com/microsoft/RD-Agent/issues/993)) ([b116807](https://github.com/microsoft/RD-Agent/commit/b11680777f116b6c40f9e535e0da10c186c95050))
+
+## [0.6.0](https://github.com/microsoft/RD-Agent/compare/v0.5.0...v0.6.0) (2025-06-26)
+
+
+### Features
+
+* async mechanism for multi-trace ([#981](https://github.com/microsoft/RD-Agent/issues/981)) ([9e60c32](https://github.com/microsoft/RD-Agent/commit/9e60c32cf348481eb55617809c059c359d7603b8))
+
+
+### Bug Fixes
+
+* add async to direct_exp_gen avoid infinite loop ([#992](https://github.com/microsoft/RD-Agent/issues/992)) ([78c203d](https://github.com/microsoft/RD-Agent/commit/78c203d8eefbba67fc120b35cb25e85b2200ac49))
+* docker container cleanup to prevent accumulation and system slowdown ([#975](https://github.com/microsoft/RD-Agent/issues/975)) ([05cf094](https://github.com/microsoft/RD-Agent/commit/05cf094913e48c903c8a4476d6c609d8bfa10681))
+* fix a bug and update the docs ([#978](https://github.com/microsoft/RD-Agent/issues/978)) ([d1ae9e1](https://github.com/microsoft/RD-Agent/commit/d1ae9e1dcc2ccd1ffe05cb1c6db3e905fa70425c))
+* merge datascience v3 and v2 ([#974](https://github.com/microsoft/RD-Agent/issues/974)) ([1ba7548](https://github.com/microsoft/RD-Agent/commit/1ba754853ce2010ce1cb0bbd217b67689fa1ebdf))
+* refine details ([#979](https://github.com/microsoft/RD-Agent/issues/979)) ([25caa3d](https://github.com/microsoft/RD-Agent/commit/25caa3d00c255286dce27915b9355987b87ed2e8))
+* refine prompt ([#987](https://github.com/microsoft/RD-Agent/issues/987)) ([76df96e](https://github.com/microsoft/RD-Agent/commit/76df96ee88212a8aee7f518b9cacf80591dc2939))
+
+## [0.5.0](https://github.com/microsoft/RD-Agent/compare/v0.4.0...v0.5.0) (2025-06-18)
+
+
+### Features
+
+* add a check for whether values in score_df are NaN ([#756](https://github.com/microsoft/RD-Agent/issues/756)) ([d9cc780](https://github.com/microsoft/RD-Agent/commit/d9cc78098beb27f3a1bf2f2d461302db177b7d41))
+* add competition level filter and extract constants to utils ([#869](https://github.com/microsoft/RD-Agent/issues/869)) ([b40b605](https://github.com/microsoft/RD-Agent/commit/b40b6055368e6c72d8435352104b1c281b06da7f))
+* add DocDev for auto-generating workspace documentation ([#781](https://github.com/microsoft/RD-Agent/issues/781)) ([bcba6ea](https://github.com/microsoft/RD-Agent/commit/bcba6eac32684ebb267c93b4e85dbfa9561d15d1))
+* add drafting pipeline ([#832](https://github.com/microsoft/RD-Agent/issues/832)) ([efedddf](https://github.com/microsoft/RD-Agent/commit/efedddf39bc19221fdffc2e39ee0a09097fc82b0))
+* add last_exp_fb to DSTrace and update feedback retrieval usage ([#910](https://github.com/microsoft/RD-Agent/issues/910)) ([10531fd](https://github.com/microsoft/RD-Agent/commit/10531fda9438c6915b26d5013bd2413e1333ceb9))
+* add mlflow logger in RD loop to log ([#815](https://github.com/microsoft/RD-Agent/issues/815)) ([b91b54f](https://github.com/microsoft/RD-Agent/commit/b91b54f355c26b751087d0c14774f466e82866de))
+* add naive experiment generator and update proposal configurations ([#759](https://github.com/microsoft/RD-Agent/issues/759)) ([75494f4](https://github.com/microsoft/RD-Agent/commit/75494f4fed5bc845acfd7f7bacef385f0f96c514))
+* add RD-Agent-Quant scenario ([#838](https://github.com/microsoft/RD-Agent/issues/838)) ([6e42d52](https://github.com/microsoft/RD-Agent/commit/6e42d523a85df67aa13927abbf0894564c71880e))
+* add reasoning_effort parameter to LiteLLMAPIBackend and LLMSett… ([#754](https://github.com/microsoft/RD-Agent/issues/754)) ([113889f](https://github.com/microsoft/RD-Agent/commit/113889fefe9b09aaea1b564704c81664b8f77ec5))
+* add reviewer in feedback ([#765](https://github.com/microsoft/RD-Agent/issues/765)) ([1a95bee](https://github.com/microsoft/RD-Agent/commit/1a95bee6aa6bc6f45fdeb484f3a6f81caa273038))
+* advanced checkpoint selectors ([#790](https://github.com/microsoft/RD-Agent/issues/790)) ([50ea033](https://github.com/microsoft/RD-Agent/commit/50ea0336e93d8cb39fb871e81a3f61abdf293bc7))
+* archive python and csv files in workspace to maintain results ([#814](https://github.com/microsoft/RD-Agent/issues/814)) ([67d0e01](https://github.com/microsoft/RD-Agent/commit/67d0e01e7c9237da1371d93cbf9d86f5f46faac4))
+* checkpoint selection ([#744](https://github.com/microsoft/RD-Agent/issues/744)) ([a15a06a](https://github.com/microsoft/RD-Agent/commit/a15a06ad643977db59d7cac9da52e637cf80395a))
+* custom data ([#810](https://github.com/microsoft/RD-Agent/issues/810)) ([6322916](https://github.com/microsoft/RD-Agent/commit/632291608cf605bd8bcfcab0017824823bdecdb8))
+* dump model ([#776](https://github.com/microsoft/RD-Agent/issues/776)) ([b49481e](https://github.com/microsoft/RD-Agent/commit/b49481e073e6f536d2b1b3bd2d01229ed05abdea))
+* enable to set different version of idea-proposal for multi traces ([#895](https://github.com/microsoft/RD-Agent/issues/895)) ([236c28f](https://github.com/microsoft/RD-Agent/commit/236c28f29c6bc5da62129632e464bbc32056ebdb))
+* enhance compatibility with more LLM models ([#905](https://github.com/microsoft/RD-Agent/issues/905)) ([8800624](https://github.com/microsoft/RD-Agent/commit/8800624ad4749d6e798785a082c9f94c306792ef))
+* idea pool integrated to exp_gen & add timer to RD-Agent & pause-resume to RD-loops ([#795](https://github.com/microsoft/RD-Agent/issues/795)) ([e62aefa](https://github.com/microsoft/RD-Agent/commit/e62aefa56e34ff45a8ed033f7bf28b95c8e63656))
+* joblib cache ([#749](https://github.com/microsoft/RD-Agent/issues/749)) ([83a0411](https://github.com/microsoft/RD-Agent/commit/83a041148ff908871b1906f9e6889d80ab513412))
+* log api status to mlflow ([#860](https://github.com/microsoft/RD-Agent/issues/860)) ([049921b](https://github.com/microsoft/RD-Agent/commit/049921beb0b4ed0ba1ab7508d9857d2c1e729349))
+* log reaching max time limit before breaking CoSTEER evolution ([#921](https://github.com/microsoft/RD-Agent/issues/921)) ([837fff2](https://github.com/microsoft/RD-Agent/commit/837fff29096fefe1369d386ef8a860395b737173))
+* merge failed and successful traces together ([#766](https://github.com/microsoft/RD-Agent/issues/766)) ([3a2aa8c](https://github.com/microsoft/RD-Agent/commit/3a2aa8cf0102647950b2dfc0007c118b0c799cd4))
+* merge selectively ([#888](https://github.com/microsoft/RD-Agent/issues/888)) ([06ba314](https://github.com/microsoft/RD-Agent/commit/06ba314ff0f91e7e78e8d456c719ac3194a8c774))
+* multi-trace online merge ([#886](https://github.com/microsoft/RD-Agent/issues/886)) ([2112d67](https://github.com/microsoft/RD-Agent/commit/2112d676d0938de6fea163b2e5eb9c36771e7041))
+* new proposal (structured outputs) prompts ([#887](https://github.com/microsoft/RD-Agent/issues/887)) ([150796a](https://github.com/microsoft/RD-Agent/commit/150796aaa72eaa5037fd7db8e785058fbc4d4967))
+* parallel loop running based on asyncio ([#932](https://github.com/microsoft/RD-Agent/issues/932)) ([c63e207](https://github.com/microsoft/RD-Agent/commit/c63e2071f3179feef69f88061c0172cb5c3157f2))
+* propose hypothesis across multiple parts in pipeline ([#827](https://github.com/microsoft/RD-Agent/issues/827)) ([acb0e21](https://github.com/microsoft/RD-Agent/commit/acb0e21a331410d044849e12e2887f41e5ff1c3a))
+* pull image with progress ([#777](https://github.com/microsoft/RD-Agent/issues/777)) ([5cad086](https://github.com/microsoft/RD-Agent/commit/5cad0860204ede974533dc7bdc9808cfd135fa24))
+* raise error when timeout in api call ([#793](https://github.com/microsoft/RD-Agent/issues/793)) ([eafd4df](https://github.com/microsoft/RD-Agent/commit/eafd4dfc6263f19a8cdaf27498a1d07b43815306))
+* raise policy violation ([#894](https://github.com/microsoft/RD-Agent/issues/894)) ([5b9d007](https://github.com/microsoft/RD-Agent/commit/5b9d0072aebe15369e9a0010af83e71684baeae7))
+* reanalyze competition info & pipeline coding evaluator prompt ([#837](https://github.com/microsoft/RD-Agent/issues/837)) ([f7b5258](https://github.com/microsoft/RD-Agent/commit/f7b52580080c75d311355bcc6193b49495801809))
+* refine merge ([#842](https://github.com/microsoft/RD-Agent/issues/842)) ([99463b4](https://github.com/microsoft/RD-Agent/commit/99463b46819b3a0dcb2bb12a823a9cdf7ec560b4))
+* refine prompt ([#760](https://github.com/microsoft/RD-Agent/issues/760)) ([a91b182](https://github.com/microsoft/RD-Agent/commit/a91b182c4c9510eb34e4aab956588e909fa5d70b))
+* replace hard-coded cache paths with dynamic cache_path config ([#952](https://github.com/microsoft/RD-Agent/issues/952)) ([db56894](https://github.com/microsoft/RD-Agent/commit/db568947f1084a80d603718f5a13fdbd72b90a47))
+* revert draft stage into a soft decay in hypothesis selection ([#849](https://github.com/microsoft/RD-Agent/issues/849)) ([d41db0c](https://github.com/microsoft/RD-Agent/commit/d41db0ca357b07091825ebd9d18c303b6db3cc6a))
+* trace merging ([#836](https://github.com/microsoft/RD-Agent/issues/836)) ([a3d5473](https://github.com/microsoft/RD-Agent/commit/a3d547369e408a05cff570c1239b6320be40418d))
+* truncate by time ([#863](https://github.com/microsoft/RD-Agent/issues/863)) ([2b9427a](https://github.com/microsoft/RD-Agent/commit/2b9427ae036ffe1e28a717502f45500fe91fe5ac))
+* update prompt to improve json respond format of some LLM models ([#928](https://github.com/microsoft/RD-Agent/issues/928)) ([0b84709](https://github.com/microsoft/RD-Agent/commit/0b84709e59c7abb9754961cd17cc9673fcf508aa))
+* using different chat model in different part ([#822](https://github.com/microsoft/RD-Agent/issues/822)) ([c052ea6](https://github.com/microsoft/RD-Agent/commit/c052ea6d1f8948183a4a6ebc873ec01b57373cce))
+
+
+### Bug Fixes
+
+* 'DSProposalV2ExpGen' object has no attribute 'COMPONENT_TASK_MAP… ([#950](https://github.com/microsoft/RD-Agent/issues/950)) ([e353895](https://github.com/microsoft/RD-Agent/commit/e353895251f231fee85abdcb1b22b022a577af77))
+* adapting UI to mock trace ([#841](https://github.com/microsoft/RD-Agent/issues/841)) ([8a5754c](https://github.com/microsoft/RD-Agent/commit/8a5754c9b9c9410d0943aeed777a93c13422e54a))
+* add missing semicolon after chmod in env shell command ([#955](https://github.com/microsoft/RD-Agent/issues/955)) ([1128eaa](https://github.com/microsoft/RD-Agent/commit/1128eaa89ec1dcab4a05ef50d64c7f7e6aae88a8))
+* add time to timer when api timeout bug ([#826](https://github.com/microsoft/RD-Agent/issues/826)) ([f45d6ae](https://github.com/microsoft/RD-Agent/commit/f45d6ae6595c1c39b389485b637a0ae53ffc8782))
+* add wait_retry to exp_gen v2 ([#783](https://github.com/microsoft/RD-Agent/issues/783)) ([b9fb7cf](https://github.com/microsoft/RD-Agent/commit/b9fb7cf4e3070062d91b5b67d0f10d6266b45142))
+* adjust ds_trace lookup and add stderr redirect to mlebench command ([#853](https://github.com/microsoft/RD-Agent/issues/853)) ([4e53108](https://github.com/microsoft/RD-Agent/commit/4e53108e020db719b39cba3a67e0c6dae3de19cf))
+* align competion_full_desc and scenario_all_desc, remove redundant info in problems proposal ([#808](https://github.com/microsoft/RD-Agent/issues/808)) ([76d8536](https://github.com/microsoft/RD-Agent/commit/76d8536d9ec53952383019306781d49cb3e9f75c))
+* bug fix in timer start ([#807](https://github.com/microsoft/RD-Agent/issues/807)) ([9af7161](https://github.com/microsoft/RD-Agent/commit/9af7161eb57bdd2e24b072335e9d185951c32472))
+* bug in problem identification ([#806](https://github.com/microsoft/RD-Agent/issues/806)) ([e1d5a29](https://github.com/microsoft/RD-Agent/commit/e1d5a2914046476f2f10d5884ed3c3ff956d65ff))
+* conda error information ([#941](https://github.com/microsoft/RD-Agent/issues/941)) ([fd39a94](https://github.com/microsoft/RD-Agent/commit/fd39a947763fb4a9be87b907c399bebe384df505))
+* default cost to NaN when calculation fails in LiteLLM backend ([#912](https://github.com/microsoft/RD-Agent/issues/912)) ([51a4048](https://github.com/microsoft/RD-Agent/commit/51a4048129cbfbc3b84bcf50fd8866fafb3e2da3))
+* ds trace ([#929](https://github.com/microsoft/RD-Agent/issues/929)) ([127e441](https://github.com/microsoft/RD-Agent/commit/127e441602e21a46d6313ff39133ab8ca841937e))
+* duplicate model names test in pipeline coder & runner ([#763](https://github.com/microsoft/RD-Agent/issues/763)) ([be3ee9d](https://github.com/microsoft/RD-Agent/commit/be3ee9da9882edda3c06ff7d1099d1bbda2203c3))
+* filter system metadata dirs and init missing DSTrace attribute ([#946](https://github.com/microsoft/RD-Agent/issues/946)) ([10050ef](https://github.com/microsoft/RD-Agent/commit/10050ef368ae7ec07cbf20ac4e52e21c2875eaab))
+* fix a bug in docker result extraction ([#824](https://github.com/microsoft/RD-Agent/issues/824)) ([e1c0f98](https://github.com/microsoft/RD-Agent/commit/e1c0f9826abcbc11dda215a600a2637c9ac6e984))
+* fix competition metric direction ([#784](https://github.com/microsoft/RD-Agent/issues/784)) ([3be0057](https://github.com/microsoft/RD-Agent/commit/3be0057556f46c899065ee1c7f9bafe33e79249c))
+* fix model input shape bug and costeer_model bug ([#821](https://github.com/microsoft/RD-Agent/issues/821)) ([b34bd89](https://github.com/microsoft/RD-Agent/commit/b34bd895d6d9c326aab85856a15be0cb72b2c4c8))
+* fix some minor bugs ([#758](https://github.com/microsoft/RD-Agent/issues/758)) ([963f96e](https://github.com/microsoft/RD-Agent/commit/963f96e5596bee04074135c2a0e31a8adc39ad8c))
+* fix some minor bugs in qlib scenario ([#817](https://github.com/microsoft/RD-Agent/issues/817)) ([79962a7](https://github.com/microsoft/RD-Agent/commit/79962a7ca40c77a3997a68da9ad1b5ab16728483))
+* fix the bug in the regular expression matching for stdout ([#890](https://github.com/microsoft/RD-Agent/issues/890)) ([ee57e37](https://github.com/microsoft/RD-Agent/commit/ee57e37a22af874b262c033d1606dbe7799706db))
+* fix the bug of Exceed-LLM-Context in online merge of multi-tarce ([#892](https://github.com/microsoft/RD-Agent/issues/892)) ([f760a3e](https://github.com/microsoft/RD-Agent/commit/f760a3eff7bd927a31e4958ed2f706312e83e3e3))
+* fix the problems weights bug ([#898](https://github.com/microsoft/RD-Agent/issues/898)) ([013d79f](https://github.com/microsoft/RD-Agent/commit/013d79f12060e908aeb57c3eb1bb56eea86df086))
+* fixed CI execution failures caused by document builds ([#857](https://github.com/microsoft/RD-Agent/issues/857)) ([5c116b2](https://github.com/microsoft/RD-Agent/commit/5c116b24ce727f6ed9ef39d5aa5b60442038c344))
+* get_metric_direction for aerial-cactus-identification ([#970](https://github.com/microsoft/RD-Agent/issues/970)) ([70dc62d](https://github.com/microsoft/RD-Agent/commit/70dc62de5fbd4272ecda1b6fcbcf898b3624a991))
+* import path of T ([#787](https://github.com/microsoft/RD-Agent/issues/787)) ([ac008a6](https://github.com/microsoft/RD-Agent/commit/ac008a61d03b4737ab3d994024e922839d8f3fe1))
+* improve eval alignment check (e.g. small-scale finetuning) ([#802](https://github.com/microsoft/RD-Agent/issues/802)) ([d391578](https://github.com/microsoft/RD-Agent/commit/d3915788082de640a4ce1eea6d2e607319b89c3e))
+* improve file tree and _walk symlink handling ([#877](https://github.com/microsoft/RD-Agent/issues/877)) ([516cb69](https://github.com/microsoft/RD-Agent/commit/516cb69357483ddd99f84b221a056d8491c34f9b))
+* log info ([#965](https://github.com/microsoft/RD-Agent/issues/965)) ([f1dbc21](https://github.com/microsoft/RD-Agent/commit/f1dbc2100498e22c8e5edbb2e4563c99c3d54775))
+* main bug ([#938](https://github.com/microsoft/RD-Agent/issues/938)) ([c6d34d6](https://github.com/microsoft/RD-Agent/commit/c6d34d67b8aedf5496bf6a875915ce657fc58448))
+* non-exist variable test_eval.py ([#847](https://github.com/microsoft/RD-Agent/issues/847)) ([4948c38](https://github.com/microsoft/RD-Agent/commit/4948c38560f4cf021d9354b201b22dfa5ccb9441))
+* refine feedback prompt ([#901](https://github.com/microsoft/RD-Agent/issues/901)) ([12bb2c4](https://github.com/microsoft/RD-Agent/commit/12bb2c4a1494b9aa29962905abb5e433a60eb716))
+* refine the time/memory constraints prompt in hypothesis proposal ([#856](https://github.com/microsoft/RD-Agent/issues/856)) ([51ce8ef](https://github.com/microsoft/RD-Agent/commit/51ce8ef84b4fe6590ce20599a56eee596f2f04e6))
+* Set PYTHONPATH in env.run_ret_code call in FBWorkspace class ([#755](https://github.com/microsoft/RD-Agent/issues/755)) ([68b5018](https://github.com/microsoft/RD-Agent/commit/68b501889caca754f27b57d9ab6f72184e93b15c))
+* task_gen for better understanding ([#752](https://github.com/microsoft/RD-Agent/issues/752)) ([6bfc1e5](https://github.com/microsoft/RD-Agent/commit/6bfc1e570449ee69ac110a4ced9a7cecbc0e6a73))
+* trace list but ([#852](https://github.com/microsoft/RD-Agent/issues/852)) ([32cdc57](https://github.com/microsoft/RD-Agent/commit/32cdc575bde103d71a358d4d99bd413076328ebd))
+* typo in workflow ([#861](https://github.com/microsoft/RD-Agent/issues/861)) ([0e54c9f](https://github.com/microsoft/RD-Agent/commit/0e54c9fe41d25a4cc45ab9e61bb2c2c01b854751))
+* update DS env setup with competition volume and timeout ([#878](https://github.com/microsoft/RD-Agent/issues/878)) ([816ada0](https://github.com/microsoft/RD-Agent/commit/816ada096afabe90578672b0e61b656802a30b62))
+* update feedback.py ([#772](https://github.com/microsoft/RD-Agent/issues/772)) ([133778c](https://github.com/microsoft/RD-Agent/commit/133778c67ee3349f1c2fe029bcf6a9ee14568efe))
+* update metric direction to return bool ([#791](https://github.com/microsoft/RD-Agent/issues/791)) ([0bf365e](https://github.com/microsoft/RD-Agent/commit/0bf365e7830aa86d2350b9d1c47410af46b3a7e8))
+* update runner max loop to 1 in DS scenario ([#820](https://github.com/microsoft/RD-Agent/issues/820)) ([3da378e](https://github.com/microsoft/RD-Agent/commit/3da378e986e8b776a17dbc694d29ef211192ed3e))
+* use fallback messages for missing submission and scores files ([#882](https://github.com/microsoft/RD-Agent/issues/882)) ([898fdea](https://github.com/microsoft/RD-Agent/commit/898fdeae80801d537ebc5c4a3b7df9de74c3403a))
+* use simple stdout and stderr ([#966](https://github.com/microsoft/RD-Agent/issues/966)) ([0b1c445](https://github.com/microsoft/RD-Agent/commit/0b1c445f1f0c212887ffff9f8fac44236df3607c))
+* use trace count as index ([#909](https://github.com/microsoft/RD-Agent/issues/909)) ([b87de56](https://github.com/microsoft/RD-Agent/commit/b87de56e54b206b3aada53850804474eff80b96d))
+* wrong variable test_eval.py ([#846](https://github.com/microsoft/RD-Agent/issues/846)) ([808ea6c](https://github.com/microsoft/RD-Agent/commit/808ea6cba541e60c35dd283cee9098ce46f2a59e))
+
 ## [0.4.0](https://github.com/microsoft/RD-Agent/compare/v0.3.0...v0.4.0) (2025-04-04)
 
 

Makefile

@@ -92,7 +92,7 @@ isort:
 # First deal with the core folder, and then gradually increase the scope of detection,
 # and eventually realize the detection of the complete project.
 mypy:
-	$(PIPRUN) python -m mypy rdagent/core   #  --exclude rdagent/scripts,git_ignore_folder
+	$(PIPRUN) python -m mypy rdagent/core
 
 # Check lint with ruff.
 # First deal with the core folder, and then gradually increase the scope of detection,

README.md

@@ -1,7 +1,11 @@
 <h4 align="center">
   <img src="docs/_static/logo.png" alt="RA-Agent logo" style="width:70%; ">
   
-  <a href="https://rdagent.azurewebsites.net" target="_blank">🖥️ Live Demo</a> | <a href="https://rdagent.azurewebsites.net/factor_loop" target="_blank">🎥 Demo Video</a> <a href="https://www.youtube.com/watch?v=JJ4JYO3HscM&list=PLALmKB0_N3_i52fhUmPQiL4jsO354uopR" target="_blank">▶️YouTube</a>   | <a href="https://rdagent.readthedocs.io/en/latest/index.html" target="_blank">📖 Documentation</a> | <a href="#-paperwork-list"> 📃 Papers </a>
+  <a href="https://rdagent.azurewebsites.net" target="_blank">🖥️ Live Demo</a> |
+  <a href="https://rdagent.azurewebsites.net/factor_loop" target="_blank">🎥 Demo Video</a> <a href="https://www.youtube.com/watch?v=JJ4JYO3HscM&list=PLALmKB0_N3_i52fhUmPQiL4jsO354uopR" target="_blank">▶️YouTube</a>   |
+  <a href="https://rdagent.readthedocs.io/en/latest/index.html" target="_blank">📖 Documentation</a> |
+  <a href="https://aka.ms/RD-Agent-Tech-Report" target="_blank">📄 Tech Report</a> |
+  <a href="#-paperwork-list"> 📃 Papers </a>
 </h3>
 
 
@@ -21,35 +25,76 @@
 [![Chat](https://img.shields.io/badge/chat-discord-blue)](https://discord.gg/ybQ97B6Jjy)
 [![Documentation Status](https://readthedocs.org/projects/rdagent/badge/?version=latest)](https://rdagent.readthedocs.io/en/latest/?badge=latest)
 [![Readthedocs Preview](https://github.com/microsoft/RD-Agent/actions/workflows/readthedocs-preview.yml/badge.svg)](https://github.com/microsoft/RD-Agent/actions/workflows/readthedocs-preview.yml) <!-- this badge is too long, please place it in the last one to make it pretty --> 
+[![arXiv](https://img.shields.io/badge/arXiv-2505.14738-00ff00.svg)](https://arxiv.org/abs/2505.14738)
+
+
+
+# 🏆 The Best Machine Learning Engineering Agent!
+
+[MLE-bench](https://github.com/openai/mle-bench) is a comprehensive benchmark evaluating the performance of AI agents on machine learning engineering tasks. Utilizing datasets from 75 Kaggle competitions, MLE-bench provides robust assessments of AI systems' capabilities in real-world ML engineering scenarios.
+
+R&D-Agent currently leads as the top-performing machine learning engineering agent on MLE-bench:
+
+| Agent | Low == Lite (%) | Medium (%) | High (%) | All (%) |
+|---------|--------|-----------|---------|----------|
+| R&D-Agent o1-preview | 48.18 ± 2.49 | 8.95 ± 2.36 | 18.67 ± 2.98 | 22.4 ± 1.1 |
+| R&D-Agent o3(R)+GPT-4.1(D) | 51.52 ± 6.21 | 7.89 ± 3.33 | 16.67 ± 3.65 | 22.45 ± 2.45 |
+| AIDE o1-preview | 34.3 ± 2.4 | 8.8 ± 1.1 | 10.0 ± 1.9 | 16.9 ± 1.1 |
+
+**Notes:**
+- **O3(R)+GPT-4.1(D)**: This version is designed to both reduce average time per loop and leverage a cost-effective combination of backend LLMs by seamlessly integrating Research Agent (o3) with Development Agent (GPT-4.1).
+- **AIDE o1-preview**: Represents the previously best public result on MLE-bench as reported in the original MLE-bench paper.
+- Average and standard deviation results for R&D-Agent o1-preview is based on a independent of 5 seeds and for R&D-Agent o3(R)+GPT-4.1(D) is based on 6 seeds.
+- According to MLE-Bench, the 75 competitions are categorized into three levels of complexity: **Low==Lite** if we estimate that an experienced ML engineer can produce a sensible solution in under 2 hours, excluding the time taken to train any models; **Medium** if it takes between 2 and 10 hours; and **High** if it takes more than 10 hours.
+
+You can inspect the detailed runs of the above results online.
+- [R&D-Agent o1-preview detailed runs](https://aka.ms/RD-Agent_MLE-Bench_O1-preview)
+- [R&D-Agent o3(R)+GPT-4.1(D) detailed runs](https://aka.ms/RD-Agent_MLE-Bench_O3_GPT41)
+
+For running R&D-Agent on MLE-bench, refer to **[MLE-bench Guide: Running ML Engineering via MLE-bench](https://rdagent.readthedocs.io/en/latest/scens/data_science.html)**
+
+# 🥇 The First Data-Centric Quant Multi-Agent Framework!
+
+R&D-Agent for Quantitative Finance, in short **RD-Agent(Q)**, is the first data-centric, multi-agent framework designed to automate the full-stack research and development of quantitative strategies via coordinated factor-model co-optimization.
+
+![image](https://github.com/user-attachments/assets/3198bc10-47ba-4ee0-8a8e-46d5ce44f45d)
+
+Extensive experiments in real stock markets show that, at a cost under $10, RD-Agent(Q) achieves approximately 2× higher ARR than benchmark factor libraries while using over 70% fewer factors. It also surpasses state-of-the-art deep time-series models under smaller resource budgets. Its alternating factor–model optimization further delivers excellent trade-off between predictive accuracy and strategy robustness.
+
+You can learn more details about **RD-Agent(Q)** through the [paper](https://arxiv.org/abs/2505.15155) and reproduce it through the [documentation](https://rdagent.readthedocs.io/en/latest/scens/quant_agent_fin.html).
+
+# 📰 News
+| 🗞️ News        | 📝 Description                 |
+| --            | ------      |
+| [Technical Report Release](#overall-technical-report) | Overall framework description and results on MLE-bench | 
+| [R&D-Agent-Quant Release](#deep-application-in-diverse-scenarios) | Apply R&D-Agent to quant trading | 
+| MLE-Bench Results Released | R&D-Agent currently leads as the [top-performing machine learning engineering agent](#-the-best-machine-learning-engineering-agent) on MLE-bench |
+| Support LiteLLM Backend | We now fully support **[LiteLLM](https://github.com/BerriAI/litellm)** as a backend for integration with multiple LLM providers. |
+| General Data Science Agent | [Data Science Agent](https://rdagent.readthedocs.io/en/latest/scens/data_science.html) |
+| Kaggle Scenario release | We release **[Kaggle Agent](https://rdagent.readthedocs.io/en/latest/scens/data_science.html)**, try the new features!                  |
+| Official WeChat group release  | We created a WeChat group, welcome to join! (🗪[QR Code](https://github.com/microsoft/RD-Agent/issues/880)) |
+| Official Discord release  | We launch our first chatting channel in Discord (🗪[![Chat](https://img.shields.io/badge/chat-discord-blue)](https://discord.gg/ybQ97B6Jjy)) |
+| First release | **R&D-Agent** is released on GitHub |
+
+
 
 # Data Science Agent Preview
 Check out our demo video showcasing the current progress of our Data Science Agent under development:
 
 https://github.com/user-attachments/assets/3eccbecb-34a4-4c81-bce4-d3f8862f7305
 
-# 📰 News
-| 🗞️ News        | 📝 Description                 |
-| --            | ------      |
-| Support LiteLLM Backend | We now fully support **[LiteLLM](https://github.com/BerriAI/litellm)** as a backend for integration with multiple LLM providers. |
-| More General Data Science Agent | 🚀Coming soon! |
-| Kaggle Scenario release | We release **[Kaggle Agent](https://rdagent.readthedocs.io/en/latest/scens/kaggle_agent.html)**, try the new features!                  |
-| Official WeChat group release  | We created a WeChat group, welcome to join! (🗪[QR Code](docs/WeChat_QR_code.jpg)) |
-| Official Discord release  | We launch our first chatting channel in Discord (🗪[![Chat](https://img.shields.io/badge/chat-discord-blue)](https://discord.gg/ybQ97B6Jjy)) |
-| First release | **RDAgent** is released on GitHub |
-
-
 # 🌟 Introduction
 <div align="center">
       <img src="docs/_static/scen.png" alt="Our focused scenario" style="width:80%; ">
 </div>
 
-RDAgent aims to automate the most critical and valuable aspects of the industrial R&D process, and we begin with focusing on the data-driven scenarios to streamline the development of models and data. 
+R&D-Agent aims to automate the most critical and valuable aspects of the industrial R&D process, and we begin with focusing on the data-driven scenarios to streamline the development of models and data. 
 Methodologically, we have identified a framework with two key components: 'R' for proposing new ideas and 'D' for implementing them.
 We believe that the automatic evolution of R&D will lead to solutions of significant industrial value.
 
 
 <!-- Tag Cloud -->
-R&D is a very general scenario. The advent of RDAgent can be your
+R&D is a very general scenario. The advent of R&D-Agent can be your
 - 💰 **Automatic Quant Factory** ([🎥Demo Video](https://rdagent.azurewebsites.net/factor_loop)|[▶️YouTube](https://www.youtube.com/watch?v=X4DK2QZKaKY&t=6s))
 - 🤖 **Data Mining Agent:** Iteratively proposing data & models ([🎥Demo Video 1](https://rdagent.azurewebsites.net/model_loop)|[▶️YouTube](https://www.youtube.com/watch?v=dm0dWL49Bc0&t=104s)) ([🎥Demo Video 2](https://rdagent.azurewebsites.net/dmm)|[▶️YouTube](https://www.youtube.com/watch?v=VIaSTZuoZg4))  and implementing them by gaining knowledge from data.
 - 🦾 **Research Copilot:** Auto read research papers ([🎥Demo Video](https://rdagent.azurewebsites.net/report_model)|[▶️YouTube](https://www.youtube.com/watch?v=BiA2SfdKQ7o)) / financial reports ([🎥Demo Video](https://rdagent.azurewebsites.net/report_factor)|[▶️YouTube](https://www.youtube.com/watch?v=ECLTXVcSx-c)) and implement model structures or building datasets.
@@ -85,8 +130,8 @@ Ensure the current user can run Docker commands **without using sudo**. You can
   conda activate rdagent
   ```
 
-### 🛠️ Install the RDAgent
-- You can directly install the RDAgent package from PyPI:
+### 🛠️ Install the R&D-Agent
+- You can directly install the R&D-Agent package from PyPI:
   ```sh
   pip install rdagent
   ```
@@ -106,46 +151,74 @@ Ensure the current user can run Docker commands **without using sudo**. You can
   - json_mode
   - embedding query
 
-- For example: If you are using the `OpenAI API`, you have to configure your GPT model in the `.env` file like this.
+  You can set your Chat Model and Embedding Model in the following ways:
+
+  > **🔥 Attention**: We now provide experimental support for **DeepSeek** models! You can use DeepSeek's official API for cost-effective and high-performance inference. See the configuration example below for DeepSeek setup.
+
+- **Using LiteLLM (Default)**: We now support LiteLLM as a backend for integration with multiple LLM providers. You can configure in multiple ways:
+
+  **Option 1: Unified API base for both models**
   ```bash
   cat << EOF  > .env
-  OPENAI_API_KEY=<replace_with_your_openai_api_key>
-  # EMBEDDING_MODEL=text-embedding-3-small
-  CHAT_MODEL=gpt-4-turbo
-  EOF
-  ```
-- However, not every API services support these features by default. For example: `AZURE OpenAI`, you have to configure your GPT model in the `.env` file like this.
-  ```bash
-  cat << EOF  > .env
-  USE_AZURE=True
-  EMBEDDING_OPENAI_API_KEY=<replace_with_your_azure_openai_api_key>
-  EMBEDDING_AZURE_API_BASE=<replace_with_your_azure_endpoint>
-  EMBEDDING_AZURE_API_VERSION=<replace_with_the_version_of_your_azure_openai_api>
+  # Set to any model supported by LiteLLM.
+  CHAT_MODEL=gpt-4o 
   EMBEDDING_MODEL=text-embedding-3-small
-  CHAT_OPENAI_API_KEY=<replace_with_your_azure_openai_api_key>
-  CHAT_AZURE_API_BASE=<replace_with_your_azure_endpoint>
-  CHAT_AZURE_API_VERSION=<replace_with_the_version_of_your_azure_openai_api>
-  CHAT_MODEL=<replace_it_with_the_name_of_your_azure_chat_model>
-  EOF
+  # Configure unified API base
+  OPENAI_API_BASE=<your_unified_api_base>
+  OPENAI_API_KEY=<replace_with_your_openai_api_key>
   ```
 
-- We now support LiteLLM as a backend for integration with multiple LLM providers. If you use LiteLLM Backend to use models, you can configure as follows:
+  **Option 2: Separate API bases for Chat and Embedding models**
   ```bash
   cat << EOF  > .env
-  BACKEND=rdagent.oai.backend.LiteLLMAPIBackend
-  # It can be modified to any model supported by LiteLLM.
-  CHAT_MODEL=gpt-4o
-  EMBEDDING_MODEL=text-embedding-3-small
-  # The backend api_key fully follow the convention of litellm.
-  OPENAI_API_KEY=<replace_with_your_openai_api_key>
-  ```
+  # Set to any model supported by LiteLLM.
+  # Configure separate API bases for chat and embedding
   
-- For more configuration information, please refer to the [documentation](https://rdagent.readthedocs.io/en/latest/installation_and_configuration.html).
+  # CHAT MODEL:
+  CHAT_MODEL=gpt-4o 
+  OPENAI_API_BASE=<your_chat_api_base>
+  OPENAI_API_KEY=<replace_with_your_openai_api_key>
+
+  # EMBEDDING MODEL:
+  # TAKE siliconflow as an example, you can use other providers.
+  # Note: embedding requires litellm_proxy prefix
+  EMBEDDING_MODEL=litellm_proxy/BAAI/bge-large-en-v1.5
+  LITELLM_PROXY_API_KEY=<replace_with_your_siliconflow_api_key>
+  LITELLM_PROXY_API_BASE=https://api.siliconflow.cn/v1
+  ```
+
+  **Configuration Example: DeepSeek Setup**:
+
+  >Since many users encounter configuration errors when setting up DeepSeek. Here's a complete working example for DeepSeek Setup:
+  ```bash
+  cat << EOF  > .env
+  # CHAT MODEL: Using DeepSeek Official API
+  CHAT_MODEL=deepseek/deepseek-chat 
+  DEEPSEEK_API_KEY=<replace_with_your_deepseek_api_key>
+
+  # EMBEDDING MODEL: Using SiliconFlow for embedding since deepseek has no embedding model.
+  # Note: embedding requires litellm_proxy prefix
+  EMBEDDING_MODEL=litellm_proxy/BAAI/bge-m3
+  LITELLM_PROXY_API_KEY=<replace_with_your_siliconflow_api_key>
+  LITELLM_PROXY_API_BASE=https://api.siliconflow.cn/v1
+  ```
+
+  Notice: If you are using reasoning models that include thought processes in their responses (such as \<think> tags), you need to set the following environment variable:
+  ```bash
+  REASONING_THINK_RM=True
+  ```
+
+- You can also use a deprecated backend if you only use `OpenAI API` or `Azure OpenAI` directly. For this deprecated setting and more configuration information, please refer to the [documentation](https://rdagent.readthedocs.io/en/latest/installation_and_configuration.html).
 
 ### 🚀 Run the Application
 
 The **[🖥️ Live Demo](https://rdagent.azurewebsites.net/)** is implemented by the following commands(each item represents one demo, you can select the one you prefer):
 
+- Run the **Automated Quantitative Trading & Iterative Factors Model Joint Evolution**:  [Qlib](http://github.com/microsoft/qlib) self-loop factor & model proposal and implementation application
+  ```sh
+  rdagent fin_quant
+  ```
+
 - Run the **Automated Quantitative Trading & Iterative Factors Evolution**:  [Qlib](http://github.com/microsoft/qlib) self-loop factor proposal and implementation application
   ```sh
   rdagent fin_factor
@@ -156,19 +229,6 @@ The **[🖥️ Live Demo](https://rdagent.azurewebsites.net/)** is implemented b
   rdagent fin_model
   ```
 
-- Run the **Automated Medical Prediction Model Evolution**: Medical self-loop model proposal and implementation application
-  >(1) Apply for an account at [PhysioNet](https://physionet.org/). <br /> (2) Request access to FIDDLE preprocessed data: [FIDDLE Dataset](https://physionet.org/content/mimic-eicu-fiddle-feature/1.0.0/). <br />
-  (3) Place your username and password in `.env`.
-  ```bash
-  cat << EOF  >> .env
-  DM_USERNAME=<your_username>
-  DM_PASSWORD=<your_password>
-  EOF
-  ```
-  ```sh
-  rdagent med_model
-  ```
-
 - Run the **Automated Quantitative Trading & Factors Extraction from Financial Reports**:  Run the [Qlib](http://github.com/microsoft/qlib) factor extraction and implementation application based on financial reports
   ```sh
   # 1. Generally, you can run this scenario using the following command:
@@ -199,24 +259,20 @@ The **[🖥️ Live Demo](https://rdagent.azurewebsites.net/)** is implemented b
   > 3. Join the competition: Click `Join the competition` -> `I Understand and Accept` at the bottom of the [competition details page](https://www.kaggle.com/competitions/sf-crime/data).
   ```bash
   # Generally, you can run the Kaggle competition program with the following command:
-  rdagent kaggle --competition <your competition name>
+  rdagent data_science --competition <your competition name>
+
+  # Specifically, you need to create a folder for storing competition files (e.g., competition description file, competition datasets, etc.), and configure the path to the folder in your environment. In addition, you need to use chromedriver when you download the competition descriptors, which you can follow for this specific example:
   
-  # Specifically, you will need to first prepare some competition description files and configure the competition description file path, which you can follow for this specific example:
-  
-  # 1. Prepare the competition description files
-  wget https://github.com/SunsetWolf/rdagent_resource/releases/download/kaggle_data/kaggle_data.zip
-  unzip kaggle_data.zip -d git_ignore_folder/kaggle_data
+  # 1. Install chromedriver.
 
   # 2. Add the competition description file path to the `.env` file.
-  dotenv set KG_LOCAL_DATA_PATH "$(pwd)/git_ignore_folder/kaggle_data"
+  mkdir -p ./git_ignore_folder/kaggle_data
+  dotenv set DS_LOCAL_DATA_PATH "$(pwd)/git_ignore_folder/kaggle_data"
+  dotenv set DS_IF_USING_MLE_DATA True
 
   # 3. run the application
-  rdagent kaggle --competition sf-crime
+  rdagent data_science --competition sf-crime
   ```
-  > **Description of the above example:** <br />
-  > - Kaggle competition data, contains two parts: competition description file (json file) and competition dataset (zip file). We prepare the competition description file for you, the competition dataset will be downloaded automatically when you run the program, as in the example. <br />
-  > - If you want to download the competition description file automatically, you need to install chromedriver, The instructions for installing chromedriver can be found in the [documentation](https://rdagent.readthedocs.io/en/latest/scens/kaggle_agent.html#example-guide). <br />
-  > - The **Competition List Available** can be found [here](https://rdagent.readthedocs.io/en/latest/scens/kaggle_agent.html#competition-list-available). <br />
 
 ### 🖥️ Monitor the Application Results
 - You can run the following command for our demo program to see the run logs.
@@ -235,7 +291,7 @@ The **[🖥️ Live Demo](https://rdagent.azurewebsites.net/)** is implemented b
 
 # 🏭 Scenarios
 
-We have applied RD-Agent to multiple valuable data-driven industrial scenarios.
+We have applied R&D-Agent to multiple valuable data-driven industrial scenarios.
 
 
 ## 🎯 Goal: Agent for Data-driven R&D
@@ -262,7 +318,7 @@ The supported scenarios are listed below:
 | **🩺 Medical**      | 🤖 [Iteratively Proposing Ideas & Evolving](https://rdagent.azurewebsites.net/dmm)[▶️YouTube](https://www.youtube.com/watch?v=VIaSTZuoZg4) | -                                                                                  |
 | **🏭 General**      | 🦾 [Auto paper reading & implementation](https://rdagent.azurewebsites.net/report_model)[▶️YouTube](https://www.youtube.com/watch?v=BiA2SfdKQ7o) <br/> 🤖 Auto Kaggle Model Tuning   | 🤖Auto Kaggle feature Engineering |
 
-- **[RoadMap](https://rdagent.readthedocs.io/en/latest/scens/kaggle_agent.html#roadmap)**: Currently, we are working hard to add new features to the Kaggle scenario.
+- **[RoadMap](https://rdagent.readthedocs.io/en/latest/scens/data_science.html#roadmap)**: Currently, we are working hard to add new features to the Kaggle scenario.
 
 Different scenarios vary in entrance and configuration. Please check the detailed setup tutorial in the scenarios documents.
 
@@ -292,6 +348,21 @@ More documents can be found in the **[📖 readthedocs](https://rdagent.readthed
 
 # 📃 Paper/Work list
 
+## Overall Technical Report
+- [R&D-Agent: Automating Data-Driven AI Solution Building Through LLM-Powered Automated Research, Development, and Evolution](https://arxiv.org/abs/2505.14738)
+```BibTeX
+@misc{yang2024rdagent,
+    title={R\&D-Agent: Automating Data-Driven AI Solution Building Through LLM-Powered Automated Research, Development, and Evolution},
+    author={Xu Yang and Xiao Yang and Shikai Fang and Bowen Xian and Yuante Li and Jian Wang and Minrui Xu and Haoran Pan and Xinpeng Hong and Weiqing Liu and Yelong Shen and Weizhu Chen and Jiang Bian},
+    year={2025},
+    eprint={2505.14738},
+    archivePrefix={arXiv},
+    primaryClass={cs.AI},
+    url={https://arxiv.org/abs/2505.14738}
+}
+```
+![image](https://github.com/user-attachments/assets/28b0488d-a546-4fef-8dc5-563ed64a9b4d)
+
 ## 📊 Benchmark
 - [Towards Data-Centric Automatic R&D](https://arxiv.org/abs/2404.11276)
 ```BibTeX
@@ -329,16 +400,31 @@ For more detail, please refer to our **[🖥️ Live Demo page](https://rdagent.
 ```
 ![image](https://github.com/user-attachments/assets/75d9769b-0edd-4caf-9d45-57d1e577054b)
 
+## Deep Application in Diverse Scenarios
+
+- [R&D-Agent-Quant: A Multi-Agent Framework for Data-Centric Factors and Model Joint Optimization](https://arxiv.org/abs/2505.15155)
+```BibTeX
+@misc{li2025rdagentquant,
+    title={R\&D-Agent-Quant: A Multi-Agent Framework for Data-Centric Factors and Model Joint Optimization},
+    author={Yuante Li and Xu Yang and Xiao Yang and Minrui Xu and Xisen Wang and Weiqing Liu and Jiang Bian},
+    year={2025},
+    eprint={2505.15155},
+    archivePrefix={arXiv},
+    primaryClass={cs.AI}
+}
+```
+![image](https://github.com/user-attachments/assets/3186f67a-c2f8-4b6b-8bb9-a9b959c13866)
+
 
 # 🤝 Contributing
 
-We welcome contributions and suggestions to improve RD-Agent. Please refer to the [Contributing Guide](CONTRIBUTING.md) for more details on how to contribute.
+We welcome contributions and suggestions to improve R&D-Agent. Please refer to the [Contributing Guide](CONTRIBUTING.md) for more details on how to contribute.
 
 Before submitting a pull request, ensure that your code passes the automatic CI checks.
 
 ## 📝 Guidelines
 This project welcomes contributions and suggestions.
-Contributing to this project is straightforward and rewarding. Whether it's solving an issue, addressing a bug, enhancing documentation, or even correcting a typo, every contribution is valuable and helps improve RDAgent.
+Contributing to this project is straightforward and rewarding. Whether it's solving an issue, addressing a bug, enhancing documentation, or even correcting a typo, every contribution is valuable and helps improve R&D-Agent.
 
 To get started, you can explore the issues list, or search for `TODO:` comments in the codebase by running the command `grep -r "TODO:"`.
 
@@ -348,7 +434,7 @@ To get started, you can explore the issues list, or search for `TODO:` comments
   <img src="https://contrib.rocks/image?repo=microsoft/RD-Agent&max=100&columns=15" />
 </a>
 
-Before we released RD-Agent as an open-source project on GitHub, it was an internal project within our group. Unfortunately, the internal commit history was not preserved when we removed some confidential code. As a result, some contributions from our group members, including Haotian Chen, Wenjun Feng, Haoxue Wang, Zeqi Ye, Xinjie Shen, and Jinhui Li, were not included in the public commits.
+Before we released R&D-Agent as an open-source project on GitHub, it was an internal project within our group. Unfortunately, the internal commit history was not preserved when we removed some confidential code. As a result, some contributions from our group members, including Haotian Chen, Wenjun Feng, Haoxue Wang, Zeqi Ye, Xinjie Shen, and Jinhui Li, were not included in the public commits.
 
 # ⚖️ Legal disclaimer
 <p style="line-height: 1; font-style: italic;">The RD-agent is provided “as is”, without warranty of any kind, express or implied, including but not limited to the warranties of merchantability, fitness for a particular purpose and noninfringement. The RD-agent is aimed to facilitate research and development process in the financial industry and not ready-to-use for any financial investment or advice. Users shall independently assess and test the risks of the RD-agent in a specific use scenario, ensure the responsible use of AI technology, including but not limited to developing and integrating risk mitigation measures, and comply with all applicable laws and regulations in all applicable jurisdictions. The RD-agent does not provide financial opinions or reflect the opinions of Microsoft, nor is it designed to replace the role of qualified financial professionals in formulating, assessing, and approving finance products. The inputs and outputs of the RD-agent belong to the users and users shall assume all liability under any theory of liability, whether in contract, torts, regulatory, negligence, products liability, or otherwise, associated with use of the RD-agent and any inputs and outputs thereof.</p>

constraints/3.10.txt

@@ -5,3 +5,4 @@ psutil==6.1.0
 rich==13.9.2
 scipy==1.14.1
 tqdm==4.66.5
+litellm==1.72.4

constraints/3.11.txt

@@ -5,3 +5,4 @@ psutil==6.1.0
 rich==13.9.2
 scipy==1.14.1
 tqdm==4.66.5
+litellm==1.72.4

docs/installation_and_configuration.rst

@@ -13,30 +13,78 @@ Installation
 **Install Docker**: RDAgent is designed for research and development, acting like a human researcher and developer. It can write and run code in various environments, primarily using Docker for code execution. This keeps the remaining dependencies simple. Users must ensure Docker is installed before attempting most scenarios. Please refer to the `official 🐳Docker page <https://docs.docker.com/engine/install/>`_ for installation instructions.
 Ensure the current user can run Docker commands **without using sudo**. You can verify this by executing `docker run hello-world`.
 
-LiteLLM Backend Configuration
-=============================
+LiteLLM Backend Configuration (Default)
+=======================================
 
-Please create a `.env` file in the root directory of the project and add environment variables.
+.. note::
+   🔥 **Attention**: We now provide experimental support for **DeepSeek** models! You can use DeepSeek's official API for cost-effective and high-performance inference. See the configuration example below for DeepSeek setup.
 
-Here is a sample configuration for using OpenAI's gpt-4o via LiteLLM. 
+Option 1: Unified API base for both models
+------------------------------------------
 
    .. code-block:: Properties
 
-      BACKEND=rdagent.oai.backend.LiteLLMAPIBackend
-      # It can be modified to any model supported by LiteLLM.
-      CHAT_MODEL=gpt-4o
+      # Set to any model supported by LiteLLM.
+      CHAT_MODEL=gpt-4o 
       EMBEDDING_MODEL=text-embedding-3-small
+      # Configure unified API base
       # The backend api_key fully follows the convention of litellm.
+      OPENAI_API_BASE=<your_unified_api_base>
       OPENAI_API_KEY=<replace_with_your_openai_api_key>
 
+Option 2: Separate API bases for Chat and Embedding models
+----------------------------------------------------------
+
+   .. code-block:: Properties
+
+      # Set to any model supported by LiteLLM.
+      
+      # CHAT MODEL:
+      CHAT_MODEL=gpt-4o 
+      OPENAI_API_BASE=<your_chat_api_base>
+      OPENAI_API_KEY=<replace_with_your_openai_api_key>
+
+      # EMBEDDING MODEL:
+      # TAKE siliconflow as an example, you can use other providers.
+      # Note: embedding requires litellm_proxy prefix
+      EMBEDDING_MODEL=litellm_proxy/BAAI/bge-large-en-v1.5
+      LITELLM_PROXY_API_KEY=<replace_with_your_siliconflow_api_key>
+      LITELLM_PROXY_API_BASE=https://api.siliconflow.cn/v1
+
+Configuration Example: DeepSeek Setup
+-------------------------------------
+
+Many users encounter configuration errors when setting up DeepSeek. Here's a complete working example:
+
+   .. code-block:: Properties
+
+      # CHAT MODEL: Using DeepSeek Official API
+      CHAT_MODEL=deepseek/deepseek-chat 
+      DEEPSEEK_API_KEY=<replace_with_your_deepseek_api_key>
+
+      # EMBEDDING MODEL: Using SiliconFlow for embedding since DeepSeek has no embedding model.
+      # Note: embedding requires litellm_proxy prefix
+      EMBEDDING_MODEL=litellm_proxy/BAAI/bge-m3
+      LITELLM_PROXY_API_KEY=<replace_with_your_siliconflow_api_key>
+      LITELLM_PROXY_API_BASE=https://api.siliconflow.cn/v1
+
 Necessary parameters include:
 
-- `BACKEND`: The backend to use. The default is `rdagent.oai.backend.DeprecBackend`. To use the LiteLLM backend, set it to `rdagent.oai.backend.LiteLLMAPIBackend`.
-
-- `CHAT_MODEL`: The model name of the chat model. 
+- `CHAT_MODEL`: The model name of the chat model.
 
 - `EMBEDDING_MODEL`: The model name of the embedding model.
 
+- `OPENAI_API_BASE`: The base URL of the API. If `EMBEDDING_MODEL` does not start with `litellm_proxy/`, this is used for both chat and embedding models; otherwise, it is used for `CHAT_MODEL` only.
+
+Optional parameters (required if your embedding model is provided by a different provider than `CHAT_MODEL`):
+
+- `LITELLM_PROXY_API_KEY`: The API key for the embedding model, required if `EMBEDDING_MODEL` starts with `litellm_proxy/`.
+
+- `LITELLM_PROXY_API_BASE`: The base URL for the embedding model, required if `EMBEDDING_MODEL` starts with `litellm_proxy/`.
+
+**Note:** If you are using an embedding model from a provider different from the chat model, remember to add the `litellm_proxy/` prefix to the `EMBEDDING_MODEL` name.
+
+
 The `CHAT_MODEL` and `EMBEDDING_MODEL` parameters will be passed into LiteLLM's completion function. 
 
 Therefore, when utilizing models provided by different providers, first review the interface configuration of LiteLLM. The model names must match those allowed by LiteLLM.
@@ -51,6 +99,12 @@ For example, if you are using a DeepSeek model, you need to set as follows:
       CHAT_MODEL=deepseek/deepseek-chat
       DEEPSEEK_API_KEY=<replace_with_your_deepseek_api_key>
 
+Besides, when you are using reasoning models, the response might include the thought process. For this case, you need to set the following environment variable:
+   
+   .. code-block:: Properties
+      
+      REASONING_THINK_RM=True
+
 For more details on LiteLLM requirements, refer to the `official LiteLLM documentation <https://docs.litellm.ai/docs>`_.
 
 
@@ -59,7 +113,7 @@ Configuration(deprecated)
 
 To run the application, please create a `.env` file in the root directory of the project and add environment variables according to your requirements.
 
-The standard configuration options for the user using the OpenAI API are provided in the `.env.example` file.
+If you are using this deprecated version,  you should set `BACKEND` to `rdagent.oai.backend.DeprecBackend`.
 
 Here are some other configuration options that you can use:
 

docs/scens/catalog.rst

@@ -13,26 +13,23 @@ In the two key areas of data-driven scenarios, model implementation and data bui
 The supported scenarios are listed below:
 
 
-
-.. list-table:: 
+.. list-table::
     :header-rows: 1
 
     * - Scenario/Target
       - Model Implementation
       - Data Building
     * - 💹 Finance
-      - :ref:`🤖Iteratively Proposing Ideas & Evolving <model_agent_fin>`
-      - :ref:`🦾Auto reports reading & implementation <data_copilot_fin>`
-
+      - :ref:`🥇The First Data-Centric Quant Multi-Agent Framework <quant_agent_fin>`
+      - :ref:`🤖Iteratively Proposing Ideas & Evolving <model_agent_fin>`  
+        
+        :ref:`🦾Auto reports reading & implementation <data_copilot_fin>`  
+        
         :ref:`🤖Iteratively Proposing Ideas & Evolving <data_agent_fin>`
-    * - 🩺 Medical
-      - :ref:`🤖Iteratively Proposing Ideas & Evolving <model_agent_med>`
-      - 
     * - 🏭 General
-      - :ref:`🦾Auto paper reading & implementation <model_copilot_general>`
-
-        :ref:`🤖Auto Kaggle Model Tuning <kaggle_agent>`
-      - :ref:`🤖Auto Kaggle feature Engineering <kaggle_agent>`
+      - :ref:`🦾Auto paper reading & implementation <model_copilot_general>`  
+        
+      - :ref:`🤖 Data Science <data_science_agent>`
 
 
 .. toctree::
@@ -40,10 +37,9 @@ The supported scenarios are listed below:
     :caption: Doctree:
     :hidden:
 
+    quant_agent_fin
     data_agent_fin
     data_copilot_fin
     model_agent_fin
-    model_agent_med
     model_copilot_general
-    kaggle_agent
-
+    data_science

docs/scens/data_science.rst

@@ -0,0 +1,184 @@
+.. _data_science_agent:
+
+=======================
+Data Science Agent
+=======================
+
+**🤖 Automated Feature Engineering & Model Tuning Evolution**
+------------------------------------------------------------------------------------------
+The Data Science Agent is an agent that can automatically perform feature engineering and model tuning. It can be used to solve various data science problems, such as image classification, time series forecasting, and text classification.
+
+🧭 Example Guide
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+- 🔧 **Set up RD-Agent Environment**
+
+  - Before you start, please make sure you have installed RD-Agent and configured the environment for RD-Agent correctly. If you want to know how to install and configure the RD-Agent, please refer to the `documentation <../installation_and_configuration.html>`_.
+
+- 🔩 **Setting the Environment variables at .env file**
+
+  - Determine the path where the data will be stored and add it to the ``.env`` file.
+
+  .. code-block:: sh
+
+    dotenv set DS_LOCAL_DATA_PATH <your local directory>/ds_data
+    dotenv set DS_SCEN rdagent.scenarios.data_science.scen.DataScienceScen
+
+- 📥 **Prepare Competition Data**
+
+  - Data Science competition data typically consists of three components: a competition description file (in Markdown format), the competition dataset, and evaluation scripts. For reference, an example of a custom user-defined dataset is provided in ``rdagent/scenarios/data_science/example``.
+
+    - **Correct directory structure (Here is an example of competition data with id custom_data)**
+
+      .. code-block:: text
+
+        ds_data
+        └── eval
+        | └── custom_data
+        |    └── grade.py
+        |    └── valid.py
+        |    └── test.csv
+        └── custom_data
+          └── train.csv
+          └── test.csv
+          └── sample_submission.csv
+          └── description.md
+          └── sample.py
+        
+      - ``ds_data/custom_data/train.csv:`` Necessary training data in csv or parquet format, or training images.
+
+      - ``ds_data/custom_data/description.md:`` (Optional) Competition description file.
+
+      - ``ds_data/custom_data/sample_submission.csv:`` (Optional) Competition sample submission file.
+
+      - ``ds_data/custom_data/sample.py:`` (Optional) Sample code for generating debug data from the competition dataset. If not provided, R&D-Agent will use its default sampling logic. For details, see the ``create_debug_data`` function in ``rdagent/scenarios/data_science/debug/data.py``.
+
+      - ``ds_data/eval/custom_data/grade.py:`` (Optional) Competition grade script, in order to calculate the score for the submission.
+
+      - ``ds_data/eval/custom_data/valid.py:`` (Optional) Competition validation script, in order to check if the submission format is correct.
+
+      - ``ds_data/eval/custom_data/submission_test.csv:`` (Optional) Competition test label file.
+
+- 🔧 **Set up Environment for Custom User-defined Dataset**
+
+  .. code-block:: sh
+
+      dotenv set DS_SCEN rdagent.scenarios.data_science.scen.DataScienceScen
+      dotenv set DS_LOCAL_DATA_PATH rdagent/scenarios/data_science/example
+      dotenv set DS_IF_USING_MLE_DATA False
+      dotenv set DS_CODER_ON_WHOLE_PIPELINE True
+      dotenv set DS_CODER_COSTEER_ENV_TYPE docker
+
+- 🚀 **Run the Application**
+
+  - You can directly run the application by using the following command:
+    
+    .. code-block:: sh
+
+        rdagent data_science --competition <Competition ID>
+
+  - Then, you can run the test set score corresponding to each round of the loop.
+
+    .. code-block:: sh
+
+        dotenv run -- python rdagent/log/mle_summary.py grade <url_to_log>
+
+    Here, <url_to_log> refers to the parent directory of the log folder generated during the run.
+
+- 📥 **Visualize the R&D Process**
+
+  - We provide a web UI to visualize the log. You just need to run:
+
+    .. code-block:: sh
+
+        streamlit run rdagent/log/ui/dsapp.py
+
+  - Then you can input the log path and visualize the R&D process.
+
+🔍 MLE-bench Guide: Running ML Engineering via MLE-bench
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+- 📝 **MLE-bench Overview**
+
+  - MLE-bench is a comprehensive benchmark designed to evaluate the ML engineering capabilities of AI systems using real-world scenarios. The dataset comprises 75 Kaggle competitions. Since Kaggle does not provide held-out test sets for these competitions, the benchmark includes preparation scripts that split the publicly available training data into new training and test sets, and grading scripts are provided for each competition to accurately evaluate submission scores.
+
+- 🔧 **Set up Environment for MLE-bench**
+
+  - Running R&D-Agent on MLE-bench is designed for full automation. There is no need for manual downloads and data preparation. Simply set the environment variable ``DS_IF_USING_MLE_DATA`` to True.  
+
+  - At runtime, R&D-Agent will automatically build the Docker image specified at ``rdagent/scenarios/kaggle/docker/mle_bench_docker/Dockerfile``. This image is responsible for downloading the required datasets and grading files for MLE-bench.  
+  
+  - Note: The first run may take longer than subsequent runs as the Docker image and data are being downloaded and set up for the first time.
+
+    .. code-block:: sh
+
+        dotenv set DS_LOCAL_DATA_PATH <your local directory>/ds_data
+        dotenv set DS_IF_USING_MLE_DATA True
+
+- 🔨 **Configuring the Kaggle API**
+
+  - Downloading Kaggle competition data requires the Kaggle API. You can set up the Kaggle API by following these steps:
+  
+    - Register and login on the `Kaggle <https://www.kaggle.com/>`_ website.
+
+    - Click on the avatar (usually in the top right corner of the page) -> ``Settings`` -> ``Create New Token``, A file called ``kaggle.json`` will be downloaded.
+
+    - Move ``kaggle.json`` to ``~/.config/kaggle/``
+
+    - Modify the permissions of the ``kaggle.json`` file.
+
+      .. code-block:: sh
+
+        chmod 600 ~/.config/kaggle/kaggle.json
+
+  - For more information about Kaggle API Settings, refer to the `Kaggle API <https://github.com/Kaggle/kaggle-api>`_.
+
+
+- 🔩 **Setting the Environment Variables for MLE-bench**
+
+  - In addition to auto-downloading the benchmark data, you must also configure the runtime environment for executing the competition code.  
+  - Use the environment variable ``DS_CODER_COSTEER_ENV_TYPE`` to select the execution mode:
+    
+    • When set to docker (the default), RD-Agent utilizes the official Kaggle Docker image (``gcr.io/kaggle-gpu-images/python:latest``) to ensure that all required packages are available.  
+    • If you prefer to use a custom Docker setup, you can modify the configuration using ``DS_DOCKER_IMAGE`` or ``DS_DOCKERFILE_FOLDER_PATH``.  
+    • Alternatively, if your competition work only demands basic libraries, you may set ``DS_CODER_COSTEER_ENV_TYPE`` to conda. In this mode, you must create a local conda environment named “kaggle” and pre-install the necessary packages. RD-Agent will execute the competition code within this “kaggle” conda environment.
+
+    .. code-block:: sh
+
+      # Configure the runtime environment: choice between 'docker' (default) or 'conda'
+      dotenv set DS_CODER_COSTEER_ENV_TYPE docker
+
+- 🚀 **Run the Application**
+
+  - You can directly run the application by using the following command:
+    
+    .. code-block:: sh
+
+        rdagent data_science --competition <Competition ID>
+
+- 📥 **Visualize the R&D Process**
+
+  - We provide a web UI to visualize the log. You just need to run:
+
+    .. code-block:: sh
+
+        streamlit run rdagent/log/ui/dsapp.py
+
+  - Then you can input the log path and visualize the R&D process.
+
+- **Additional Guidance**
+
+  - **Combine different LLM Models at R&D Stage**
+
+    - You can combine different LLM models at the R&D stage. 
+
+    - By default, when you set environment variable ``CHAT_MODEL``, it covers both R&D stages. When customizing the model for the development stage, you can set:
+    
+    .. code-block:: sh
+
+      # This example sets the model to "o3-mini". For some models, the reasoning effort shoule be set to "None".
+      dotenv set LITELLM_CHAT_MODEL_MAP '{"coding":{"model":"o3-mini","reasoning_effort":"high"},"running":{"model":"o3-mini","reasoning_effort":"high"}}'
+
+
+
+

docs/scens/kaggle_agent.rst

@@ -1,272 +0,0 @@
-.. _kaggle_agent:
-
-=======================
-Kaggle Agent
-=======================
-
-**🤖 Automated Feature Engineering & Model Tuning Evolution**
-------------------------------------------------------------------------------------------
-
-🎨 Design
-~~~~~~~~~~~
-
-.. image:: kaggle_design.png
-   :alt: Design of Kaggle Agent
-   :align: center
-
-📖 Background
-~~~~~~~~~~~~~~
-In the landscape of data science competitions, Kaggle serves as the ultimate arena where data enthusiasts harness the power of algorithms to tackle real-world challenges.
-The Kaggle Agent stands as a pivotal tool, empowering participants to seamlessly integrate cutting-edge models and datasets, transforming raw data into actionable insights.
-
-By utilizing the **Kaggle Agent**, data scientists can craft innovative solutions that not only uncover hidden patterns but also drive significant advancements in predictive accuracy and model robustness.
-
-
-🌟 Introduction
-~~~~~~~~~~~~~~~~
-
-In this scenario, our automated system proposes hypothesis, choose action, implements code, conducts validation, and utilizes feedback in a continuous, iterative process.
-
-The goal is to automatically optimize performance metrics within the validation set or Kaggle Leaderboard, ultimately discovering the most efficient features and models through autonomous research and development.
-
-Here's an enhanced outline of the steps:
-
-**Step 1 : Hypothesis Generation 🔍**
-
-- Generate and propose initial hypotheses based on previous experiment analysis and domain expertise, with thorough reasoning and financial justification.
-
-**Step 2 : Experiment Creation ✨**
-
-- Transform the hypothesis into a task.
-- Choose a specific action within feature engineering or model tuning.
-- Develop, define, and implement a new feature or model, including its name, description, and formulation.
-
-**Step 3 : Model/Feature Implementation 👨‍💻**
-
-- Implement the model code based on the detailed description.
-- Evolve the model iteratively as a developer would, ensuring accuracy and efficiency.
-
-**Step 4 : Validation on Test Set or Kaggle 📉**
-
-- Validate the newly developed model using the test set or Kaggle dataset.
-- Assess the model's effectiveness and performance based on the validation results.
-
-**Step 5: Feedback Analysis 🔍**
-
-- Analyze validation results to assess performance.
-- Use insights to refine hypotheses and enhance the model.
-
-**Step 6: Hypothesis Refinement ♻️**
-
-- Adjust hypotheses based on validation feedback.
-- Iterate the process to continuously improve the model.
-
-🧭 Example Guide
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-- 🔧 **Set up RD-Agent Environment**
-
-  - Before you start, please make sure you have installed RD-Agent and configured the environment for RD-Agent correctly. If you want to know how to install and configure the RD-Agent, please refer to the `documentation <../installation_and_configuration.html>`_.
-
-- 🔨 **Configuring the Kaggle API**
-  
-  - Register and login on the `Kaggle <https://www.kaggle.com/>`_ website.
-  - Click on the avatar (usually in the top right corner of the page) -> ``Settings`` -> ``Create New Token``, A file called ``kaggle.json`` will be downloaded.
-  - Move ``kaggle.json`` to ``~/.config/kaggle/``
-  - Modify the permissions of the ``kaggle.json`` file.
-
-    .. code-block:: sh
-
-      chmod 600 ~/.config/kaggle/kaggle.json
-
-  - For more information about Kaggle API Settings, refer to the `Kaggle API <https://github.com/Kaggle/kaggle-api>`_.
-
-- 🔩 **Setting the Environment variables at .env file**
-
-  - Determine the path where the data will be stored and add it to the ``.env`` file.
-
-  .. code-block:: sh
-
-    dotenv set KG_LOCAL_DATA_PATH <your local directory>/kaggle_data
-
-- 📥 **Download Competition Data**
-
-  - Kaggle competition data, contains two parts: competition description file (json file) and competition dataset (zip file).
-
-    - **How to get the competition description file**
-
-      - *Manual Download (General User Suggestions):*
-
-        - Download the competition description file prepared in advance, and extract it to the specified directory.
-
-          .. code-block:: sh
-
-            wget https://github.com/SunsetWolf/rdagent_resource/releases/download/kaggle_data/kaggle_data.zip
-            unzip kaggle_data.zip -d <your local directory>/kaggle_data
-
-      - *Automatic Download (Developer Suggestions):*
-      
-        - Alternatively, you can choose to download the competition description file automatically when you run the program, but it requires ``chromedriver`` to be installed, as follows:
-
-          .. code-block:: sh
-
-            # install chrome
-            wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb
-            sudo apt install ./google-chrome-stable_current_amd64.deb
-            google-chrome --version
-
-            # install chromedriver
-            wget "https://storage.googleapis.com/chrome-for-testing-public/$(google-chrome --version | grep -oP '\d+\.\d+\.\d+\.\d+')/linux64/chromedriver-linux64.zip"
-            unzip chromedriver-linux64.zip
-            cd chromedriver-linux64
-            sudo mv chromedriver /usr/local/bin
-            sudo chmod +x /usr/local/bin/chromedriver
-            chromedriver --version
-
-    - **How to get the competition dataset**
-
-      - The competition dataset is downloaded and extracted automatically when the program is run. If the zip file exists, the download will be skipped, if the unzip folder exists, the unzip will be skipped.
-
-    - **Correct directory structure (Here is an example of competition data with id sf-crime)**
-
-      .. code-block:: text
-
-        kaggle_data
-        └── zip_files
-        | └── sf-crime.zip
-        ├── sf-crime.json
-        └── sf-crime
-          └── ...
-        
-      - ``kaggle_data/zip_files/sf-crime.zip:`` Competition dataset zip files downloaded from the Kaggle website.
-
-      - ``kaggle_data/sf-crime.json:`` Competition description file.
-
-      - ``kaggle_data/sf-crime:`` The target folder for unzipping the competition dataset.
-
-- 🗳️ **Join the competition**
-
-  - If your Kaggle API account has not joined a competition, you will need to join the competition before running the program.
-    
-    - At the bottom of the competition details page, you can find the ``Join the competition`` button, click on it and select ``I Understand and Accept`` to join the competition.
-    
-    - In the **Competition List Available** below, you can jump to the competition details page.
-
-- 🚀 **Run the Application**
-
-  - You can directly run the application by using the following command:
-    
-    .. code-block:: sh
-
-        rdagent kaggle --competition <Competition ID>
-
-- 📤 **Submit the Result Automatically or Manually**
-
-  - If Auto: You need to set ``KG_AUTO_SUBMIT`` to ``true`` in the ``.env`` file.
-
-    .. code-block:: sh
-
-      dotenv set KG_AUTO_SUBMIT true
-  
-  - Else: You can download the prediction results from the UI interface and submit them manually. For more details, refer to the :doc:`UI guide <../ui>`.
-
-📋 Competition List Available
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-+-----------+-----------------------------------+------------------+-----------+---------------------------------------------------------------------------------------------------------+
-| **index** | **Competition Name**              | **Task**         | **Modal** | **ID**                                                                                                  |
-+===========+===================================+==================+===========+=========================================================================================================+
-| 01        | Media Campaign Cost Dataset       | Regression       | Tabular   | `playground-series-s3e11 <https://www.kaggle.com/competitions/playground-series-s3e11/data>`_           |
-+-----------+-----------------------------------+------------------+-----------+---------------------------------------------------------------------------------------------------------+
-| 02        | Wild Blueberry Yield Dataset      | Regression       | Tabular   | `playground-series-s3e14 <https://www.kaggle.com/competitions/playground-series-s3e14/data>`_           |
-+-----------+-----------------------------------+------------------+-----------+---------------------------------------------------------------------------------------------------------+
-| 03        | Crab Age Dataset                  | Regression       | Tabular   | `playground-series-s3e16 <https://www.kaggle.com/competitions/playground-series-s3e16/data>`_           |
-+-----------+-----------------------------------+------------------+-----------+---------------------------------------------------------------------------------------------------------+
-| 04        | Flood Prediction Dataset          | Regression       | Tabular   | `playground-series-s4e5 <https://www.kaggle.com/competitions/playground-series-s4e5/data>`_             |
-+-----------+-----------------------------------+------------------+-----------+---------------------------------------------------------------------------------------------------------+
-| 05        | Used Car Prices Dataset           | Regression       | Tabular   | `playground-series-s4e9 <https://www.kaggle.com/competitions/playground-series-s4e9/data>`_             |
-+-----------+-----------------------------------+------------------+-----------+---------------------------------------------------------------------------------------------------------+
-| 06        | Cirrhosis Outcomes Dataset        | Multi-Class      | Tabular   | `playground-series-s3e26 <https://www.kaggle.com/competitions/playground-series-s3e26/data>`_           |
-+-----------+-----------------------------------+------------------+-----------+---------------------------------------------------------------------------------------------------------+
-| 07        | San Francisco Crime Classification| Multi-Class      | Tabular   | `sf-crime <https://www.kaggle.com/competitions/sf-crime/data>`_                                         |
-+-----------+-----------------------------------+------------------+-----------+---------------------------------------------------------------------------------------------------------+
-| 08        | Poisonous Mushrooms Dataset       | Classification   | Tabular   | `playground-series-s4e8 <https://www.kaggle.com/competitions/playground-series-s4e8/data>`_             |
-+-----------+-----------------------------------+------------------+-----------+---------------------------------------------------------------------------------------------------------+
-| 09        | Spaceship Titanic                 | Classification   | Tabular   | `spaceship-titanic <https://www.kaggle.com/competitions/spaceship-titanic/data>`_                       |
-+-----------+-----------------------------------+------------------+-----------+---------------------------------------------------------------------------------------------------------+
-| 10        | Forest Cover Type Prediction      | Classification   | Tabular   | `forest-cover-type-prediction <https://www.kaggle.com/competitions/forest-cover-type-prediction/data>`_ |
-+-----------+-----------------------------------+------------------+-----------+---------------------------------------------------------------------------------------------------------+
-| 11        | Digit Recognizer                  | Classification   | Image     | `digit-recognizer <https://www.kaggle.com/competitions/digit-recognizer>`_                              |
-+-----------+-----------------------------------+------------------+-----------+---------------------------------------------------------------------------------------------------------+
-| To be continued ...                                                                                                                                                                    |
-+-----------+-----------------------------------+------------------+-----------+---------------------------------------------------------------------------------------------------------+
-
-
-
-🎨 Customize one template for a new competition
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-In order to facilitate RD-Agent to generate competition codes, we have specified a competition code structure:
-
-.. image:: kaggle_template.png
-   :alt: Design of Kaggle Code Template
-   :align: center
-
-- **feature directory** contains the feature engineering code. Generally no modification is required.
-- **model directory** contains the model codes.
-  select_xx.py is used to select different features according to different models.
-  model_xx.py is the basic code of different models. Generally, only some initial parameters need to be adjusted.
-- **fea_share_preprocess.py** is some basic preprocessing code shared by different models. The degree of customization here is high, but the preprocess_script() function needs to be retained, which will be called by train.py
-- **train.py** is the main code, which connects all the codes and is also the code called during the final execution.
-
-**We will soon provide a tool for automatic/semi-automatic template generation.**
-If you want to try a different competition now, you can refer to our current template structure and content to write a new template.
-
-
-🎯 Roadmap
-~~~~~~~~~~~
-
-**Completed:**
-
-- **Kaggle Project Schema Design** ✅
-
-- **RD-Agent Integration with kaggle schema** ✅
-
-**Ongoing:**
-
-- **Template auto generation**
-
-- **Bench Optimization**
-
-  - **Online Bench**
-
-    - **RealMLBench**
-
-      - Ongoing integration
-
-      - Auto online submission
-
-      - Batch Evaluation
-
-  - **Offline Bench**
-  
-    - MLE-Bench
-
-
-🛠️ Usage of modules
-~~~~~~~~~~~~~~~~~~~~~
-
-.. _Env Config: 
-
-- **Env Config**
-
-The following environment variables can be set in the `.env` file to customize the application's behavior:
-
-.. autopydantic_settings:: rdagent.app.kaggle.conf.KaggleBasePropSetting
-    :settings-show-field-summary: False
-    :exclude-members: Config
-
-.. autopydantic_settings:: rdagent.components.coder.factor_coder.config.FactorCoSTEERSettings
-    :settings-show-field-summary: False
-    :members: coder_use_cache, file_based_execution_timeout, select_method, max_loop
-    :exclude-members: Config, fail_task_trial_limit, v1_query_former_trace_limit, v1_query_similar_success_limit, v2_query_component_limit, v2_query_error_limit, v2_query_former_trace_limit, v2_error_summary, v2_knowledge_sampler, v2_add_fail_attempt_to_latest_successful_execution, new_knowledge_base_path, knowledge_base_path, data_folder, data_folder_debug
-    :no-index:

docs/scens/model_agent_med.rst

@@ -1,128 +0,0 @@
-.. _model_agent_med:
-
-=======================
-Medical Model Agent
-=======================
-
-**🤖 Automated Medical Predtion Model Evolution**
-------------------------------------------------------------------------------------------
-
-📖 Background
-~~~~~~~~~~~~~~
-In this scenario, we consider the problem of risk prediction from patients' ICU monitoring data. We use the a public EHR dataset - MIMIC-III and extract a binary classification task for evaluating the framework.
-In this task, we aim at predicting the whether the patients will suffer from Acute Respiratory Failure (ARF) based their first 12 hours ICU monitoring data. 
-
-🎥 `Demo <https://rdagent.azurewebsites.net/dmm>`_
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. raw:: html
-
-    <div style="display: flex; justify-content: center; align-items: center;">
-      <video width="600" controls>
-        <source src="https://rdagent.azurewebsites.net/media/1653542fc1b9fa14a306c35c1b1fc48288f980793f38abe82b023af9.mp4" type="video/mp4">
-        Your browser does not support the video tag.
-      </video>
-    </div>
-
-
-🌟 Introduction
-~~~~~~~~~~~~~~~~
-
-In this scenario, our automated system proposes hypothesis, constructs model, implements code, receives back-testing, and uses feedbacks. 
-Hypothesis is iterated in this continuous process. 
-The system aims to automatically optimise performance metrics of medical prediction thereby finding the optimised code through autonomous research and development.
-
-Here's an enhanced outline of the steps:
-
-**Step 1 : Hypothesis Generation 🔍**
-
-- Generate and propose initial hypotheses based on previous experiment analysis and domain expertise, with thorough reasoning and justification.
-
-**Step 2 : Model Creation ✨**
-
-- Transform the hypothesis into a model.
-- Develop, define, and implement a machine learning model, including its name, description, and formulation.
-
-**Step 3 : Model Implementation 👨‍💻**
-
-- Implement the model code based on the detailed description.
-- Evolve the model iteratively as a developer would, ensuring accuracy and efficiency.
-
-**Step 4 : Backtesting with MIMIC-III 📉**
-
-- Conduct backtesting using the newly developed model on the extracted task from MIMIC-III.
-- Evaluate the model's effectiveness and performance in terms of AUROC score.
-
-**Step 5 : Feedback Analysis 🔍**
-
-- Analyze backtest results to assess performance.
-- Incorporate feedback to refine hypotheses and improve the model.
-
-**Step 6 :Hypothesis Refinement ♻️**
-
-- Refine hypotheses based on feedback from backtesting.
-- Repeat the process to continuously improve the model.
-
-⚡ Quick Start
-~~~~~~~~~~~~~~~~~
-
-Please refer to the installation part in :doc:`../installation_and_configuration` to prepare your system dependency.
-
-You can try our demo by running the following command:
-
-- 🐍 Create a Conda Environment
-  
-  - Create a new conda environment with Python (3.10 and 3.11 are well tested in our CI):
-
-    .. code-block:: sh
-    
-        conda create -n rdagent python=3.10
-
-  - Activate the environment:
-
-    .. code-block:: sh
-
-        conda activate rdagent
-
-- 📦 Install the RDAgent
-    
-  - You can install the RDAgent package from PyPI:
-
-    .. code-block:: sh
-
-        pip install rdagent
-
-- 📦 Request PhysioNet Account
-    
-  - Apply for an account at `PhysioNet <https://physionet.org/>`_.
-  - Request access to FIDDLE preprocessed data: `FIDDLE Dataset <https://physionet.org/content/mimic-eicu-fiddle-feature/1.0.0/>`_.
-  - Place your username and password in `.env`.
-
-    .. code-block:: bash
-
-        cat << EOF  >> .env
-        DM_USERNAME=<your_username>
-        DM_PASSWORD=<your_password>
-        EOF
-
-
-- 🚀 Run the Application
-    
-  - You can directly run the application by using the following command:
-    
-    .. code-block:: sh
-
-        rdagent med_model
-
-🛠️ Usage of modules
-~~~~~~~~~~~~~~~~~~~~~
-
-.. _Env Config: 
-
-- **Env Config**
-
-The following environment variables can be set in the `.env` file to customize the application's behavior:
-
-.. autopydantic_settings:: rdagent.app.data_mining.conf.MedBasePropSetting
-    :settings-show-field-summary: False
-    :exclude-members: Config

docs/scens/quant_agent_fin.rst

@@ -0,0 +1,113 @@
+.. _quant_agent_fin:
+
+=====================
+Finance Quant Agent
+=====================
+
+
+**🥇The First Data-Centric Quant Multi-Agent Framework RD-Agent(Q)**
+---------------------------------------------------------------------
+
+R&D-Agent for Quantitative Finance, in short **RD-Agent(Q)**, is the first data-centric, multi-agent framework designed to automate the full-stack research and development of quantitative strategies via coordinated factor-model co-optimization.
+
+You can learn more details about **RD-Agent(Q)** through the `paper <https://arxiv.org/abs/2505.15155>`_.
+
+⚡ Quick Start
+~~~~~~~~~~~~~~~~~
+
+Before you start, please make sure you have installed RD-Agent and configured the environment for RD-Agent correctly. If you want to know how to install and configure the RD-Agent, please refer to the `documentation <../installation_and_configuration.html>`_.
+
+Then, you can run the framework by running the following command:
+
+- 🐍 Create a Conda Environment
+
+  - Create a new conda environment with Python (3.10 and 3.11 are well tested in our CI):
+
+    .. code-block:: sh
+
+          conda create -n rdagent python=3.10
+
+  - Activate the environment:
+
+    .. code-block:: sh
+
+        conda activate rdagent
+
+- 📦 Install the RDAgent
+  
+  - You can install the RDAgent package from PyPI:
+
+    .. code-block:: sh
+
+        pip install rdagent
+
+- 🚀 Run the Application
+    
+  - You can directly run the application by using the following command:
+    
+    .. code-block:: sh
+
+        rdagent fin_quant
+
+
+🛠️ Usage of modules
+~~~~~~~~~~~~~~~~~~~~~
+
+.. _Env Config: 
+
+- **Env Config**
+
+The following environment variables can be set in the `.env` file to customize the application's behavior:
+
+.. autopydantic_settings:: rdagent.app.qlib_rd_loop.conf.QuantBasePropSetting
+    :settings-show-field-summary: False
+    :exclude-members: Config
+
+.. autopydantic_settings:: rdagent.components.coder.factor_coder.config.FactorCoSTEERSettings
+    :settings-show-field-summary: False
+    :members: coder_use_cache, data_folder, data_folder_debug, file_based_execution_timeout, select_method, max_loop, knowledge_base_path, new_knowledge_base_path
+    :exclude-members: Config, fail_task_trial_limit, v1_query_former_trace_limit, v1_query_similar_success_limit, v2_query_component_limit, v2_query_error_limit, v2_query_former_trace_limit, v2_error_summary, v2_knowledge_sampler
+    :no-index:
+
+- **Qlib Configuration**
+    - The `.yaml` files in both the `model_template` and `factor_template` directories contain some configurations for running the corresponding models or factors within the Qlib framework. Below is an overview of their contents and roles:
+        - **General Settings**:
+            - **provider_uri**: Specifies the local Qlib data path, set to `~/.qlib/qlib_data/cn_data`.
+            - **market**: Configured to `csi300`, representing the CSI 300 index constituents.
+            - **benchmark**: Set to `SH000300`, used for backtesting evaluation.
+        
+        - **Data Handling**:
+            - **start_time** and **end_time**: Define the full data range, from `2008-01-01` to `2022-08-01`.
+            - **fit_start_time**: The start date for fitting the model, set to `2008-01-01`.
+            - **fit_end_time**: The end date for fitting the model, set to `2014-12-31`.
+            - **features and labels**: Generated via a nested data loader combining `Alpha158DL` (for engineered features such as `RESI5`, `WVMA5`, `RSQR5`, `KLEN`, etc.) and a `StaticDataLoader` that loads precomputed factor files (`combined_factors_df.parquet`).
+            -  **normalization**: The pipeline includes `RobustZScoreNorm` (with clipping) and `Fillna` for inference, and `DropnaLabel` with `CSZScoreNorm` for training.
+        
+        - **Training Configuration**:
+            - **Model**: Uses `GeneralPTNN`, a PyTorch-based neural network model.
+            - **Dataset Splits**:
+                - **train**: `2008-01-01` to `2014-12-31`
+                - **valid**: `2015-01-01` to `2016-12-31`
+                - **test**: `2017-01-01` to `2020-08-01`
+
+        - **Default Hyperparameters** (can be overridden by command-line arguments):
+            - **n_epochs**: `100`
+            - **lr**: `2e-4`
+            - **early_stop**: `10`
+            - **batch_size**: `256`
+            - **weight_decay**: `0.0`
+            - **metric**: `loss`
+            - **loss**: `mse`
+            - **n_jobs**: `20`
+            - **GPU**: `0` (uses GPU 0 if available)
+            
+        - **Backtesting and Evaluation**:
+            - **strategy**: `TopkDropoutStrategy`, which selects the top 50 stocks and randomly drops 5 to introduce exploration.
+            - **backtest period**: `2017-01-01` to `2020-08-01`
+            - **initial capital**: `100,000,000`
+            - **cost configuration**: Includes open/close costs, minimum transaction costs, and slippage control.
+            
+        - **Recording and Analysis**:
+            - **SignalRecord**: Logs predicted signals.
+            - **SigAnaRecord**: Performs signal analysis without long-short separation.
+            - **PortAnaRecord**: Conducts portfolio analysis using the configured strategy and backtest settings.

rdagent/app/cli.py

@@ -17,14 +17,14 @@ from importlib.resources import path as rpath
 
 import fire
 
-from rdagent.app.data_mining.model import main as med_model
+from rdagent.app.data_science.loop import main as data_science
 from rdagent.app.general_model.general_model import (
     extract_models_and_implement as general_model,
 )
-from rdagent.app.kaggle.loop import main as kaggle_main
 from rdagent.app.qlib_rd_loop.factor import main as fin_factor
 from rdagent.app.qlib_rd_loop.factor_from_report import main as fin_factor_report
 from rdagent.app.qlib_rd_loop.model import main as fin_model
+from rdagent.app.qlib_rd_loop.quant import main as fin_quant
 from rdagent.app.utils.health_check import health_check
 from rdagent.app.utils.info import collect_info
 
@@ -44,17 +44,25 @@ def ui(port=19899, log_dir="", debug=False):
         subprocess.run(cmds)
 
 
+def server_ui(port=19899):
+    """
+    start web app to show the log traces in real time
+    """
+    subprocess.run(["python", "rdagent/log/server/app.py", f"--port={port}"])
+
+
 def app():
     fire.Fire(
         {
             "fin_factor": fin_factor,
             "fin_factor_report": fin_factor_report,
             "fin_model": fin_model,
-            "med_model": med_model,
+            "fin_quant": fin_quant,
             "general_model": general_model,
             "ui": ui,
             "health_check": health_check,
             "collect_info": collect_info,
-            "kaggle": kaggle_main,
+            "data_science": data_science,
+            "server_ui": server_ui,
         }
     )

rdagent/app/data_mining/conf.py

@@ -1,45 +0,0 @@
-from pathlib import Path
-
-from pydantic_settings import SettingsConfigDict
-
-from rdagent.components.workflow.conf import BasePropSetting
-
-
-class MedBasePropSetting(BasePropSetting):
-    model_config = SettingsConfigDict(env_prefix="DM_", protected_namespaces=())
-
-    # 1) overriding the default
-    scen: str = "rdagent.scenarios.data_mining.experiment.model_experiment.DMModelScenario"
-    """Scenario class for data mining model"""
-
-    hypothesis_gen: str = "rdagent.scenarios.data_mining.proposal.model_proposal.DMModelHypothesisGen"
-    """Hypothesis generation class"""
-
-    hypothesis2experiment: str = "rdagent.scenarios.data_mining.proposal.model_proposal.DMModelHypothesis2Experiment"
-    """Hypothesis to experiment class"""
-
-    coder: str = "rdagent.scenarios.data_mining.developer.model_coder.DMModelCoSTEER"
-    """Coder class"""
-
-    runner: str = "rdagent.scenarios.data_mining.developer.model_runner.DMModelRunner"
-    """Runner class"""
-
-    summarizer: str = "rdagent.scenarios.data_mining.developer.feedback.DMModelExperiment2Feedback"
-    """Summarizer class"""
-
-    evolving_n: int = 10
-    """Number of evolutions"""
-
-    evolving_n: int = 10
-
-    # 2) Extra config for the scenario
-    # physionet account
-    # NOTE: You should apply the account in https://physionet.org/
-    username: str = ""
-    """Physionet account username"""
-
-    password: str = ""
-    """Physionet account password"""
-
-
-MED_PROP_SETTING = MedBasePropSetting()

rdagent/app/data_mining/model.py

@@ -1,31 +0,0 @@
-import fire
-
-from rdagent.app.data_mining.conf import MED_PROP_SETTING
-from rdagent.components.workflow.rd_loop import RDLoop
-from rdagent.core.exception import ModelEmptyError
-
-
-class ModelRDLoop(RDLoop):
-    skip_loop_error = (ModelEmptyError,)
-
-
-def main(path=None, step_n=None):
-    """
-    Auto R&D Evolving loop for models in a medical scenario.
-
-    You can continue running session by
-
-    .. code-block:: python
-
-        dotenv run -- python rdagent/app/data_mining/model.py $LOG_PATH/__session__/1/0_propose  --step_n 1   # `step_n` is a optional paramter
-
-    """
-    if path is None:
-        model_loop = ModelRDLoop(MED_PROP_SETTING)
-    else:
-        model_loop = ModelRDLoop.load(path)
-    model_loop.run(step_n=step_n)
-
-
-if __name__ == "__main__":
-    fire.Fire(main)

rdagent/app/data_science/conf.py

@@ -1,19 +1,32 @@
+from typing import Literal
+
 from pydantic_settings import SettingsConfigDict
 
 from rdagent.app.kaggle.conf import KaggleBasePropSetting
 
 
 class DataScienceBasePropSetting(KaggleBasePropSetting):
+    # TODO: Kaggle Setting should be the subclass of DataScience
     model_config = SettingsConfigDict(env_prefix="DS_", protected_namespaces=())
 
     # Main components
     ## Scen
     scen: str = "rdagent.scenarios.data_science.scen.KaggleScen"
-    """Scenario class for data mining model"""
+    """
+    Scenario class for data science tasks.
+    - For Kaggle competitions, use: "rdagent.scenarios.data_science.scen.KaggleScen"
+    - For custom data science scenarios, use: "rdagent.scenarios.data_science.scen.DataScienceScen"
+    """
+
+    hypothesis_gen: str = "rdagent.scenarios.data_science.proposal.exp_gen.proposal.DSProposalV2ExpGen"
+    """Hypothesis generation class"""
 
     ## Workflow Related
     consecutive_errors: int = 5
 
+    ## Coding Related
+    coding_fail_reanalyze_threshold: int = 3
+
     debug_timeout: int = 600
     """The timeout limit for running on debugging data"""
     full_timeout: int = 3600
@@ -24,11 +37,79 @@ class DataScienceBasePropSetting(KaggleBasePropSetting):
     #### enable specification
     spec_enabled: bool = True
 
+    #### proposal related
     proposal_version: str = "v1"
     coder_on_whole_pipeline: bool = False
+    max_trace_hist: int = 3
 
     coder_max_loop: int = 10
-    runner_max_loop: int = 3
+    runner_max_loop: int = 1
+
+    rule_base_eval: bool = False
+    sample_data_by_LLM: bool = False
+    use_raw_description: bool = False
+    show_nan_columns: bool = False
+
+    #### model dump
+    enable_model_dump: bool = False
+    enable_doc_dev: bool = False
+    model_dump_check_level: Literal["medium", "high"] = "medium"
+
+    ### knowledge base
+    enable_knowledge_base: bool = False
+    knowledge_base_version: str = "v1"
+    knowledge_base_path: str | None = None
+    idea_pool_json_path: str | None = None
+
+    ### archive log folder after each loop
+    enable_log_archive: bool = True
+    log_archive_path: str | None = None
+    log_archive_temp_path: str | None = (
+        None  # This is to store the mid tar file since writing the tar file is preferred in local storage then copy to target storage
+    )
+
+    #### Evaluation on Test related
+    eval_sub_dir: str = "eval"  # TODO: fixme, this is not a good name
+    """We'll use f"{DS_RD_SETTING.local_data_path}/{DS_RD_SETTING.eval_sub_dir}/{competition}"
+    to find the scriipt to evaluate the submission on test"""
+
+    """---below are the settings for multi-trace---"""
+
+    ### multi-trace related
+    max_trace_num: int = 3
+    """The maximum number of traces to grow before merging"""
+
+    #### multi-trace:checkpoint selector
+    selector_name: str = "rdagent.scenarios.data_science.proposal.exp_gen.ckp_select.LatestCKPSelector"
+    """The name of the selector to use"""
+    sota_count_window: int = 5
+    """The number of trials to consider for SOTA count"""
+    sota_count_threshold: int = 1
+    """The threshold for SOTA count"""
+
+    #### multi-trace: SOTA experiment selector
+    sota_exp_selector_name: str = "rdagent.scenarios.data_science.proposal.exp_gen.sota_exp_select.GlobalSOTASelector"
+    """The name of the SOTA experiment selector to use"""
+
+    ### multi-trace:inject optimals for multi-trace
+    # inject diverse when start a new sub-trace
+    enable_inject_diverse: bool = False
+
+    # inject knowledge at the root of the trace
+    enable_inject_knowledge_at_root: bool = False
+
+    # enable different version of DSExpGen for multi-trace
+    enable_multi_version_exp_gen: bool = False
+    exp_gen_version_list: str = "v3,v2"
+
+    #### multi-trace: time for final multi-trace merge
+    merge_hours: int = 2
+    """The time for merge"""
+
+    #### multi-trace: max SOTA-retrieved number, used in AutoSOTAexpSelector
+    # constrains the number of SOTA experiments to retrieve, otherwise too many SOTA experiments to retrieve will cause the exceed of the context window of LLM
+    max_sota_retrieved_num: int = 10
+    """The maximum number of SOTA experiments to retrieve in a LLM call"""
 
 
 DS_RD_SETTING = DataScienceBasePropSetting()

rdagent/app/data_science/loop.py

@@ -1,203 +1,73 @@
+import asyncio
 from pathlib import Path
-from typing import Any
 
 import fire
 
 from rdagent.app.data_science.conf import DS_RD_SETTING
-from rdagent.components.coder.data_science.ensemble import EnsembleCoSTEER
-from rdagent.components.coder.data_science.ensemble.exp import EnsembleTask
-from rdagent.components.coder.data_science.feature import FeatureCoSTEER
-from rdagent.components.coder.data_science.feature.exp import FeatureTask
-from rdagent.components.coder.data_science.model import ModelCoSTEER
-from rdagent.components.coder.data_science.model.exp import ModelTask
-from rdagent.components.coder.data_science.pipeline import PipelineCoSTEER
-from rdagent.components.coder.data_science.pipeline.exp import PipelineTask
-from rdagent.components.coder.data_science.raw_data_loader import DataLoaderCoSTEER
-from rdagent.components.coder.data_science.raw_data_loader.exp import DataLoaderTask
-from rdagent.components.coder.data_science.workflow import WorkflowCoSTEER
-from rdagent.components.coder.data_science.workflow.exp import WorkflowTask
-from rdagent.components.workflow.conf import BasePropSetting
-from rdagent.components.workflow.rd_loop import RDLoop
-from rdagent.core.exception import CoderError, RunnerError
-from rdagent.core.proposal import ExperimentFeedback
-from rdagent.core.scenario import Scenario
 from rdagent.core.utils import import_class
 from rdagent.log import rdagent_logger as logger
-from rdagent.scenarios.data_science.dev.feedback import DSExperiment2Feedback
-from rdagent.scenarios.data_science.dev.runner import DSCoSTEERRunner
-from rdagent.scenarios.data_science.experiment.experiment import DSExperiment
-from rdagent.scenarios.data_science.proposal.exp_gen import DSExpGen, DSTrace
-from rdagent.scenarios.kaggle.kaggle_crawler import download_data
-
-
-class DataScienceRDLoop(RDLoop):
-    skip_loop_error = (CoderError, RunnerError)
-
-    def __init__(self, PROP_SETTING: BasePropSetting):
-        logger.log_object(PROP_SETTING.competition, tag="competition")
-        scen: Scenario = import_class(PROP_SETTING.scen)(PROP_SETTING.competition)
-
-        ### shared components in the workflow  # TODO: check if
-        knowledge_base = (
-            import_class(PROP_SETTING.knowledge_base)(PROP_SETTING.knowledge_base_path, scen)
-            if PROP_SETTING.knowledge_base != ""
-            else None
-        )
-
-        # 1) task generation from scratch
-        # self.scratch_gen: tuple[HypothesisGen, Hypothesis2Experiment] = DummyHypothesisGen(scen),
-
-        # 2) task generation from a complete solution
-        # self.exp_gen: ExpGen = import_class(PROP_SETTING.exp_gen)(scen)
-        self.exp_gen = DSExpGen(scen)
-        self.data_loader_coder = DataLoaderCoSTEER(scen)
-        self.feature_coder = FeatureCoSTEER(scen)
-        self.model_coder = ModelCoSTEER(scen)
-        self.ensemble_coder = EnsembleCoSTEER(scen)
-        self.workflow_coder = WorkflowCoSTEER(scen)
-
-        self.pipeline_coder = PipelineCoSTEER(scen)
-
-        self.runner = DSCoSTEERRunner(scen)
-        # self.summarizer: Experiment2Feedback = import_class(PROP_SETTING.summarizer)(scen)
-        # logger.log_object(self.summarizer, tag="summarizer")
-
-        # self.trace = KGTrace(scen=scen, knowledge_base=knowledge_base)
-        self.trace = DSTrace(scen=scen)
-        self.summarizer = DSExperiment2Feedback(scen)
-        super(RDLoop, self).__init__()
-
-    def direct_exp_gen(self, prev_out: dict[str, Any]):
-        exp = self.exp_gen.gen(self.trace)
-        logger.log_object(exp)
-
-        # FIXME: this is for LLM debug webapp, remove this when the debugging is done.
-        logger.log_object(exp, tag="debug_exp_gen")
-        return exp
-
-    def coding(self, prev_out: dict[str, Any]):
-        exp = prev_out["direct_exp_gen"]
-        for tasks in exp.pending_tasks_list:
-            exp.sub_tasks = tasks
-            with logger.tag(f"{exp.sub_tasks[0].__class__.__name__}"):
-                if isinstance(exp.sub_tasks[0], DataLoaderTask):
-                    exp = self.data_loader_coder.develop(exp)
-                elif isinstance(exp.sub_tasks[0], FeatureTask):
-                    exp = self.feature_coder.develop(exp)
-                elif isinstance(exp.sub_tasks[0], ModelTask):
-                    exp = self.model_coder.develop(exp)
-                elif isinstance(exp.sub_tasks[0], EnsembleTask):
-                    exp = self.ensemble_coder.develop(exp)
-                elif isinstance(exp.sub_tasks[0], WorkflowTask):
-                    exp = self.workflow_coder.develop(exp)
-                elif isinstance(exp.sub_tasks[0], PipelineTask):
-                    exp = self.pipeline_coder.develop(exp)
-                else:
-                    raise NotImplementedError(f"Unsupported component in DataScienceRDLoop: {exp.hypothesis.component}")
-            exp.sub_tasks = []
-        logger.log_object(exp)
-        return exp
-
-    def running(self, prev_out: dict[str, Any]):
-        exp: DSExperiment = prev_out["coding"]
-        if exp.is_ready_to_run():
-            new_exp = self.runner.develop(exp)
-            logger.log_object(new_exp)
-            return new_exp
-        return exp
-
-    def feedback(self, prev_out: dict[str, Any]) -> ExperimentFeedback:
-        """
-        Assumption:
-        - If we come to feedback phase, the previous development steps are successful.
-        """
-        exp: DSExperiment = prev_out["running"]
-        if self.trace.next_incomplete_component() is None or DS_RD_SETTING.coder_on_whole_pipeline:
-            # we have alreadly completed components in previous trace. So current loop is focusing on a new proposed idea.
-            # So we need feedback for the proposal.
-            feedback = self.summarizer.generate_feedback(exp, self.trace)
-        else:
-            # Otherwise, it is on drafting stage, don't need complicated feedbacks.
-            feedback = ExperimentFeedback(
-                reason=f"{exp.hypothesis.component} is completed.",
-                decision=True,
-            )
-        logger.log_object(feedback)
-        return feedback
-
-    def record(self, prev_out: dict[str, Any]):
-        e = prev_out.get(self.EXCEPTION_KEY, None)
-        if e is None:
-            self.trace.hist.append((prev_out["running"], prev_out["feedback"]))
-        else:
-            self.trace.hist.append(
-                (
-                    prev_out["direct_exp_gen"] if isinstance(e, CoderError) else prev_out["coding"],
-                    ExperimentFeedback.from_exception(e),
-                )
-            )
-            if (
-                self.trace.sota_experiment() is None
-                and len(self.trace.hist) >= DS_RD_SETTING.consecutive_errors
-                and not DS_RD_SETTING.coder_on_whole_pipeline
-            ):
-                # if {in inital/drafting stage} and {tried enough times}
-                for _, fb in self.trace.hist[-DS_RD_SETTING.consecutive_errors :]:
-                    if fb:
-                        break  # any success will stop restarting.
-                else:  # otherwise restart it
-                    logger.error("Consecutive errors reached the limit. Dumping trace.")
-                    logger.log_object(self.trace, tag="trace before restart")
-                    self.trace = DSTrace(scen=self.trace.scen, knowledge_base=self.trace.knowledge_base)
-        logger.log_object(self.trace, tag="trace")
-        logger.log_object(self.trace.sota_experiment(), tag="SOTA experiment")
+from rdagent.scenarios.data_science.loop import DataScienceRDLoop
 
 
 def main(
-    path=None, output_path=None, step_n=None, loop_n=None, competition="bms-molecular-translation", do_truncate=True
+    path: str | None = None,
+    checkout: bool | str | Path = True,
+    step_n: int | None = None,
+    loop_n: int | None = None,
+    timeout: str | None = None,
+    competition="bms-molecular-translation",
+    replace_timer=True,
+    exp_gen_cls: str | None = None,
 ):
     """
 
     Parameters
     ----------
     path :
-        path like `$LOG_PATH/__session__/1/0_propose`. It indicates that we restore the state that after finish the step 0 in loop 1
-    output_path :
-        path like `$LOG_PATH`. It indicates that where we want to save our session and log information.
+        A path like `$LOG_PATH/__session__/1/0_propose`. This indicates that we restore the state after finishing step 0 in loop 1.
+    checkout :
+        Used only when a path is provided.
+        Can be True, False, or a path.
+        Default is True.
+        - If True, the new loop will use the existing folder and clear logs for sessions after the one corresponding to the given path.
+        - If False, the new loop will use the existing folder but keep the logs for sessions after the one corresponding to the given path.
+        - If a path (or a str like Path) is provided, the new loop will be saved to that path, leaving the original path unchanged.
     step_n :
-        How many steps to run; if None, it will run forever until error or KeyboardInterrupt
+        Number of steps to run; if None, the process will run indefinitely until an error or KeyboardInterrupt occurs.
     loop_n :
-        How many loops to run; if None, it will run forever until error or KeyboardInterrupt
-        - if current loop is incomplete, it will be counted as the first loop for completion.
-        - if both step_n and loop_n are provided, the process will stop as soon as either condition is met.
+        Number of loops to run; if None, the process will run indefinitely until an error or KeyboardInterrupt occurs.
+        - If the current loop is incomplete, it will be counted as the first loop for completion.
+        - If both step_n and loop_n are provided, the process will stop as soon as either condition is met.
     competition :
-    do_truncate :
-        If set to True, the logger will truncate the future log messages by calling `logger.storage.truncate`.
+        Competition name.
+    replace_timer :
+        If a session is loaded, determines whether to replace the timer with session.timer.
+    exp_gen_cls :
+        When there are different stages, the exp_gen can be replaced with the new proposal.
 
 
     Auto R&D Evolving loop for models in a Kaggle scenario.
-    You can continue running session by
+    You can continue running a session by using the command:
     .. code-block:: bash
-        dotenv run -- python rdagent/app/data_science/loop.py [--competition titanic] $LOG_PATH/__session__/1/0_propose  --step_n 1   # `step_n` is a optional parameter
-        rdagent kaggle --competition playground-series-s4e8  # You are encouraged to use this one.
+        dotenv run -- python rdagent/app/data_science/loop.py [--competition titanic] $LOG_PATH/__session__/1/0_propose  --step_n 1   # `step_n` is an optional parameter
+        rdagent kaggle --competition playground-series-s4e8  # This command is recommended.
     """
     if competition is not None:
         DS_RD_SETTING.competition = competition
 
-    if DS_RD_SETTING.competition:
-        if DS_RD_SETTING.scen.endswith("KaggleScen"):
-            download_data(competition=DS_RD_SETTING.competition, settings=DS_RD_SETTING)
-        else:
-            if not Path(f"{DS_RD_SETTING.local_data_path}/{competition}").exists():
-                logger.error(f"Please prepare data for competition {competition} first.")
-                return
-    else:
+    if not DS_RD_SETTING.competition:
         logger.error("Please specify competition name.")
+
     if path is None:
         kaggle_loop = DataScienceRDLoop(DS_RD_SETTING)
     else:
-        kaggle_loop = DataScienceRDLoop.load(path, output_path, do_truncate)
-    kaggle_loop.run(step_n=step_n, loop_n=loop_n)
+        kaggle_loop: DataScienceRDLoop = DataScienceRDLoop.load(path, checkout=checkout, replace_timer=replace_timer)
+
+    # replace exp_gen if we have new class
+    if exp_gen_cls is not None:
+        kaggle_loop.exp_gen = import_class(exp_gen_cls)(kaggle_loop.exp_gen.scen)
+
+    asyncio.run(kaggle_loop.run(step_n=step_n, loop_n=loop_n, all_duration=timeout))
 
 
 if __name__ == "__main__":

rdagent/app/general_model/general_model.py

@@ -30,18 +30,15 @@ def extract_models_and_implement(report_file_path: str) -> None:
     Returns:
     None
     """
-    with logger.tag("init"):
-        scenario = GeneralModelScenario()
-        logger.log_object(scenario, tag="scenario")
-    with logger.tag("r"):
-        # Save Relevant Images
-        img = extract_first_page_screenshot_from_pdf(report_file_path)
-        logger.log_object(img, tag="pdf_image")
-        exp = ModelExperimentLoaderFromPDFfiles().load(report_file_path)
-        logger.log_object(exp, tag="load_experiment")
-    with logger.tag("d"):
-        exp = QlibModelCoSTEER(scenario).develop(exp)
-        logger.log_object(exp, tag="developed_experiment")
+    scenario = GeneralModelScenario()
+    logger.log_object(scenario, tag="scenario")
+    # Save Relevant Images
+    img = extract_first_page_screenshot_from_pdf(report_file_path)
+    logger.log_object(img, tag="pdf_image")
+    exp = ModelExperimentLoaderFromPDFfiles().load(report_file_path)
+    logger.log_object(exp, tag="load_experiment")
+    exp = QlibModelCoSTEER(scenario).develop(exp)
+    logger.log_object(exp, tag="developed_experiment")
 
 
 if __name__ == "__main__":

rdagent/app/kaggle/conf.py

@@ -46,9 +46,11 @@ class KaggleBasePropSetting(ExtendedBaseSettings):
     local_data_path: str = ""
     """Folder storing Kaggle competition data"""
 
+    # Evaluation on Test related
     if_using_mle_data: bool = False
     auto_submit: bool = False
     """Automatically upload and submit each experiment result to Kaggle platform"""
+
     # Conditionally set the knowledge_base based on the use of graph RAG
     knowledge_base: str = ""
     """Knowledge base class, uses 'KGKnowledgeGraph' when advanced graph-based RAG is enabled, otherwise empty."""

rdagent/app/kaggle/loop.py

@@ -28,90 +28,83 @@ from rdagent.scenarios.kaggle.proposal.proposal import KGTrace
 
 class KaggleRDLoop(RDLoop):
     def __init__(self, PROP_SETTING: BasePropSetting):
-        with logger.tag("init"):
-            scen: Scenario = import_class(PROP_SETTING.scen)(PROP_SETTING.competition)
-            logger.log_object(scen, tag="scenario")
-            knowledge_base = (
-                import_class(PROP_SETTING.knowledge_base)(PROP_SETTING.knowledge_base_path, scen)
-                if PROP_SETTING.knowledge_base != ""
-                else None
-            )
-            logger.log_object(knowledge_base, tag="knowledge_base")
-            self.hypothesis_gen: HypothesisGen = import_class(PROP_SETTING.hypothesis_gen)(scen)
-            logger.log_object(self.hypothesis_gen, tag="hypothesis generator")
-            self.hypothesis2experiment: Hypothesis2Experiment = import_class(PROP_SETTING.hypothesis2experiment)()
-            logger.log_object(self.hypothesis2experiment, tag="hypothesis2experiment")
-            self.feature_coder: Developer = import_class(PROP_SETTING.feature_coder)(scen)
-            logger.log_object(self.feature_coder, tag="feature coder")
-            self.model_feature_selection_coder: Developer = import_class(PROP_SETTING.model_feature_selection_coder)(
-                scen
-            )
-            logger.log_object(self.model_feature_selection_coder, tag="model feature selection coder")
-            self.model_coder: Developer = import_class(PROP_SETTING.model_coder)(scen)
-            logger.log_object(self.model_coder, tag="model coder")
-            self.feature_runner: Developer = import_class(PROP_SETTING.feature_runner)(scen)
-            logger.log_object(self.feature_runner, tag="feature runner")
-            self.model_runner: Developer = import_class(PROP_SETTING.model_runner)(scen)
-            logger.log_object(self.model_runner, tag="model runner")
-            self.summarizer: Experiment2Feedback = import_class(PROP_SETTING.summarizer)(scen)
-            logger.log_object(self.summarizer, tag="summarizer")
-            self.trace = KGTrace(scen=scen, knowledge_base=knowledge_base)
-            super(RDLoop, self).__init__()
+        scen: Scenario = import_class(PROP_SETTING.scen)(PROP_SETTING.competition)
+        logger.log_object(scen, tag="scenario")
+        knowledge_base = (
+            import_class(PROP_SETTING.knowledge_base)(PROP_SETTING.knowledge_base_path, scen)
+            if PROP_SETTING.knowledge_base != ""
+            else None
+        )
+        logger.log_object(knowledge_base, tag="knowledge_base")
+        self.hypothesis_gen: HypothesisGen = import_class(PROP_SETTING.hypothesis_gen)(scen)
+        logger.log_object(self.hypothesis_gen, tag="hypothesis generator")
+        self.hypothesis2experiment: Hypothesis2Experiment = import_class(PROP_SETTING.hypothesis2experiment)()
+        logger.log_object(self.hypothesis2experiment, tag="hypothesis2experiment")
+        self.feature_coder: Developer = import_class(PROP_SETTING.feature_coder)(scen)
+        logger.log_object(self.feature_coder, tag="feature coder")
+        self.model_feature_selection_coder: Developer = import_class(PROP_SETTING.model_feature_selection_coder)(scen)
+        logger.log_object(self.model_feature_selection_coder, tag="model feature selection coder")
+        self.model_coder: Developer = import_class(PROP_SETTING.model_coder)(scen)
+        logger.log_object(self.model_coder, tag="model coder")
+        self.feature_runner: Developer = import_class(PROP_SETTING.feature_runner)(scen)
+        logger.log_object(self.feature_runner, tag="feature runner")
+        self.model_runner: Developer = import_class(PROP_SETTING.model_runner)(scen)
+        logger.log_object(self.model_runner, tag="model runner")
+        self.summarizer: Experiment2Feedback = import_class(PROP_SETTING.summarizer)(scen)
+        logger.log_object(self.summarizer, tag="summarizer")
+        self.trace = KGTrace(scen=scen, knowledge_base=knowledge_base)
+        super(RDLoop, self).__init__()
 
     def coding(self, prev_out: dict[str, Any]):
-        with logger.tag("d"):  # develop
-            if prev_out["direct_exp_gen"]["propose"].action in [
-                KG_ACTION_FEATURE_ENGINEERING,
-                KG_ACTION_FEATURE_PROCESSING,
-            ]:
-                exp = self.feature_coder.develop(prev_out["direct_exp_gen"]["exp_gen"])
-            elif prev_out["direct_exp_gen"]["propose"].action == KG_ACTION_MODEL_FEATURE_SELECTION:
-                exp = self.model_feature_selection_coder.develop(prev_out["direct_exp_gen"]["exp_gen"])
-            else:
-                exp = self.model_coder.develop(prev_out["direct_exp_gen"]["exp_gen"])
-            logger.log_object(exp.sub_workspace_list, tag="coder result")
+        if prev_out["direct_exp_gen"]["propose"].action in [
+            KG_ACTION_FEATURE_ENGINEERING,
+            KG_ACTION_FEATURE_PROCESSING,
+        ]:
+            exp = self.feature_coder.develop(prev_out["direct_exp_gen"]["exp_gen"])
+        elif prev_out["direct_exp_gen"]["propose"].action == KG_ACTION_MODEL_FEATURE_SELECTION:
+            exp = self.model_feature_selection_coder.develop(prev_out["direct_exp_gen"]["exp_gen"])
+        else:
+            exp = self.model_coder.develop(prev_out["direct_exp_gen"]["exp_gen"])
+        logger.log_object(exp.sub_workspace_list, tag="coder result")
         return exp
 
     def running(self, prev_out: dict[str, Any]):
-        with logger.tag("ef"):  # evaluate and feedback
-            if prev_out["direct_exp_gen"]["propose"].action in [
-                KG_ACTION_FEATURE_ENGINEERING,
-                KG_ACTION_FEATURE_PROCESSING,
-            ]:
-                exp = self.feature_runner.develop(prev_out["coding"])
-            else:
-                exp = self.model_runner.develop(prev_out["coding"])
-            logger.log_object(exp, tag="runner result")
-            if KAGGLE_IMPLEMENT_SETTING.competition in [
-                "optiver-realized-volatility-prediction",
-                "covid19-global-forecasting-week-1",
-            ]:
-                try:
-                    python_files_to_notebook(
-                        KAGGLE_IMPLEMENT_SETTING.competition, exp.experiment_workspace.workspace_path
-                    )
-                except Exception as e:
-                    logger.error(f"Merge python files to one file failed: {e}")
-            if KAGGLE_IMPLEMENT_SETTING.auto_submit:
-                csv_path = exp.experiment_workspace.workspace_path / "submission.csv"
-                try:
-                    subprocess.run(
-                        [
-                            "kaggle",
-                            "competitions",
-                            "submit",
-                            "-f",
-                            str(csv_path.absolute()),
-                            "-m",
-                            str(csv_path.parent.absolute()),
-                            KAGGLE_IMPLEMENT_SETTING.competition,
-                        ],
-                        check=True,
-                    )
-                except subprocess.CalledProcessError as e:
-                    logger.error(f"Auto submission failed: \n{e}")
-                except Exception as e:
-                    logger.error(f"Other exception when use kaggle api:\n{e}")
+        if prev_out["direct_exp_gen"]["propose"].action in [
+            KG_ACTION_FEATURE_ENGINEERING,
+            KG_ACTION_FEATURE_PROCESSING,
+        ]:
+            exp = self.feature_runner.develop(prev_out["coding"])
+        else:
+            exp = self.model_runner.develop(prev_out["coding"])
+        logger.log_object(exp, tag="runner result")
+        if KAGGLE_IMPLEMENT_SETTING.competition in [
+            "optiver-realized-volatility-prediction",
+            "covid19-global-forecasting-week-1",
+        ]:
+            try:
+                python_files_to_notebook(KAGGLE_IMPLEMENT_SETTING.competition, exp.experiment_workspace.workspace_path)
+            except Exception as e:
+                logger.error(f"Merge python files to one file failed: {e}")
+        if KAGGLE_IMPLEMENT_SETTING.auto_submit:
+            csv_path = exp.experiment_workspace.workspace_path / "submission.csv"
+            try:
+                subprocess.run(
+                    [
+                        "kaggle",
+                        "competitions",
+                        "submit",
+                        "-f",
+                        str(csv_path.absolute()),
+                        "-m",
+                        str(csv_path.parent.absolute()),
+                        KAGGLE_IMPLEMENT_SETTING.competition,
+                    ],
+                    check=True,
+                )
+            except subprocess.CalledProcessError as e:
+                logger.error(f"Auto submission failed: \n{e}")
+            except Exception as e:
+                logger.error(f"Other exception when use kaggle api:\n{e}")
 
         return exp
 

rdagent/app/qlib_rd_loop/conf.py

@@ -67,10 +67,54 @@ class FactorFromReportPropSetting(FactorBasePropSetting):
     max_factors_per_exp: int = 10000
     """Maximum number of factors implemented per experiment"""
 
-    is_report_limit_enabled: bool = False
-    """Limits report processing count if True; processes all if False"""
+    report_limit: int = 10000
+    """Maximum number of reports to process"""
+
+
+class QuantBasePropSetting(BasePropSetting):
+    model_config = SettingsConfigDict(env_prefix="QLIB_QUANT_", protected_namespaces=())
+
+    # 1) override base settings
+    scen: str = "rdagent.scenarios.qlib.experiment.quant_experiment.QlibQuantScenario"
+    """Scenario class for Qlib Model"""
+
+    quant_hypothesis_gen: str = "rdagent.scenarios.qlib.proposal.quant_proposal.QlibQuantHypothesisGen"
+    """Hypothesis generation class"""
+
+    model_hypothesis2experiment: str = "rdagent.scenarios.qlib.proposal.model_proposal.QlibModelHypothesis2Experiment"
+    """Hypothesis to experiment class"""
+
+    model_coder: str = "rdagent.scenarios.qlib.developer.model_coder.QlibModelCoSTEER"
+    """Coder class"""
+
+    model_runner: str = "rdagent.scenarios.qlib.developer.model_runner.QlibModelRunner"
+    """Runner class"""
+
+    model_summarizer: str = "rdagent.scenarios.qlib.developer.feedback.QlibModelExperiment2Feedback"
+    """Summarizer class"""
+
+    factor_hypothesis2experiment: str = (
+        "rdagent.scenarios.qlib.proposal.factor_proposal.QlibFactorHypothesis2Experiment"
+    )
+    """Hypothesis to experiment class"""
+
+    factor_coder: str = "rdagent.scenarios.qlib.developer.factor_coder.QlibFactorCoSTEER"
+    """Coder class"""
+
+    factor_runner: str = "rdagent.scenarios.qlib.developer.factor_runner.QlibFactorRunner"
+    """Runner class"""
+
+    factor_summarizer: str = "rdagent.scenarios.qlib.developer.feedback.QlibFactorExperiment2Feedback"
+    """Summarizer class"""
+
+    evolving_n: int = 10
+    """Number of evolutions"""
+
+    action_selection: str = "bandit"
+    """Action selection strategy: 'bandit' for bandit-based selection, 'llm' for LLM-based selection, 'random' for random selection"""
 
 
 FACTOR_PROP_SETTING = FactorBasePropSetting()
 FACTOR_FROM_REPORT_PROP_SETTING = FactorFromReportPropSetting()
 MODEL_PROP_SETTING = ModelBasePropSetting()
+QUANT_PROP_SETTING = QuantBasePropSetting()

rdagent/app/qlib_rd_loop/factor.py

@@ -2,6 +2,8 @@
 Factor workflow with session control
 """
 
+import asyncio
+from pathlib import Path
 from typing import Any
 
 import fire
@@ -16,16 +18,21 @@ class FactorRDLoop(RDLoop):
     skip_loop_error = (FactorEmptyError,)
 
     def running(self, prev_out: dict[str, Any]):
-        with logger.tag("ef"):  # evaluate and feedback
-            exp = self.runner.develop(prev_out["coding"])
-            if exp is None:
-                logger.error(f"Factor extraction failed.")
-                raise FactorEmptyError("Factor extraction failed.")
-            logger.log_object(exp, tag="runner result")
+        exp = self.runner.develop(prev_out["coding"])
+        if exp is None:
+            logger.error(f"Factor extraction failed.")
+            raise FactorEmptyError("Factor extraction failed.")
+        logger.log_object(exp, tag="runner result")
         return exp
 
 
-def main(path=None, step_n=None):
+def main(
+    path: str | None = None,
+    step_n: int | None = None,
+    loop_n: int | None = None,
+    all_duration: str | None = None,
+    checkout: bool | str | Path = True,
+):
     """
     Auto R&D Evolving loop for fintech factors.
 
@@ -39,8 +46,8 @@ def main(path=None, step_n=None):
     if path is None:
         model_loop = FactorRDLoop(FACTOR_PROP_SETTING)
     else:
-        model_loop = FactorRDLoop.load(path)
-    model_loop.run(step_n=step_n)
+        model_loop = FactorRDLoop.load(path, checkout=checkout)
+    asyncio.run(model_loop.run(step_n=step_n, loop_n=loop_n, all_duration=all_duration))
 
 
 if __name__ == "__main__":

rdagent/app/qlib_rd_loop/factor_from_report.py

@@ -1,9 +1,9 @@
+import asyncio
 import json
 from pathlib import Path
 from typing import Any, Dict, Tuple
 
 import fire
-from jinja2 import Environment, StrictUndefined
 
 from rdagent.app.qlib_rd_loop.conf import FACTOR_FROM_REPORT_PROP_SETTING
 from rdagent.app.qlib_rd_loop.factor import FactorRDLoop
@@ -11,7 +11,7 @@ from rdagent.components.document_reader.document_reader import (
     extract_first_page_screenshot_from_pdf,
     load_and_process_pdfs_by_langchain,
 )
-from rdagent.core.prompts import Prompts
+from rdagent.core.conf import RD_AGENT_SETTINGS
 from rdagent.core.proposal import Hypothesis
 from rdagent.log import rdagent_logger as logger
 from rdagent.oai.llm_utils import APIBackend
@@ -19,11 +19,9 @@ from rdagent.scenarios.qlib.experiment.factor_experiment import QlibFactorExperi
 from rdagent.scenarios.qlib.factor_experiment_loader.pdf_loader import (
     FactorExperimentLoaderFromPDFfiles,
 )
+from rdagent.utils.agent.tpl import T
 from rdagent.utils.workflow import LoopMeta
 
-prompts_path = Path(__file__).parent / "prompts.yaml"
-prompts = Prompts(file_path=prompts_path)
-
 
 def generate_hypothesis(factor_result: dict, report_content: str) -> str:
     """
@@ -36,13 +34,9 @@ def generate_hypothesis(factor_result: dict, report_content: str) -> str:
     Returns:
         str: The generated hypothesis.
     """
-    system_prompt = (
-        Environment(undefined=StrictUndefined).from_string(prompts["hypothesis_generation"]["system"]).render()
-    )
-    user_prompt = (
-        Environment(undefined=StrictUndefined)
-        .from_string(prompts["hypothesis_generation"]["user"])
-        .render(factor_descriptions=json.dumps(factor_result), report_content=report_content)
+    system_prompt = T(".prompts:hypothesis_generation.system").r()
+    user_prompt = T(".prompts:hypothesis_generation.user").r(
+        factor_descriptions=json.dumps(factor_result), report_content=report_content
     )
 
     response = APIBackend().build_messages_and_create_chat_completion(
@@ -64,7 +58,7 @@ def generate_hypothesis(factor_result: dict, report_content: str) -> str:
     )
 
 
-def extract_hypothesis_and_exp_from_reports(report_file_path: str) -> Tuple[QlibFactorExperiment, Hypothesis]:
+def extract_hypothesis_and_exp_from_reports(report_file_path: str) -> QlibFactorExperiment | None:
     """
     Extract hypothesis and experiment details from report files.
 
@@ -72,17 +66,15 @@ def extract_hypothesis_and_exp_from_reports(report_file_path: str) -> Tuple[Qlib
         report_file_path (str): Path to the report file.
 
     Returns:
-        Tuple[QlibFactorExperiment, Hypothesis]: The extracted experiment and generated hypothesis.
+        QlibFactorExperiment: An instance of QlibFactorExperiment containing the extracted details.
+        None: If no valid experiment is found in the report.
     """
-    with logger.tag("extract_factors_and_implement"):
-        with logger.tag("load_factor_tasks"):
-            exp = FactorExperimentLoaderFromPDFfiles().load(report_file_path)
-            if exp is None or exp.sub_tasks == []:
-                return None, None
+    exp = FactorExperimentLoaderFromPDFfiles().load(report_file_path)
+    if exp is None or exp.sub_tasks == []:
+        return None
 
-        with logger.tag("load_pdf_screenshot"):
-            pdf_screenshot = extract_first_page_screenshot_from_pdf(report_file_path)
-            logger.log_object(pdf_screenshot)
+    pdf_screenshot = extract_first_page_screenshot_from_pdf(report_file_path)
+    logger.log_object(pdf_screenshot, tag="load_pdf_screenshot")
 
     docs_dict = load_and_process_pdfs_by_langchain(report_file_path)
 
@@ -99,7 +91,7 @@ def extract_hypothesis_and_exp_from_reports(report_file_path: str) -> Tuple[Qlib
     report_content = "\n".join(docs_dict.values())
     hypothesis = generate_hypothesis(factor_result, report_content)
     exp.hypothesis = hypothesis
-    return exp, hypothesis
+    return exp
 
 
 class FactorReportLoop(FactorRDLoop, metaclass=LoopMeta):
@@ -112,49 +104,40 @@ class FactorReportLoop(FactorRDLoop, metaclass=LoopMeta):
         else:
             self.judge_pdf_data_items = [i for i in Path(report_folder).rglob("*.pdf")]
 
-        self.pdf_file_index = 0
-        self.valid_pdf_file_count = 0
-        self.current_loop_hypothesis = None
-        self.current_loop_exp = None
-        self.steps = ["propose_hypo_exp", "propose", "direct_exp_gen", "coding", "running", "feedback"]
+        self.loop_n = min(len(self.judge_pdf_data_items), FACTOR_FROM_REPORT_PROP_SETTING.report_limit)
+        self.shift_report = (
+            0  # some reports does not contain viable factor, so we ship some of them to avoid infinite loop
+        )
 
-    def propose_hypo_exp(self, prev_out: dict[str, Any]):
-        with logger.tag("r"):
-            while True:
-                if FACTOR_FROM_REPORT_PROP_SETTING.is_report_limit_enabled and self.valid_pdf_file_count > 15:
-                    break
-                report_file_path = self.judge_pdf_data_items[self.pdf_file_index]
-                logger.info(f"Processing number {self.pdf_file_index} report: {report_file_path}")
-                self.pdf_file_index += 1
-                exp, hypothesis = extract_hypothesis_and_exp_from_reports(str(report_file_path))
+    async def direct_exp_gen(self, prev_out: dict[str, Any]):
+        while True:
+            if self.get_unfinished_loop_cnt(self.loop_idx) < RD_AGENT_SETTINGS.get_max_parallel():
+                report_file_path = self.judge_pdf_data_items[self.loop_idx + self.shift_report]
+                logger.info(f"Processing number {self.loop_idx} report: {report_file_path}")
+                exp = extract_hypothesis_and_exp_from_reports(str(report_file_path))
                 if exp is None:
+                    self.shift_report += 1
+                    self.loop_n -= 1
+                    if self.loop_n < 0:  # NOTE: on every step, we self.loop_n -= 1 at first.
+                        raise self.LoopTerminationError("Reach stop criterion and stop loop")
                     continue
-                self.valid_pdf_file_count += 1
-                exp.based_experiments = [QlibFactorExperiment(sub_tasks=[], hypothesis=hypothesis)] + [
+                exp.based_experiments = [QlibFactorExperiment(sub_tasks=[], hypothesis=exp.hypothesis)] + [
                     t[0] for t in self.trace.hist if t[1]
                 ]
                 exp.sub_workspace_list = exp.sub_workspace_list[: FACTOR_FROM_REPORT_PROP_SETTING.max_factors_per_exp]
                 exp.sub_tasks = exp.sub_tasks[: FACTOR_FROM_REPORT_PROP_SETTING.max_factors_per_exp]
-                logger.log_object(hypothesis, tag="hypothesis generation")
+                logger.log_object(exp.hypothesis, tag="hypothesis generation")
                 logger.log_object(exp.sub_tasks, tag="experiment generation")
-                self.current_loop_hypothesis = hypothesis
-                self.current_loop_exp = exp
-                return None
-
-    def propose(self, prev_out: dict[str, Any]):
-        return self.current_loop_hypothesis
-
-    def direct_exp_gen(self, prev_out: dict[str, Any]):
-        return {"propose": self.current_loop_hypothesis, "exp_gen": self.current_loop_exp}
+                return exp
+            await asyncio.sleep(1)
 
     def coding(self, prev_out: dict[str, Any]):
-        with logger.tag("d"):  # develop
-            exp = self.coder.develop(prev_out["direct_exp_gen"]["exp_gen"])
-            logger.log_object(exp.sub_workspace_list, tag="coder result")
+        exp = self.coder.develop(prev_out["direct_exp_gen"])
+        logger.log_object(exp.sub_workspace_list, tag="coder result")
         return exp
 
 
-def main(report_folder=None, path=None, step_n=None):
+def main(report_folder=None, path=None, all_duration=None, checkout=True):
     """
     Auto R&D Evolving loop for fintech factors (the factors are extracted from finance reports).
 
@@ -166,11 +149,11 @@ def main(report_folder=None, path=None, step_n=None):
     if path is None and report_folder is None:
         model_loop = FactorReportLoop()
     elif path is not None:
-        model_loop = FactorReportLoop.load(path)
+        model_loop = FactorReportLoop.load(path, checkout=checkout)
     else:
         model_loop = FactorReportLoop(report_folder=report_folder)
 
-    model_loop.run(step_n=step_n)
+    asyncio.run(model_loop.run(all_duration=all_duration))
 
 
 if __name__ == "__main__":

rdagent/app/qlib_rd_loop/model.py

@@ -2,6 +2,8 @@
 Model workflow with session control
 """
 
+import asyncio
+
 import fire
 
 from rdagent.app.qlib_rd_loop.conf import MODEL_PROP_SETTING
@@ -13,7 +15,7 @@ class ModelRDLoop(RDLoop):
     skip_loop_error = (ModelEmptyError,)
 
 
-def main(path=None, step_n=None):
+def main(path=None, step_n=None, loop_n=None, all_duration=None, checkout=True):
     """
     Auto R&D Evolving loop for fintech models
 
@@ -27,8 +29,8 @@ def main(path=None, step_n=None):
     if path is None:
         model_loop = ModelRDLoop(MODEL_PROP_SETTING)
     else:
-        model_loop = ModelRDLoop.load(path)
-    model_loop.run(step_n=step_n)
+        model_loop = ModelRDLoop.load(path, checkout=checkout)
+    asyncio.run(model_loop.run(step_n=step_n, loop_n=loop_n, all_duration=all_duration))
 
 
 if __name__ == "__main__":

rdagent/app/qlib_rd_loop/prompts.yaml

@@ -5,10 +5,6 @@ hypothesis_generation:
     {
       "hypothesis": "A clear and concise hypothesis based on the provided information.",
       "reason": "A detailed explanation supporting the generated hypothesis.",
-      "concise_reason": "One line summary that focuses on the justification for the change that leads to the hypothesis (like a part of a knowledge that we are building)",
-      "concise_observation": "One line summary. It focuses on the observation of the given scenario, data characteristics, or previous experiences (failures & succeses).",
-      "concise_justification": "One line summary. It focuses on the justification for the change in new hypothesis and the route of exploration supporting the growth of the hypothesis, based on the observation. ",
-      "concise_knowledge": "One line summary. It focuses on a transferable knowledege that comes with the new hypothesis. Use conditional grammar. eg. "If...., ..; When..., .; and etc"
     }
 
   user: |-

rdagent/app/qlib_rd_loop/quant.py

@@ -0,0 +1,137 @@
+"""
+Quant (Factor & Model) workflow with session control
+"""
+
+import asyncio
+from typing import Any
+
+import fire
+
+from rdagent.app.qlib_rd_loop.conf import QUANT_PROP_SETTING
+from rdagent.components.workflow.conf import BasePropSetting
+from rdagent.components.workflow.rd_loop import RDLoop
+from rdagent.core.conf import RD_AGENT_SETTINGS
+from rdagent.core.developer import Developer
+from rdagent.core.exception import FactorEmptyError, ModelEmptyError
+from rdagent.core.proposal import (
+    Experiment2Feedback,
+    Hypothesis2Experiment,
+    HypothesisFeedback,
+    HypothesisGen,
+)
+from rdagent.core.scenario import Scenario
+from rdagent.core.utils import import_class
+from rdagent.log import rdagent_logger as logger
+from rdagent.scenarios.qlib.proposal.quant_proposal import QuantTrace
+
+
+class QuantRDLoop(RDLoop):
+    skip_loop_error = (
+        FactorEmptyError,
+        ModelEmptyError,
+    )
+
+    def __init__(self, PROP_SETTING: BasePropSetting):
+        scen: Scenario = import_class(PROP_SETTING.scen)()
+        logger.log_object(scen, tag="scenario")
+
+        self.hypothesis_gen: HypothesisGen = import_class(PROP_SETTING.quant_hypothesis_gen)(scen)
+        logger.log_object(self.hypothesis_gen, tag="quant hypothesis generator")
+
+        self.factor_hypothesis2experiment: Hypothesis2Experiment = import_class(
+            PROP_SETTING.factor_hypothesis2experiment
+        )()
+        logger.log_object(self.factor_hypothesis2experiment, tag="factor hypothesis2experiment")
+        self.model_hypothesis2experiment: Hypothesis2Experiment = import_class(
+            PROP_SETTING.model_hypothesis2experiment
+        )()
+        logger.log_object(self.model_hypothesis2experiment, tag="model hypothesis2experiment")
+
+        self.factor_coder: Developer = import_class(PROP_SETTING.factor_coder)(scen)
+        logger.log_object(self.factor_coder, tag="factor coder")
+        self.model_coder: Developer = import_class(PROP_SETTING.model_coder)(scen)
+        logger.log_object(self.model_coder, tag="model coder")
+
+        self.factor_runner: Developer = import_class(PROP_SETTING.factor_runner)(scen)
+        logger.log_object(self.factor_runner, tag="factor runner")
+        self.model_runner: Developer = import_class(PROP_SETTING.model_runner)(scen)
+        logger.log_object(self.model_runner, tag="model runner")
+
+        self.factor_summarizer: Experiment2Feedback = import_class(PROP_SETTING.factor_summarizer)(scen)
+        logger.log_object(self.factor_summarizer, tag="factor summarizer")
+        self.model_summarizer: Experiment2Feedback = import_class(PROP_SETTING.model_summarizer)(scen)
+        logger.log_object(self.model_summarizer, tag="model summarizer")
+
+        self.trace = QuantTrace(scen=scen)
+        super(RDLoop, self).__init__()
+
+    async def direct_exp_gen(self, prev_out: dict[str, Any]):
+        while True:
+            if self.get_unfinished_loop_cnt(self.loop_idx) < RD_AGENT_SETTINGS.get_max_parallel():
+                hypo = self._propose()
+                assert hypo.action in ["factor", "model"]
+                if hypo.action == "factor":
+                    exp = self.factor_hypothesis2experiment.convert(hypo, self.trace)
+                else:
+                    exp = self.model_hypothesis2experiment.convert(hypo, self.trace)
+                logger.log_object(exp.sub_tasks, tag="experiment generation")
+                return {"propose": hypo, "exp_gen": exp}
+            await asyncio.sleep(1)
+
+    def coding(self, prev_out: dict[str, Any]):
+        if prev_out["direct_exp_gen"]["propose"].action == "factor":
+            exp = self.factor_coder.develop(prev_out["direct_exp_gen"]["exp_gen"])
+        elif prev_out["direct_exp_gen"]["propose"].action == "model":
+            exp = self.model_coder.develop(prev_out["direct_exp_gen"]["exp_gen"])
+        logger.log_object(exp, tag="coder result")
+        return exp
+
+    def running(self, prev_out: dict[str, Any]):
+        if prev_out["direct_exp_gen"]["propose"].action == "factor":
+            exp = self.factor_runner.develop(prev_out["coding"])
+            if exp is None:
+                logger.error(f"Factor extraction failed.")
+                raise FactorEmptyError("Factor extraction failed.")
+        elif prev_out["direct_exp_gen"]["propose"].action == "model":
+            exp = self.model_runner.develop(prev_out["coding"])
+        logger.log_object(exp, tag="runner result")
+        return exp
+
+    def feedback(self, prev_out: dict[str, Any]):
+        e = prev_out.get(self.EXCEPTION_KEY, None)
+        if e is not None:
+            feedback = HypothesisFeedback(
+                observations=str(e),
+                hypothesis_evaluation="",
+                new_hypothesis="",
+                reason="",
+                decision=False,
+            )
+            logger.log_object(feedback, tag="feedback")
+            self.trace.hist.append((prev_out["direct_exp_gen"]["exp_gen"], feedback))
+        else:
+            if prev_out["direct_exp_gen"]["propose"].action == "factor":
+                feedback = self.factor_summarizer.generate_feedback(prev_out["running"], self.trace)
+            elif prev_out["direct_exp_gen"]["propose"].action == "model":
+                feedback = self.model_summarizer.generate_feedback(prev_out["running"], self.trace)
+            logger.log_object(feedback, tag="feedback")
+            self.trace.hist.append((prev_out["running"], feedback))
+
+
+def main(path=None, step_n=None, loop_n=None, all_duration=None, checkout=True):
+    """
+    Auto R&D Evolving loop for fintech factors.
+    You can continue running session by
+    .. code-block:: python
+        dotenv run -- python rdagent/app/qlib_rd_loop/quant.py $LOG_PATH/__session__/1/0_propose  --step_n 1   # `step_n` is a optional paramter
+    """
+    if path is None:
+        quant_loop = QuantRDLoop(QUANT_PROP_SETTING)
+    else:
+        quant_loop = QuantRDLoop.load(path, checkout=checkout)
+
+    asyncio.run(quant_loop.run(step_n=step_n, loop_n=loop_n, all_duration=all_duration))
+
+
+if __name__ == "__main__":
+    fire.Fire(main)

rdagent/app/utils/ape.py

@@ -5,7 +5,7 @@ This is the preliminary version of the APE (Automated Prompt Engineering)
 import pickle
 from pathlib import Path
 
-from rdagent.core.conf import RD_AGENT_SETTINGS
+from rdagent.log.conf import LOG_SETTINGS
 
 
 def get_llm_qa(file_path):
@@ -21,7 +21,7 @@ def get_llm_qa(file_path):
 
 # Example usage
 # use
-file_path = Path(RD_AGENT_SETTINGS.log_trace_path) / "debug_llm.pkl"
+file_path = Path(LOG_SETTINGS.trace_path) / "debug_llm.pkl"
 llm_qa = get_llm_qa(file_path)
 print(len(llm_qa))
 

rdagent/app/utils/health_check.py

@@ -3,22 +3,25 @@ import socket
 import docker
 
 from rdagent.log import rdagent_logger as logger
+from rdagent.utils.env import cleanup_container
 
 
 def check_docker() -> None:
+    container = None
     try:
         client = docker.from_env()
         client.images.pull("hello-world")
         container = client.containers.run("hello-world", detach=True)
         logs = container.logs().decode("utf-8")
         print(logs)
-        container.remove()
         logger.info(f"The docker status is normal")
     except docker.errors.DockerException as e:
         logger.error(f"An error occurred: {e}")
         logger.warning(
             f"Docker status is exception, please check the docker configuration or reinstall it. Refs: https://docs.docker.com/engine/install/ubuntu/."
         )
+    finally:
+        cleanup_container(container, "health check")
 
 
 def is_port_in_use(port):

rdagent/components/coder/CoSTEER/__init__.py

@@ -103,8 +103,9 @@ class CoSTEER(Developer[Experiment]):
             assert isinstance(evo_exp, Experiment)  # multiple inheritance
             logger.log_object(evo_exp.sub_workspace_list, tag="evolving code")
             for sw in evo_exp.sub_workspace_list:
-                logger.info(f"evolving code workspace: {sw}")
+                logger.info(f"evolving workspace: {sw}")
             if (datetime.now() - start_datetime).seconds > self.max_seconds:
+                logger.info(f"Reached max time limit {self.max_seconds} seconds, stop evolving")
                 break
 
         if self.with_feedback and self.filter_final_evo:

rdagent/components/coder/CoSTEER/evaluators.py

@@ -1,4 +1,5 @@
 from abc import abstractmethod
+from copy import deepcopy
 from dataclasses import dataclass
 from typing import TYPE_CHECKING, List
 
@@ -113,21 +114,35 @@ class CoSTEERSingleFeedbackDeprecated(CoSTEERSingleFeedback):
         # Instead, we should create subclass for it.
         self.shape_feedback = shape_feedback  # Not general enough. So
 
-    # TODO: @property
     @property
     def execution(self):
         return self.execution_feedback
 
+    @execution.setter
+    def execution(self, value):
+        self.execution_feedback = value
+
     @property
     def return_checking(self):
         if self.value_generated_flag:
             return f"value feedback: {self.value_feedback}\n\nshape feedback: {self.shape_feedback}"
         return None
 
+    @return_checking.setter
+    def return_checking(self, value):
+        # Since return_checking is derived from value_feedback and shape_feedback,
+        # we don't need to do anything here
+        self.value_feedback = value
+        self.shape_feedback = value
+
     @property
     def code(self):
         return self.code_feedback
 
+    @code.setter
+    def code(self, value):
+        self.code_feedback = value
+
     def __str__(self) -> str:
         return f"""------------------Execution Feedback------------------
 {self.execution_feedback if self.execution_feedback is not None else 'No execution feedback'}
@@ -197,7 +212,7 @@ class CoSTEEREvaluator(Evaluator):
 class CoSTEERMultiEvaluator(CoSTEEREvaluator):
     """This is for evaluation of experiment. Due to we have multiple tasks, so we will return a list of evaluation feebacks"""
 
-    def __init__(self, single_evaluator: CoSTEEREvaluator, *args, **kwargs) -> None:
+    def __init__(self, single_evaluator: CoSTEEREvaluator | list[CoSTEEREvaluator], *args, **kwargs) -> None:
         super().__init__(*args, **kwargs)
         self.single_evaluator = single_evaluator
 
@@ -207,30 +222,56 @@ class CoSTEERMultiEvaluator(CoSTEEREvaluator):
         queried_knowledge: QueriedKnowledge = None,
         **kwargs,
     ) -> CoSTEERMultiFeedback:
-        multi_implementation_feedback = multiprocessing_wrapper(
-            [
-                (
-                    self.single_evaluator.evaluate,
+        eval_l = self.single_evaluator if isinstance(self.single_evaluator, list) else [self.single_evaluator]
+        task_li_feedback_li = []
+        for ev in eval_l:
+            multi_implementation_feedback = multiprocessing_wrapper(
+                [
                     (
-                        evo.sub_tasks[index],
-                        evo.sub_workspace_list[index],
-                        evo.sub_gt_implementations[index] if evo.sub_gt_implementations is not None else None,
-                        queried_knowledge,
+                        ev.evaluate,
+                        (
+                            evo.sub_tasks[index],
+                            evo.sub_workspace_list[index],
+                            evo.sub_gt_implementations[index] if evo.sub_gt_implementations is not None else None,
+                            queried_knowledge,
+                        ),
+                    )
+                    for index in range(len(evo.sub_tasks))
+                ],
+                n=RD_AGENT_SETTINGS.multi_proc_n,
+            )
+            task_li_feedback_li.append(multi_implementation_feedback)
+        # merge the feedbacks
+        merged_task_feedback = []
+        for task_id, fb in enumerate(task_li_feedback_li[0]):
+            fb = deepcopy(fb)  # deep copy to make it more robust
+
+            fb.final_decision = all(
+                task_li_feedback[task_id].final_decision for task_li_feedback in task_li_feedback_li
+            )
+            for attr in "execution", "return_checking", "code":
+                setattr(
+                    fb,
+                    attr,
+                    "\n\n".join(
+                        [
+                            getattr(task_li_feedback[task_id], attr)
+                            for task_li_feedback in task_li_feedback_li
+                            if getattr(task_li_feedback[task_id], attr) is not None
+                        ]
                     ),
                 )
-                for index in range(len(evo.sub_tasks))
-            ],
-            n=RD_AGENT_SETTINGS.multi_proc_n,
-        )
+            merged_task_feedback.append(fb)
 
         final_decision = [
             None if single_feedback is None else single_feedback.final_decision
-            for single_feedback in multi_implementation_feedback
+            for single_feedback in merged_task_feedback
         ]
         logger.info(f"Final decisions: {final_decision} True count: {final_decision.count(True)}")
 
+        # TODO: this is to be compatible with factor_implementation;
         for index in range(len(evo.sub_tasks)):
             if final_decision[index]:
                 evo.sub_tasks[index].factor_implementation = True
 
-        return CoSTEERMultiFeedback(multi_implementation_feedback)
+        return CoSTEERMultiFeedback(merged_task_feedback)

rdagent/components/coder/CoSTEER/evolving_strategy.py

@@ -1,7 +1,6 @@
 from __future__ import annotations
 
 from abc import abstractmethod
-from pathlib import Path
 
 from rdagent.components.coder.CoSTEER.config import CoSTEERSettings
 from rdagent.components.coder.CoSTEER.evaluators import (
@@ -15,12 +14,9 @@ from rdagent.components.coder.CoSTEER.knowledge_management import (
 from rdagent.core.conf import RD_AGENT_SETTINGS
 from rdagent.core.evolving_framework import EvolvingStrategy, EvoStep, QueriedKnowledge
 from rdagent.core.experiment import FBWorkspace, Task
-from rdagent.core.prompts import Prompts
 from rdagent.core.scenario import Scenario
 from rdagent.core.utils import multiprocessing_wrapper
 
-implement_prompts = Prompts(file_path=Path(__file__).parent / "prompts.yaml")
-
 
 class MultiProcessEvolvingStrategy(EvolvingStrategy):
     def __init__(self, scen: Scenario, settings: CoSTEERSettings):

rdagent/components/coder/CoSTEER/knowledge_management.py

@@ -8,8 +8,6 @@ from itertools import combinations
 from pathlib import Path
 from typing import List, Union
 
-from jinja2 import Environment, StrictUndefined
-
 from rdagent.components.coder.CoSTEER.config import CoSTEERSettings
 from rdagent.components.coder.CoSTEER.evaluators import CoSTEERSingleFeedback
 from rdagent.components.knowledge_management.graph import (
@@ -26,12 +24,12 @@ from rdagent.core.evolving_framework import (
     RAGStrategy,
 )
 from rdagent.core.experiment import FBWorkspace, Task
-from rdagent.core.prompts import Prompts
 from rdagent.log import rdagent_logger as logger
 from rdagent.oai.llm_utils import (
     APIBackend,
     calculate_embedding_distance_between_str_list,
 )
+from rdagent.utils.agent.tpl import T
 
 
 class CoSTEERKnowledge(Knowledge):
@@ -216,8 +214,6 @@ class CoSTEERQueriedKnowledgeV2(CoSTEERQueriedKnowledgeV1):
 
 
 class CoSTEERRAGStrategyV2(RAGStrategy):
-    prompt = Prompts(file_path=Path(__file__).parent / "prompts.yaml")
-
     def __init__(self, knowledgebase: CoSTEERKnowledgeBaseV2, settings: CoSTEERSettings) -> None:
         super().__init__(knowledgebase)
         self.current_generated_trace_count = 0
@@ -324,12 +320,8 @@ class CoSTEERRAGStrategyV2(RAGStrategy):
         all_component_content = ""
         for _, component_node in enumerate(all_component_nodes):
             all_component_content += f"{component_node.content}, \n"
-        analyze_component_system_prompt = (
-            Environment(undefined=StrictUndefined)
-            .from_string(self.prompt["analyze_component_prompt_v1_system"])
-            .render(
-                all_component_content=all_component_content,
-            )
+        analyze_component_system_prompt = T(".prompts:analyze_component_prompt_v1_system").r(
+            all_component_content=all_component_content,
         )
 
         analyze_component_user_prompt = target_task_information

rdagent/components/coder/data_science/conf.py

@@ -1,5 +1,6 @@
 from typing import Literal
 
+from rdagent.app.data_science.conf import DS_RD_SETTING
 from rdagent.components.coder.CoSTEER.config import CoSTEERSettings
 from rdagent.utils.env import (
     CondaConf,
@@ -23,7 +24,13 @@ class DSCoderCoSTEERSettings(CoSTEERSettings):
     # TODO: extract a function for env and conf.
 
 
-def get_ds_env(conf_type: Literal["kaggle", "mlebench"] = "kaggle") -> Env:
+def get_ds_env(
+    conf_type: Literal["kaggle", "mlebench"] = "kaggle",
+    extra_volumes: dict = {},
+    running_timeout_period: int = (
+        DS_RD_SETTING.debug_timeout if not DS_RD_SETTING.sample_data_by_LLM else DS_RD_SETTING.full_timeout
+    ),
+) -> Env:
     """
     Retrieve the appropriate environment configuration based on the env_type setting.
 
@@ -47,4 +54,18 @@ def get_ds_env(conf_type: Literal["kaggle", "mlebench"] = "kaggle") -> Env:
         )
     else:
         raise ValueError(f"Unknown env type: {conf.env_type}")
+    env.conf.extra_volumes = extra_volumes
+    env.conf.running_timeout_period = running_timeout_period
     return env
+
+
+def get_clear_ws_cmd(stage: Literal["before_training", "before_inference"] = "before_training") -> str:
+    """
+    Clean the files in workspace to a specific stage
+    """
+    assert stage in ["before_training", "before_inference"], f"Unknown stage: {stage}"
+    if DS_RD_SETTING.enable_model_dump and stage == "before_training":
+        cmd = "rm -r submission.csv scores.csv models"
+    else:
+        cmd = "rm submission.csv scores.csv"
+    return cmd

rdagent/components/coder/data_science/ensemble/__init__.py

@@ -11,9 +11,7 @@ File structure
     - Each coder could be tested.
 """
 
-import json
 from pathlib import Path
-from typing import Dict
 
 from jinja2 import Environment, StrictUndefined
 
@@ -74,7 +72,7 @@ class EnsembleMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
         )
 
         # Generate code with knowledge integration
-        competition_info = self.scen.get_scenario_all_desc()
+        competition_info = self.scen.get_scenario_all_desc(eda_output=workspace.file_dict.get("EDA.md", None))
         system_prompt = T(".prompts:ensemble_coder.system").r(
             task_desc=ensemble_information_str,
             competition_info=competition_info,

rdagent/components/coder/data_science/ensemble/eval.py

@@ -10,6 +10,7 @@ from rdagent.components.coder.CoSTEER.evaluators import (
     CoSTEERSingleFeedback,
 )
 from rdagent.components.coder.data_science.conf import get_ds_env
+from rdagent.components.coder.data_science.utils import remove_eda_part
 from rdagent.core.evolving_framework import QueriedKnowledge
 from rdagent.core.experiment import FBWorkspace, Task
 from rdagent.utils.agent.tpl import T
@@ -46,8 +47,7 @@ class EnsembleCoSTEEREvaluator(CoSTEEREvaluator):
                 final_decision=False,
             )
 
-        env = get_ds_env()
-        env.conf.extra_volumes = {f"{DS_RD_SETTING.local_data_path}/sample/{self.scen.competition}": "/kaggle/input"}
+        env = get_ds_env(extra_volumes={self.scen.debug_path: T("scenarios.data_science.share:scen.input_path").r()})
 
         fname = "test/ensemble_test.txt"
         test_code = (DIRNAME / "eval_tests" / "ensemble_test.txt").read_text()
@@ -63,13 +63,15 @@ class EnsembleCoSTEEREvaluator(CoSTEEREvaluator):
         )
 
         implementation.inject_files(**{fname: test_code})
-        stdout, ret_code = implementation.execute_ret_code(env=env, entry=f"python {fname}")
+        result = implementation.run(env=env, entry=f"python {fname}")
+        stdout = result.stdout
+        ret_code = result.exit_code
 
         stdout += f"\nNOTE: the above scripts run with return code {ret_code}"
 
         if "main.py" in implementation.file_dict and ret_code == 0:
             workflow_stdout = implementation.execute(env=env, entry="python main.py")
-            workflow_stdout = re.sub(r"=== Start of EDA part ===(.*)=== End of EDA part ===", "", workflow_stdout)
+            workflow_stdout = remove_eda_part(workflow_stdout)
         else:
             workflow_stdout = None
 

rdagent/components/coder/data_science/ensemble/eval_tests/ensemble_test.txt

@@ -128,5 +128,10 @@ assert model_set_in_scores == set({{model_names}}).union({"ensemble"}), (
 assert score_df.index.is_unique, "The scores dataframe has duplicate model names."
 assert score_df.columns.tolist() == ["{{metric_name}}"], f"The column names of the scores dataframe should be ['{{metric_name}}'], but is '{score_df.columns.tolist()}'"
 
+# Check for NaN values in score_df
+assert not score_df.isnull().values.any(), (
+    f"The scores dataframe contains NaN values at the following locations:\n{score_df[score_df.isnull().any(axis=1)]}"
+)
+
 
 print("Ensemble test end.")

rdagent/components/coder/data_science/ensemble/prompts.yaml

@@ -39,6 +39,7 @@ ensemble_coder:
     1. The function's code is associated with several other functions including a data loader, feature engineering, and model training. all codes are as follows:
     {{ all_code }}
     2. You should avoid using logging module to output information in your generated code, and instead use the print() function.
+    {% include "scenarios.data_science.share:guidelines.coding" %}
 
     ## Output Format
     {% if out_spec %}

rdagent/components/coder/data_science/feature/__init__.py

@@ -61,7 +61,7 @@ class FeatureMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
 
         # 2. code
         system_prompt = T(".prompts:feature_coder.system").r(
-            competition_info=self.scen.get_scenario_all_desc(),
+            competition_info=self.scen.get_scenario_all_desc(eda_output=workspace.file_dict.get("EDA.md", None)),
             task_desc=feature_information_str,
             data_loader_code=workspace.file_dict.get("load_data.py"),
             queried_similar_successful_knowledge=queried_similar_successful_knowledge,

rdagent/components/coder/data_science/feature/eval.py

@@ -8,6 +8,7 @@ from rdagent.components.coder.CoSTEER.evaluators import (
     CoSTEERSingleFeedback,
 )
 from rdagent.components.coder.data_science.conf import get_ds_env
+from rdagent.components.coder.data_science.utils import remove_eda_part
 from rdagent.core.evolving_framework import QueriedKnowledge
 from rdagent.core.experiment import FBWorkspace, Task
 from rdagent.utils.agent.tpl import T
@@ -20,7 +21,6 @@ FeatureEvalFeedback = CoSTEERSingleFeedback
 
 
 class FeatureCoSTEEREvaluator(CoSTEEREvaluator):
-
     def evaluate(
         self,
         target_task: Task,
@@ -29,7 +29,6 @@ class FeatureCoSTEEREvaluator(CoSTEEREvaluator):
         queried_knowledge: QueriedKnowledge = None,
         **kwargs,
     ) -> FeatureEvalFeedback:
-
         target_task_information = target_task.get_task_information()
         if (
             queried_knowledge is not None
@@ -44,19 +43,18 @@ class FeatureCoSTEEREvaluator(CoSTEEREvaluator):
                 final_decision=False,
             )
 
-        env = get_ds_env()
-        env.conf.extra_volumes = {f"{DS_RD_SETTING.local_data_path}/sample/{self.scen.competition}": "/kaggle/input"}
+        env = get_ds_env(extra_volumes={self.scen.debug_path: T("scenarios.data_science.share:scen.input_path").r()})
 
         # TODO: do we need to clean the generated temporary content?
         fname = "test/feature_test.py"
         test_code = (DIRNAME / "eval_tests" / "feature_test.txt").read_text()
         implementation.inject_files(**{fname: test_code})
 
-        stdout, ret_code = implementation.execute_ret_code(env=env, entry=f"python {fname}")
+        result = implementation.run(env=env, entry=f"python {fname}")
 
-        if "main.py" in implementation.file_dict and ret_code == 0:
+        if "main.py" in implementation.file_dict and result.exit_code == 0:
             workflow_stdout = implementation.execute(env=env, entry="python main.py")
-            workflow_stdout = re.sub(r"=== Start of EDA part ===(.*)=== End of EDA part ===", "", workflow_stdout)
+            workflow_stdout = remove_eda_part(workflow_stdout)
         else:
             workflow_stdout = None
 
@@ -68,13 +66,16 @@ class FeatureCoSTEEREvaluator(CoSTEEREvaluator):
             workflow_code=implementation.all_codes,
         )
         user_prompt = T(".prompts:feature_eval.user").r(
-            stdout=shrink_text(stdout),
+            stdout=shrink_text(result.stdout),
             workflow_stdout=workflow_stdout,
         )
 
-        return build_cls_from_json_with_retry(
+        fb = build_cls_from_json_with_retry(
             FeatureEvalFeedback,
             system_prompt=system_prompt,
             user_prompt=user_prompt,
             init_kwargs_update_func=FeatureEvalFeedback.val_and_update_init_dict,
         )
+        fb.final_decision = fb.final_decision and result.exit_code == 0
+
+        return fb

rdagent/components/coder/data_science/feature/eval_tests/feature_test.txt

@@ -28,10 +28,22 @@ X_loaded = deepcopy(X)
 y_loaded = deepcopy(y)
 X_test_loaded = deepcopy(X_test)
 
+import sys
+import reprlib
+from joblib.memory import MemorizedFunc
+
+
+def get_original_code(func):
+    if isinstance(func, MemorizedFunc):
+        return func.func.__code__
+    return func.__code__
+
+
 def debug_info_print(func):
     def wrapper(*args, **kwargs):
+        original_code = get_original_code(func)
         def local_trace(frame, event, arg):
-            if event == "return" and frame.f_code == func.__code__:
+            if event == "return" and frame.f_code == original_code:
                 print("\n" + "="*20 + "Running feat_eng code, local variable values:" + "="*20)
                 for k, v in frame.f_locals.items():
                     printed = aRepr.repr(v)

rdagent/components/coder/data_science/feature/prompts.yaml

@@ -39,10 +39,19 @@ feature_coder:
     ```python
     {{ data_loader_code }}
     ```
-    3. **Additional Guidance:**
+    4. **Additional Guidance:**
       - If a previous attempt exists, improve upon it without repeating mistakes.
       - If errors indicate a missing file, find a way to download it or implement an alternative solution.
       - You should avoid using logging module to output information in your generated code, and instead use the print() function.
+    5. You should use the following cache decorator to cache the results of the function:
+    ```python
+    from joblib import Memory
+    memory = Memory(location='{% include "scenarios.data_science.share:scen.cache_path" %}', verbose=0)
+    @memory.cache```
+    6. Coding tricks:
+      - If the input consists of a batch of file paths and you need to modify the file contents to complete your feature engineering task, you can accomplish your feature engineering task by modifying these files and creating new files in a subfolder within "{% include "scenarios.data_science.share:scen.cache_path" %}" (this path is persistent, otherwise you may lose your created file). Then the new file paths are returned.
+
+    {% include "scenarios.data_science.share:guidelines.coding" %}
 
     ## Output Format
     {% if out_spec %}

rdagent/components/coder/data_science/model/__init__.py

@@ -62,7 +62,7 @@ class ModelMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
         # 2. code
         system_prompt = T(".prompts:model_coder.system").r(
             task_desc=model_information_str,
-            competition_info=self.scen.get_scenario_all_desc(),
+            competition_info=self.scen.get_scenario_all_desc(eda_output=workspace.file_dict.get("EDA.md", None)),
             data_loader_code=workspace.file_dict.get("load_data.py"),
             feature_code=workspace.file_dict["feature.py"],
             queried_similar_successful_knowledge=queried_similar_successful_knowledge,

rdagent/components/coder/data_science/model/eval.py

@@ -13,6 +13,7 @@ from rdagent.components.coder.CoSTEER.evaluators import (
     CoSTEERSingleFeedback,
 )
 from rdagent.components.coder.data_science.conf import get_ds_env
+from rdagent.components.coder.data_science.utils import remove_eda_part
 from rdagent.core.evolving_framework import QueriedKnowledge
 from rdagent.core.exception import CoderError
 from rdagent.core.experiment import FBWorkspace, Task
@@ -56,8 +57,7 @@ class ModelGeneralCaseSpecEvaluator(CoSTEEREvaluator):
                 final_decision=False,
             )
 
-        env = get_ds_env()
-        env.conf.extra_volumes = {f"{DS_RD_SETTING.local_data_path}/sample/{self.scen.competition}": "/kaggle/input"}
+        env = get_ds_env(extra_volumes={self.scen.debug_path: T("scenarios.data_science.share:scen.input_path").r()})
 
         if_model_removed = False
 
@@ -67,7 +67,9 @@ class ModelGeneralCaseSpecEvaluator(CoSTEEREvaluator):
                 (DIRNAME / "eval_tests" / "model_test.txt").read_text().replace("model01", target_task.name)
             )  # only check the model changed this time
             implementation.inject_files(**{fname: test_code})
-            stdout, ret_code = implementation.execute_ret_code(env=env, entry=f"python {fname}")
+            result = implementation.run(env=env, entry=f"python {fname}")
+            stdout = result.stdout
+            ret_code = result.exit_code
 
             if stdout is None:
                 raise CoderError(
@@ -80,7 +82,7 @@ class ModelGeneralCaseSpecEvaluator(CoSTEEREvaluator):
 
         if "main.py" in implementation.file_dict and ret_code == 0:
             workflow_stdout = implementation.execute(env=env, entry="python main.py")
-            workflow_stdout = re.sub(r"=== Start of EDA part ===(.*)=== End of EDA part ===", "", workflow_stdout)
+            workflow_stdout = remove_eda_part(workflow_stdout)
         else:
             workflow_stdout = None
 
@@ -107,9 +109,12 @@ class ModelGeneralCaseSpecEvaluator(CoSTEEREvaluator):
                 workflow_stdout=workflow_stdout,
             )
 
-        return build_cls_from_json_with_retry(
+        fb = build_cls_from_json_with_retry(
             ModelSingleFeedback,
             system_prompt=system_prompt,
             user_prompt=user_prompt,
             init_kwargs_update_func=ModelSingleFeedback.val_and_update_init_dict,
         )
+        fb.final_decision = fb.final_decision and result.exit_code == 0
+
+        return fb

rdagent/components/coder/data_science/model/eval_tests/model_test.txt

@@ -35,6 +35,17 @@ print(f"test_ids length: {len(test_ids)}")
 
 train_X, val_X, train_y, val_y = train_test_split(X, y, test_size=0.8, random_state=42)
 
+
+import sys
+import reprlib
+from joblib.memory import MemorizedFunc
+
+
+def get_original_code(func):
+    if isinstance(func, MemorizedFunc):
+        return func.func.__code__
+    return func.__code__
+
 print("train_X:", aRepr.repr(train_X))
 print("train_y:", aRepr.repr(train_y))
 print("val_X:", aRepr.repr(val_X))
@@ -46,10 +57,12 @@ print(f"val_X.shape: {val_X.shape}" if hasattr(val_X, 'shape') else f"val_X leng
 print(f"val_y.shape: {val_y.shape}" if hasattr(val_y, 'shape') else f"val_y length: {len(val_y)}")
 
 
+
 def debug_info_print(func):
     def wrapper(*args, **kwargs):
+        original_code = get_original_code(func)
         def local_trace(frame, event, arg):
-            if event == "return" and frame.f_code == func.__code__:
+            if event == "return" and frame.f_code == original_code:
                 print("\n" + "="*20 + "Running model training code, local variable values:" + "="*20)
                 for k, v in frame.f_locals.items():
                     printed = aRepr.repr(v)

rdagent/components/coder/data_science/model/prompts.yaml

@@ -40,6 +40,12 @@ model_coder:
     {{ feature_code }}
     2. You should avoid using logging module to output information in your generated code, and instead use the print() function.
     3. If the model can both be implemented by PyTorch and Tensorflow, please use pytorch for broader compatibility.
+    4. You should use the following cache decorator to cache the results of the function:
+    ```python
+    from joblib import Memory
+    memory = Memory(location='{% include "scenarios.data_science.share:scen.cache_path" %}', verbose=0)
+    @memory.cache``
+    {% include "scenarios.data_science.share:guidelines.coding" %}
 
     ## Output Format
     {% if out_spec %}

rdagent/components/coder/data_science/pipeline/__init__.py

@@ -48,6 +48,7 @@ from rdagent.components.coder.data_science.raw_data_loader.eval import (
     DataLoaderCoSTEEREvaluator,
 )
 from rdagent.components.coder.data_science.raw_data_loader.exp import DataLoaderTask
+from rdagent.components.coder.data_science.share.eval import ModelDumpEvaluator
 from rdagent.core.exception import CoderError
 from rdagent.core.experiment import FBWorkspace
 from rdagent.core.scenario import Scenario
@@ -66,7 +67,7 @@ class PipelineMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
         workspace: FBWorkspace | None = None,
         prev_task_feedback: CoSTEERSingleFeedback | None = None,
     ) -> dict[str, str]:
-        competition_info = self.scen.get_scenario_all_desc()
+        competition_info = self.scen.get_scenario_all_desc(eda_output=workspace.file_dict.get("EDA.md", None))
         runtime_environment = self.scen.get_runtime_environment()
         data_folder_info = self.scen.processed_data_folder_description
         pipeline_task_info = target_task.get_task_information()
@@ -95,6 +96,8 @@ class PipelineMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
             out_spec=PythonAgentOut.get_spec(),
             runtime_environment=runtime_environment,
             spec=T("scenarios.data_science.share:component_spec.Pipeline").r(),
+            enable_model_dump=DS_RD_SETTING.enable_model_dump,
+            enable_debug_mode=DS_RD_SETTING.sample_data_by_LLM,
         )
         user_prompt = T(".prompts:pipeline_coder.user").r(
             competition_info=competition_info,
@@ -146,8 +149,12 @@ class PipelineCoSTEER(CoSTEER):
         **kwargs,
     ) -> None:
         settings = DSCoderCoSTEERSettings()
+        eval_l = [PipelineCoSTEEREvaluator(scen=scen)]
+        if DS_RD_SETTING.enable_model_dump:
+            eval_l.append(ModelDumpEvaluator(scen=scen, data_type="sample"))
+
         eva = CoSTEERMultiEvaluator(
-            PipelineCoSTEEREvaluator(scen=scen), scen=scen
+            single_evaluator=eval_l, scen=scen
         )  # Please specify whether you agree running your eva in parallel or not
         es = PipelineMultiProcessEvolvingStrategy(scen=scen, settings=settings)
 

rdagent/components/coder/data_science/pipeline/eval.py

@@ -15,8 +15,10 @@ from rdagent.components.coder.CoSTEER.evaluators import (
 from rdagent.components.coder.CoSTEER.knowledge_management import (
     CoSTEERQueriedKnowledgeV2,
 )
-from rdagent.components.coder.data_science.conf import get_ds_env
+from rdagent.components.coder.data_science.conf import get_clear_ws_cmd, get_ds_env
+from rdagent.components.coder.data_science.utils import remove_eda_part
 from rdagent.core.experiment import FBWorkspace, Task
+from rdagent.scenarios.data_science.test_eval import get_test_eval
 from rdagent.utils.agent.tpl import T
 from rdagent.utils.agent.workflow import build_cls_from_json_with_retry
 
@@ -51,13 +53,35 @@ class PipelineCoSTEEREvaluator(CoSTEEREvaluator):
                 final_decision=False,
             )
 
-        env = get_ds_env()
-        env.conf.extra_volumes = {f"{DS_RD_SETTING.local_data_path}/sample/{self.scen.competition}": "/kaggle/input"}
+        env = get_ds_env(extra_volumes={self.scen.debug_path: T("scenarios.data_science.share:scen.input_path").r()})
 
-        # Clean the scores.csv & submission.csv.
-        implementation.execute(env=env, entry=f"rm submission.csv scores.csv")
-        stdout = implementation.execute(env=env, entry=f"python main.py")
-        stdout = re.sub(r"=== Start of EDA part ===(.*)=== End of EDA part ===", "", stdout)
+        stdout = ""
+        implementation.execute(env=env, entry=get_clear_ws_cmd())
+        if DS_RD_SETTING.sample_data_by_LLM:
+            # Because coder runs on full data, we need to run debug mode in advance to save time
+            result = implementation.run(env=env, entry=f"python -m coverage run main.py --debug")
+        else:
+            result = implementation.run(env=env, entry=f"python -m coverage run main.py")
+        execute_ret_code = result.exit_code
+        result.stdout = remove_eda_part(result.stdout)
+        if result.exit_code != 0:
+            stdout += f"Code failed to run. Please check the stdout:\n Following the stdout of the debug mode run:\n{result.stdout.strip()}\n"
+        else:
+            stdout += f"Code ran successfully.\n Following the stdout of the debug mode run:\n{result.stdout.strip()}\n"
+        if DS_RD_SETTING.sample_data_by_LLM:
+            debug_time, full_estimated_time = None, None
+            if match := re.search(r"debug_time:\s*(\d+(?:.\d+)?)", result.stdout, re.DOTALL):
+                debug_time = float(match.group(1))
+            if match := re.search(r"estimated_time:\s*(\d+(?:.\d+)?)", result.stdout, re.DOTALL):
+                full_estimated_time = float(match.group(1))
+            if debug_time is not None and full_estimated_time is not None:
+                stdout += f"Debug mode ran in {debug_time:.2f} seconds, estimated full run time is {full_estimated_time:.2f} seconds.\n"
+                if full_estimated_time < env.conf.running_timeout_period * 3:
+                    stdout += "The estimated full run time is less than three times the timeout period.\n"
+                else:
+                    stdout += f"The estimated full run time is more than three times the timeout period.\n"
+            else:
+                stdout += "Debug mode did not provide debug_time or estimated_time, it's a buggy implementation.\n"
 
         score_fp = implementation.workspace_path / "scores.csv"
         score_ret_code = 0
@@ -71,34 +95,69 @@ class PipelineCoSTEEREvaluator(CoSTEEREvaluator):
                 model_set_in_scores = set(score_df.index)
 
                 # Check model names (index)
-                if "ensemble" not in model_set_in_scores:
-                    score_check_text += (
-                        f"\n[Error] The score dataframe doesn't contain the ensemble model.\nscore_df is:\n{score_df}"
-                    )
+                if not score_df.index.is_unique:
+                    score_check_text += "\n[Error] The file 'scores.csv' contains duplicate model names."
                     score_ret_code = 1
+                if "ensemble" not in model_set_in_scores:
+                    score_check_text += "\n[Error] The file 'scores.csv' doesn't contain the ensemble model."
+                    score_ret_code = 1
+                if score_ret_code != 0:
+                    score_check_text += f"The dataframe in file 'scores.csv' is:\n{score_df}"
 
                 # Check metric name (columns)
                 if score_df.columns.tolist() != [self.scen.metric_name]:
                     score_check_text += f"\n[Error] The scores dataframe does not contain the correct column names.\nCorrect columns is: ['{self.scen.metric_name}']\nBut got: {score_df.columns.tolist()}"
                     score_ret_code = 1
 
+                # Check if scores contain NaN (values)
+                if score_df.isnull().values.any():
+                    nan_locations = score_df[score_df.isnull().any(axis=1)]
+                    score_check_text += f"\n[Error] The scores dataframe contains NaN values at the following locations:\n{nan_locations}"
+                    score_ret_code = 1
+
             except Exception as e:
                 score_check_text += f"\n[Error] in checking the scores.csv file: {e}\nscores.csv's content:\n-----\n{score_fp.read_text()}\n-----"
                 score_ret_code = 1
 
-        # Check submission file
-        base_check_code = (DIRNAME / "eval_tests" / "submission_format_test.txt").read_text()
-        implementation.inject_files(**{"test/submission_format_test.py": base_check_code})
-        # stdout += "----Submission Check 1-----\n"
-        submission_check_out, submission_ret_code = implementation.execute_ret_code(
-            env=env, entry="python test/submission_format_test.py"
-        )
-        stdout += "\n" + submission_check_out
+        test_eval = get_test_eval()
+        if not test_eval.is_sub_enabled(self.scen.competition):
+            submission_ret_code = 0
+        else:
+            # Check submission file
+            base_check_code = T(".eval_tests.submission_format_test", ftype="txt").r()
+            implementation.inject_files(**{"test/submission_format_test.py": base_check_code})
+            # stdout += "----Submission Check 1-----\n"
+            submission_result = implementation.run(env=env, entry="python test/submission_format_test.py")
+            submission_check_out = submission_result.stdout
+            submission_ret_code = submission_result.exit_code
+            if DS_RD_SETTING.rule_base_eval:
+                if execute_ret_code == 0 and score_ret_code == 0 and submission_ret_code == 0:
+                    return PipelineSingleFeedback(
+                        execution=stdout,
+                        return_checking=score_check_text + "\n" + submission_check_out,
+                        code="Code evaluation is not available.",
+                        final_decision=True,
+                    )
+                else:
+                    return PipelineSingleFeedback(
+                        execution=stdout,
+                        return_checking=score_check_text + "\n" + submission_check_out,
+                        code="Code evaluation is not available.",
+                        final_decision=False,
+                    )
+            stdout += "\n" + submission_check_out
+
+        if not isinstance(implementation, FBWorkspace):
+            eda_output = None
+        else:
+            eda_output = implementation.file_dict.get("EDA.md", None)
 
         system_prompt = T(".prompts:pipeline_eval.system").r(
-            scenario=self.scen.get_scenario_all_desc(),
+            scenario=self.scen.get_scenario_all_desc(eda_output=eda_output),
             task_desc=target_task.get_task_information(),
+            is_sub_enabled=test_eval.is_sub_enabled(self.scen.competition),
             spec=T("scenarios.data_science.share:component_spec.Pipeline").r(),
+            debug_mode=DS_RD_SETTING.sample_data_by_LLM,
         )
         user_prompt = T(".prompts:pipeline_eval.user").r(
             stdout=stdout.strip(),

rdagent/components/coder/data_science/pipeline/eval_tests/submission_format_test.txt

@@ -1,14 +1,20 @@
+import hashlib
 from pathlib import Path
 import pandas as pd
-import hashlib
+
 
 def calculate_md5(file_path):
     with open(file_path, "rb") as f:
         file_hash = hashlib.md5(f.read()).hexdigest()
     return file_hash
 
-file_md5 = calculate_md5("scores.csv")
 
+if Path("scores.csv").exists():
+    file_md5 = calculate_md5("scores.csv")
+else:
+    print("Warning: scores.csv does not exist. MD5 check will be skipped.")
+    file_md5 = None
+    
 """
 find . | grep -i sample | grep -i submission | grep -v sample_submission.csv | grep -v zip_files  | grep -v 'sample/'
 ./denoising-dirty-documents/sampleSubmission.csv
@@ -22,56 +28,67 @@ find . | grep -i sample | grep -i submission | grep -v sample_submission.csv | g
 """
 
 # Find sample submission file dynamically
-input_dir = Path("/kaggle/input")
-# Look for common variations of sample submission filenames
-sample_submission_files = list(input_dir.glob("*sample_submission*.csv")) + \
-                         list(input_dir.glob("*sampleSubmission*.csv"))
+input_dir = Path('{% include "scenarios.data_science.share:scen.input_path" %}')
+sample_submission_files = list(input_dir.glob("*sample_submission*.csv")) + list(
+    input_dir.glob("*sampleSubmission*.csv")
+)
 
-assert sample_submission_files, "Error: No sample submission file found in /kaggle/input/"
+if not sample_submission_files:
+    print(f'Error: No sample submission file found in {% include "scenarios.data_science.share:scen.input_path" %}')
+    sample_submission_name = None
+    SAMPLE_SUBMISSION_PATH = None
+else:
+    sample_submission_name = sample_submission_files[0].name
+    SAMPLE_SUBMISSION_PATH = str(sample_submission_files[0])
+    print(f"Using sample submission file: {sample_submission_name}")
 
-# Use first matching file
-sample_submission_name = sample_submission_files[0].name
-SAMPLE_SUBMISSION_PATH = str(sample_submission_files[0])
-print(f"Using sample submission file: {sample_submission_name}")
+if SAMPLE_SUBMISSION_PATH is not None and not Path(SAMPLE_SUBMISSION_PATH).exists():
+    print(f"Error: {sample_submission_name} not found at {SAMPLE_SUBMISSION_PATH}")
 
-# Check if the sample submission file exists
-assert Path(SAMPLE_SUBMISSION_PATH).exists(), f"Error: {sample_submission_name} not found at {SAMPLE_SUBMISSION_PATH}"
+if not Path("submission.csv").exists():
+    print("Error: submission.csv not found")
 
-# Check if our submission file exists
-assert Path('submission.csv').exists(), "Error: submission.csv not found"
+if SAMPLE_SUBMISSION_PATH is not None and Path(SAMPLE_SUBMISSION_PATH).exists() and Path("submission.csv").exists():
+    sample_submission = pd.read_csv(SAMPLE_SUBMISSION_PATH)
+    our_submission = pd.read_csv("submission.csv")
 
-sample_submission = pd.read_csv(SAMPLE_SUBMISSION_PATH)
-our_submission = pd.read_csv('submission.csv')
+    success = True
+    print(f"Columns in {sample_submission_name}:", sample_submission.columns)
+    print("Columns in our_submission.csv:", our_submission.columns)
 
-success = True
-# Print the columns of the sample submission file
-print(f"Columns in {sample_submission_name}:", sample_submission.columns)
-print("Columns in our_submission.csv:", our_submission.columns)
+    for col in sample_submission.columns:
+        if col not in our_submission.columns:
+            success = False
+            print(f"Column {col} not found in submission.csv")
 
-for col in sample_submission.columns:
-    if col not in our_submission.columns:
-        success = False
-        print(f'Column {col} not found in submission.csv')
-
-if success:
-    print(f'submission.csv\'s columns aligns with {sample_submission_name} .')
+    if success:
+        print(f"submission.csv's columns aligns with {sample_submission_name} .")
+    else:
+        print(f"submission.csv's columns does not align with {sample_submission_name} .")
 
 
-# Print the first 5 rows of the two submission files, with columns separated by commas.
-def print_first_rows(file_path, file_name, num_rows=5):
-    print(f"\nFirst {num_rows} rows of {file_name}:")
-    try:
-        with open(file_path, 'r') as file:
-            for i, line in enumerate(file):
-                if i < num_rows:
-                    print(line.strip())
-                else:
-                    break
-    except FileNotFoundError:
-        print(f"Error: {file_name} not found.")
+    def print_first_rows(file_path, file_name, num_rows=5):
+        print(f"\nFirst {num_rows} rows of {file_name}:")
+        try:
+            with open(file_path, "r") as file:
+                for i, line in enumerate(file):
+                    if i < num_rows:
+                        print(line.strip())
+                    else:
+                        break
+        except FileNotFoundError:
+            print(f"Error: {file_name} not found.")
 
-print_first_rows(SAMPLE_SUBMISSION_PATH, sample_submission_name)
-print_first_rows('submission.csv', 'submission.csv')
 
-assert calculate_md5("scores.csv") == file_md5, "scores.csv should not be rewritten"
-print(f"\nPlease Checked the content of the submission file(submission.csv should has the same format with {sample_submission_name} but might not the same index with {sample_submission_name}). ")
+    print_first_rows(SAMPLE_SUBMISSION_PATH, sample_submission_name)
+    print_first_rows("submission.csv", "submission.csv")
+
+    if file_md5 is not None:
+        if calculate_md5("scores.csv") != file_md5:
+            print("Warning: scores.csv has been rewritten in the test script!")
+else:
+    print("Skipping comparison and preview due to missing files.")
+
+print(
+    f"\nPlease Checked the content of the submission file(submission.csv should has the same format with {sample_submission_name} but might not the same index with {sample_submission_name}). "
+)

rdagent/components/coder/data_science/pipeline/exp.py

@@ -3,4 +3,5 @@ from rdagent.components.coder.CoSTEER.task import CoSTEERTask
 
 # Because we use isinstance to distinguish between different types of tasks, we need to use sub classes to represent different types of tasks
 class PipelineTask(CoSTEERTask):
-    pass
+    def __init__(self, name: str = "Pipeline", *args, **kwargs) -> None:
+        super().__init__(name=name, *args, **kwargs)

rdagent/components/coder/data_science/pipeline/prompts.yaml

@@ -37,8 +37,9 @@ pipeline_coder:
 
 
     ## Guidelines
-    1. Ensure that the dataset is loaded strictly from `/kaggle/input/`, following the exact folder structure described in the **Data Folder Description**, and do not attempt to load data from the current directory (`./`).
+    1. Ensure that the dataset is loaded strictly from `{% include "scenarios.data_science.share:scen.input_path" %}`, following the exact folder structure described in the **Data Folder Description**, and do not attempt to load data from the current directory (`./`).
     2. You should avoid using logging module to output information in your generated code, and instead use the print() function.
+    3. You should be very careful about the try catch block in your code. You may use it to handle missing files in data reading, but you should not use it to handle the errors in your code. Especially use it to bypass the errors in your code. Directly solve the errors in your code instead of using try catch block to bypass them.
     
     ## Exploratory Data Analysis (EDA) part(Required):
     - Before returning the data, you should always add an EDA part describing the data to help the following steps understand the data better.
@@ -57,6 +58,37 @@ pipeline_coder:
       User will use the following code to match: re.search(r"(.*?)=== Start of EDA part ===(.*)=== End of EDA part ===", stdout, re.DOTALL).groups()[1]
     - An evaluation agent will help to check whether the EDA part is added correctly.
     - During the EDA part, you should try to avoid any irrelevant information sending to the standard output.
+    {% include "scenarios.data_science.share:guidelines.coding" %}
+
+    {% if enable_model_dump %}
+    ## Model Dumping
+    {% include "components.coder.data_science.share.prompts:dump_model_coder.guideline" %}
+    {% endif %}
+
+    {% if enable_debug_mode %}
+    Your code will be executed in a debug mode with following command: 
+    ```bash
+    python main.py --debug
+    ```
+    In debug mode, you should only sample ten percent of the data and run the minimum epochs to quickly test the correctness of the code.
+    In debug mode, you should implement a timer to measure the time taken for your debug configuration and estimate the time required for the full run.
+    For example, you can sample ten percent of the data and run for one epoch, then the full run with ten epochs will take one hundred times the time taken for the debug run. The scale is calculated by yourself depending on the data sampling and epoch number you choose. If your full run enables early stopping, the scale should be smaller considering the early stopping will stop the training earlier than the full epochs.
+    Your debug code should run exactly the same as the full run, except for the data sampling and epoch number, to ensure the correctness of the code.
+    You should print total time and estimated time in standard output using print function in the following schema:
+    === Start of Debug Information ===
+    debug_time: time_taken_for_debug_run_in_seconds (e.g., 'debug_time: 10.0')
+    estimated_time: estimated_time_for_full_run_in_seconds (e.g., 'estimated_time: 100.0')
+    === End of Debug Information ===
+    User will use the following code to match: re.search(r"(.*?)=== Start of Debug Information ===(.*)=== End of Debug Information ===", stdout, re.DOTALL).groups()[1]
+    Notice, data sampling should only be applied in debug mode. Always use the full data in the full run!
+    Example code:
+    ```python
+    if args.debug:
+      sample_size = int(0.1 * len(train_dataset))  # 10% for debug
+    else:
+      sample_size = len(train_dataset)
+    ```
+    {% endif %}
 
     ## Output Format
     {% if out_spec %}
@@ -72,7 +104,7 @@ pipeline_coder:
     --------- Competition Information ---------
     {{ competition_info }}
 
-    --------- Data Folder Description (All path are relative to the data folder) ---------
+    --------- Data Folder Description (All path are relative to the data folder, i.e. "{% include "scenarios.data_science.share:scen.input_path" %}") ---------
     {{ folder_spec }}
     
     {% if latest_code %}
@@ -82,6 +114,7 @@ pipeline_coder:
     --------- Feedback to former code ---------
     {{ latest_code_feedback }}
     The former code contains errors. You should correct the code based on the provided information, ensuring you do not repeat the same mistakes.
+    Keep the part that already seem correct intact. Avoid modifying them to refrain from introducing new errors.
     {% else %}
     The former code is correct. You should try to improve the code based on the provided task while not changing the irrelevant parts.
     {% endif %}
@@ -89,7 +122,6 @@ pipeline_coder:
 
     You should strictly follow the code specifications provided by the specification to implement the function.
 
-
 pipeline_eval:
   system: |-
     You are a data scientist responsible for evaluating code generation.
@@ -103,11 +135,32 @@ pipeline_eval:
 
     The details on how to structure the code are given in the specification:
     {{ spec }}
-
+    
+    {% if is_sub_enabled %}
     ## Evaluation Scope
     Your focus is to check whether the workflow code:
-    1. Executes successfully, correctly generating a final submission.
-    2. Generates predictions in the correct format, ensuring they align with the submission structure!
+    Step 1: Executes successfully without any errors. Please distinguish between the errors and warnings.
+
+    Step 2: Correctly generates a final submission in the correct format, ensuring: they align with the submission structure, the index names and column names should match the sample, and the items should not be empty or apparently incorrect. The final submission and validation metrics should not be generated with mock data or dummy values. They should only the results of the actual code execution. Once you found the coding containing any data mock or dummy values, you should emphasize it in your feedback and reject this implementation.
+
+    Step 3: No data sampling is applied in the full run, and the code runs on the full dataset. The code should not contain any data sampling or data reduction logic in the full run. Once you found any data sampling or reduction logic in the full run, you should emphasize it in your feedback and reject this implementation.
+
+    Step 4: Aligns with the competition requirements. This includes:
+    - CAREFULLY ANALYZE WHETHER THE EXPERIMENTAL SETUP AND CODE MAY CAUSE MISALIGNMENT BETWEEN VALIDATION AND TEST PERFORMANCE.
+    - Confirm strict adherence to the competition's evaluation rules listed in `scenario`:
+      - Exact match between the implementation code of metric and the requirements of the scenario. The metric number is not the focus.
+      - Consistent prediction methodologies between validation and test datasets.
+      - No shortcuts or fold-specific strategies applied inconsistently.
+      - Rigorous checks for corner-case consistency.
+    - If such discrepancies or risks are found:
+      - Clearly document these issues in `code`.
+      - Begin your `code` with `[Evaluation error]`, explicitly stating the evaluation alignment issues causing experiment failure.
+    - If no issues are found, begin your `code` with `[Code analysis]`, providing a detailed analysis of the code quality, readability, and adherence to specifications.
+
+    {% if debug_mode %}
+    Step 5: The code is executed in a debug mode with the command `python main.py --debug`. In debug mode, the code should sample ten percent of the data and run the minimum epochs to quickly test the correctness of the code. You should check whether the code follows this requirements. If not, you should emphasize it in your feedback and reject this implementation.
+    You will also be given the execution time and estimated time for the full run. If not time information is provided or the estimated full run time is more than three times the timeout period, you should reject the implementation because the full run will not finish in time. 
+    {% endif %}
 
     ## Evaluation Criteria
     You will be given the execution output (`stdout`) to determine correctness.  
@@ -121,14 +174,34 @@ pipeline_eval:
     {
         "execution": "Describe whether the code executed successfully, correctly integrating all components and generating the final submission. Include any errors or issues encountered, and append all error messages and full traceback details without summarizing or omitting any information.",
         "return_checking": "Verify the generated files, particularly the submission file. Ensure that its format matches the sample submission, checking the index, column names, and CSV content.",
+        "code": "Begin explicitly with [Code analysis] or [Evaluation error]. Provide feedback on code quality, readability, adherence to the given specifications, and alignment with competition requirements.",
+        "final_decision": <true/false>
+    }
+    ```
+    {% else %}
+    ## Evaluation Scope
+    Your focus is to check whether the workflow code executes successfully.
+
+    You will be given the execution output (`stdout`) to determine correctness.  
+
+    [Note] 
+    1. Model performance is NOT a concern in this evaluation—only correct execution and formatting matter.
+
+    Please respond with your feedback in the following JSON format and order
+    ```json
+    {
+        "execution": "Describe whether the code executed successfully. Include any errors or issues encountered, and append all error messages and full traceback details without summarizing or omitting any information.",
+        "return_checking": "Describe the expected file to be generated.",
         "code": "Provide feedback on code quality, readability, and adherence to the given specifications.",
         "final_decision": <true/false>
     }
     ```
-  
+    {% endif %}
+# NOTE: when is_sub_enabled == False, we don't have any checking about the return. So it is just placeholder currently
+
   user: |-
     --------- code generated by user ---------
     {{ code }}
 
     --------- code running stdout ---------
-    {{ stdout }}
+    {{ stdout }}

rdagent/components/coder/data_science/raw_data_loader/__init__.py

@@ -67,7 +67,7 @@ class DataLoaderMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
     ) -> dict[str, str]:
         # return a workspace with "load_data.py", "spec/load_data.md" inside
         # assign the implemented code to the new workspace.
-        competition_info = self.scen.get_scenario_all_desc()
+        competition_info = self.scen.get_scenario_all_desc(eda_output=workspace.file_dict.get("EDA.md", None))
         runtime_environment = self.scen.get_runtime_environment()
         data_folder_info = self.scen.processed_data_folder_description
         data_loader_task_info = target_task.get_task_information()
@@ -225,11 +225,21 @@ class DataLoaderCoSTEER(CoSTEER):
     def develop(self, exp):
         new_exp = super().develop(exp)
 
-        env = get_ds_env()
-        env.conf.extra_volumes = {f"{DS_RD_SETTING.local_data_path}/{self.scen.competition}": "/kaggle/input"}
+        env = get_ds_env(
+            extra_volumes={
+                f"{DS_RD_SETTING.local_data_path}/{self.scen.competition}": T(
+                    "scenarios.data_science.share:scen.input_path"
+                ).r()
+            },
+            running_timeout_period=DS_RD_SETTING.full_timeout,
+        )
 
         stdout = new_exp.experiment_workspace.execute(env=env, entry=f"python test/data_loader_test.py")
         match = re.search(r"(.*?)=== Start of EDA part ===(.*)=== End of EDA part ===", stdout, re.DOTALL)
         eda_output = match.groups()[1] if match else None
-        self.scen.eda_output = eda_output
+        if eda_output is not None:
+            new_exp.experiment_workspace.inject_files(**{"EDA.md": eda_output})
+        else:
+            eda_output = "No EDA output."
+            new_exp.experiment_workspace.inject_files(**{"EDA.md": eda_output})
         return new_exp

rdagent/components/coder/data_science/raw_data_loader/eval.py

@@ -13,6 +13,7 @@ from rdagent.components.coder.CoSTEER.knowledge_management import (
     CoSTEERQueriedKnowledgeV2,
 )
 from rdagent.components.coder.data_science.conf import get_ds_env
+from rdagent.components.coder.data_science.utils import remove_eda_part
 from rdagent.core.experiment import FBWorkspace, Task
 from rdagent.utils.agent.tpl import T
 from rdagent.utils.agent.workflow import build_cls_from_json_with_retry
@@ -23,7 +24,6 @@ DataLoaderEvalFeedback = CoSTEERSingleFeedback
 
 
 class DataLoaderCoSTEEREvaluator(CoSTEEREvaluator):
-
     def evaluate(
         self,
         target_task: Task,
@@ -32,7 +32,6 @@ class DataLoaderCoSTEEREvaluator(CoSTEEREvaluator):
         queried_knowledge: CoSTEERQueriedKnowledgeV2 = None,
         **kwargs,
     ) -> DataLoaderEvalFeedback:
-
         target_task_information = target_task.get_task_information()
         if (
             queried_knowledge is not None
@@ -47,14 +46,15 @@ class DataLoaderCoSTEEREvaluator(CoSTEEREvaluator):
                 final_decision=False,
             )
 
-        env = get_ds_env()
-        env.conf.extra_volumes = {f"{DS_RD_SETTING.local_data_path}/sample/{self.scen.competition}": "/kaggle/input"}
+        env = get_ds_env(extra_volumes={self.scen.debug_path: T("scenarios.data_science.share:scen.input_path").r()})
 
         # TODO: do we need to clean the generated temporary content?
         fname = "test/data_loader_test.py"
         test_code = (DIRNAME / "eval_tests" / "data_loader_test.txt").read_text()
         implementation.inject_files(**{fname: test_code})
-        stdout, ret_code = implementation.execute_ret_code(env=env, entry=f"python {fname}")
+        result = implementation.run(env=env, entry=f"python {fname}")
+        stdout = result.stdout
+        ret_code = result.exit_code
         match = re.search(r"(.*?)=== Start of EDA part ===(.*)=== End of EDA part ===(.*)", stdout, re.DOTALL)
         stdout_part_1, eda_output, stdout_part_2 = match.groups() if match else (stdout, None, "")
         stdout = stdout_part_1 + stdout_part_2
@@ -63,7 +63,7 @@ class DataLoaderCoSTEEREvaluator(CoSTEEREvaluator):
 
         if "main.py" in implementation.file_dict and ret_code == 0:
             workflow_stdout = implementation.execute(env=env, entry="python main.py")
-            workflow_stdout = re.sub(r"=== Start of EDA part ===(.*)=== End of EDA part ===", "", workflow_stdout)
+            workflow_stdout = remove_eda_part(workflow_stdout)
         else:
             workflow_stdout = None
 
@@ -80,9 +80,12 @@ class DataLoaderCoSTEEREvaluator(CoSTEEREvaluator):
             workflow_stdout=workflow_stdout,
         )
 
-        return build_cls_from_json_with_retry(
+        fb = build_cls_from_json_with_retry(
             DataLoaderEvalFeedback,
             system_prompt=system_prompt,
             user_prompt=user_prompt,
             init_kwargs_update_func=DataLoaderEvalFeedback.val_and_update_init_dict,
         )
+        fb.final_decision = fb.final_decision and ret_code == 0
+
+        return fb

rdagent/components/coder/data_science/raw_data_loader/eval_tests/data_loader_test.txt

@@ -9,12 +9,22 @@ from load_data import load_data
 
 import sys
 import reprlib
+from joblib.memory import MemorizedFunc
+
+
+def get_original_code(func):
+    if isinstance(func, MemorizedFunc):
+        return func.func.__code__
+    return func.__code__
+
+
 def debug_info_print(func):
     aRepr = reprlib.Repr()
     aRepr.maxother=300
     def wrapper(*args, **kwargs):
+        original_code = get_original_code(func)
         def local_trace(frame, event, arg):
-            if event == "return" and frame.f_code == func.__code__:
+            if event == "return" and frame.f_code == original_code:
                 print("\n" + "="*20 + "Running data_load code, local variable values:" + "="*20)
                 for k, v in frame.f_locals.items():
                     printed = aRepr.repr(v)

rdagent/components/coder/data_science/raw_data_loader/prompts.yaml

@@ -48,7 +48,7 @@ spec:
           - `test_ids` (DT): Identifiers for the test data.
         - Docstring Requirements:
           - Describe the purpose of the function.
-          - Specify the data source location (`/kaggle/input/`).
+          - Specify the data source location (`{% include "scenarios.data_science.share:scen.input_path" %}`).
           - Clearly define the structure and type of the output.
           - Inferred data shape to each input and output data variables. To uncertain dimension, use -1.
       2. Notes:
@@ -224,71 +224,7 @@ spec:
       You should return the specification in markdown format directly, while the **function definition** within it should be in code format, tailored to the Competition Information, with detailed explanations provided in the docstring.
 
     workflow: |-
-      Your task is to implement the main workflow script (`main.py`) for a Kaggle-style machine learning competition project. 
-      Follow the provided project structure and specifications to ensure consistency and maintainability:
-      1. Workflow Integration:
-        - Integrate the following components into the workflow:
-          - Data loading (`load_data.py`).
-          - Feature engineering (`feature.py`).
-          - Model workflow for training and testing (`model_*.py`). 
-          - Ensemble workflow that combines results from the model workflow to obtain the final prediction (`ensemble.py`).
-        - Treat each component as a modular and callable Python function.
-        - The workflow script should be flexible enough to handle either a single model or multiple models, with filenames (model_*.py) that are not determined at the outset.
-          For multiple model selection, utilize Python code to identify eligible models based on filenames, for example:
-          ```python
-          available_models = [f for f in os.listdir('.') if f.startswith('model_') and 'test' not in f]
-          ```
-      2. Feature Engineering
-        - The feature engineering should be called only once. For example:
-          `X_transformed, y_transformed, X_test_transformed = feat_eng(X, y, X_test)`
-        - It should be called before dataset splitting.
-
-      3. Dataset Splitting
-        - The dataset returned by `load_data` is not pre-split. After calling `feat_eng`, split the data into training and test sets.
-        - [Notice] If feasible, apply cross-validation on the training set (`X_transformed`, `y_transformed`) to ensure a reliable assessment of model performance.
-        - Keep the test set (`X_test_transformed`) unchanged, as it is only used for generating the final predictions.
-        - Pseudocode logic for reference:
-          ```
-          Set number of splits and initialize KFold cross-validator.
-          Create dictionaries for validation and test predictions.
-          For each model file:
-              Import the model dynamically.
-              Initialize arrays for out-of-fold (OOF) and test predictions.
-              For each fold in KFold:
-                  Split data into training and validation sets.
-                  Run model workflow to get validation and test predictions.
-                  Validate shapes.
-                  Store validation and test predictions.
-              Compute average test predictions across folds.
-              Save OOF and averaged test predictions.
-          Ensemble predictions from all models and print the final shape.
-          ```
-
-      4. Submission File:
-        - Save the final predictions as `submission.csv`, ensuring the format matches the competition requirements (refer to `sample_submission` in the Folder Description for the correct structure).
-        - Present the required submission format explicitly and ensure the output adheres to it.
-
-      5. Code Standards:
-        - Do not use progress bars (e.g., tqdm) in the code.
-
-      6. Ensemble Strategy:
-        Consolidate all model outputs into a dictionary, where each key is the model's filename (excluding the .py extension) and its corresponding value is the model's output.
-        Sample code:
-        {% raw %}
-        {% for model_name in model_names %}
-        model_module = __import__(model_name.replace('.py', ''))
-        val_pred, test_pred, _ = model_module.model_workflow(
-          X=train_X,
-          y=train_y,
-          val_X=val_X,
-          val_y=val_y,
-          test_X=X_test_transformed
-        )
-        val_preds_dict[model_module.__name__] = val_pred
-        test_preds_dict[model_module.__name__] = test_pred
-        {% endfor %}
-        final_pred = ensemble_workflow(test_preds_dict, val_preds_dict, val_y)
-        {% endraw %}
+      {% include "scenarios.data_science.share:component_spec.Workflow" %}
 
       {% if latest_spec %}
       7. Former Specification:
@@ -313,8 +249,8 @@ data_loader_coder:
     {% endif %}
     
     {% if queried_similar_successful_knowledge|length != 0 %}
-    --------- Successful Implementations for Similar Models ---------
-    ====={% for similar_successful_knowledge in queried_similar_successful_knowledge %} Model {{ loop.index }}:=====
+    --------- Successful Implementation Examples for Similar Task ---------
+    ====={% for similar_successful_knowledge in queried_similar_successful_knowledge %} Example {{ loop.index }}:=====
     {{ similar_successful_knowledge.target_task.get_task_information() }}
     =====Code:=====
     {{ similar_successful_knowledge.implementation.all_codes }}
@@ -332,8 +268,14 @@ data_loader_coder:
     {% endif %}
 
     ## Guidelines
-    1. Ensure that the dataset is loaded strictly from `/kaggle/input/`, following the exact folder structure described in the **Data Folder Description**, and do not attempt to load data from the current directory (`./`).
+    1. Ensure that the dataset is loaded strictly from `{% include "scenarios.data_science.share:scen.input_path" %}`, following the exact folder structure described in the **Data Folder Description**, and do not attempt to load data from the current directory (`./`).
     2. You should avoid using logging module to output information in your generated code, and instead use the print() function.
+    3. You should use the following cache decorator to cache the results of the function:
+    ```python
+    from joblib import Memory
+    memory = Memory(location='{% include "scenarios.data_science.share:scen.cache_path" %}', verbose=0)
+    @memory.cache```
+    {% include "scenarios.data_science.share:guidelines.coding" %}
     
     ## Exploratory Data Analysis (EDA) part(Required):
     - Before returning the data, you should always add an EDA part describing the data to help the following steps understand the data better.
@@ -370,7 +312,7 @@ data_loader_coder:
     --------- Code Specification ---------
     {{ code_spec }}
 
-    --------- Data Folder Description (All path are relative to the data folder) ---------
+    --------- Data Folder Description (All path are relative to the data folder, i.e. "{% include "scenarios.data_science.share:scen.input_path" %}") ---------
     {{ folder_spec }}
     
     {% if latest_code %}

rdagent/components/coder/data_science/share/doc.py

@@ -0,0 +1,37 @@
+"""
+Developers concentrating on writing documents for a workspace
+"""
+
+from rdagent.core.developer import Developer
+from rdagent.core.experiment import Experiment, FBWorkspace
+from rdagent.oai.llm_utils import APIBackend
+from rdagent.utils.agent.ret import MarkdownAgentOut
+from rdagent.utils.agent.tpl import T
+
+
+class DocDev(Developer[Experiment]):
+    """
+    The developer is responsible for writing documents for a workspace.
+    """
+
+    def develop(self, exp: Experiment) -> None:
+        """
+        Write documents for the workspace.
+        """
+        ws: FBWorkspace = exp.experiment_workspace
+
+        file_li = [str(file.relative_to(ws.workspace_path)) for file in ws.workspace_path.rglob("*") if file.is_file()]
+
+        key_file_list = ["main.py", "scores.csv"]
+
+        system_prompt = T(".prompts:docdev.system").r()
+        user_prompt = T(".prompts:docdev.user").r(
+            file_li=file_li,
+            key_files={f: (ws.workspace_path / f).read_text() for f in key_file_list},
+        )
+
+        resp = APIBackend().build_messages_and_create_chat_completion(
+            user_prompt=user_prompt, system_prompt=system_prompt
+        )
+        markdown = MarkdownAgentOut.extract_output(resp)
+        ws.inject_files(**{"README.md": markdown})

rdagent/components/coder/data_science/share/eval.py

@@ -0,0 +1,147 @@
+from pathlib import Path
+from typing import Literal
+
+import pandas as pd
+
+from rdagent.app.data_science.conf import DS_RD_SETTING
+from rdagent.components.coder.CoSTEER import CoSTEERMultiFeedback
+from rdagent.components.coder.CoSTEER.evaluators import (
+    CoSTEEREvaluator,
+    CoSTEERSingleFeedback,
+)
+from rdagent.components.coder.data_science.conf import get_clear_ws_cmd, get_ds_env
+from rdagent.components.coder.data_science.utils import remove_eda_part
+from rdagent.core.experiment import FBWorkspace, Task
+from rdagent.core.scenario import Scenario
+from rdagent.utils.agent.tpl import T
+from rdagent.utils.agent.workflow import build_cls_from_json_with_retry
+
+DIRNAME = Path(__file__).absolute().resolve().parent
+
+PipelineSingleFeedback = CoSTEERSingleFeedback
+PipelineMultiFeedback = CoSTEERMultiFeedback
+
+NO_SUB = "<No submission.csv file found.>"
+NO_SCORE = "<No scores.csv file found.>"
+
+
+class ModelDumpEvaluator(CoSTEEREvaluator):
+    """This evaluator assumes that it runs after the model"""
+
+    def __init__(self, scen: Scenario, data_type: Literal["sample", "full"]):
+        super().__init__(scen)
+        self.data_type = data_type
+
+    def evaluate(
+        self, target_task: Task, implementation: FBWorkspace, gt_implementation: FBWorkspace, *kargs, **kwargs
+    ) -> CoSTEERSingleFeedback:
+
+        model_folder = implementation.workspace_path / "models"
+        # 1) Check if the model_folder is not empty
+        if not model_folder.exists() or not any(model_folder.iterdir()):
+            err_msg = "Model folder (`models` sub folder) is empty or does not exist. The model is not dumped."
+            return CoSTEERSingleFeedback(
+                execution=err_msg,
+                return_checking=err_msg,
+                code=err_msg,
+                final_decision=False,
+            )
+
+        data_source_path = (
+            f"{DS_RD_SETTING.local_data_path}/{self.scen.competition}"
+            if self.data_type == "full"
+            else self.scen.debug_path
+        )
+        env = get_ds_env(
+            extra_volumes={data_source_path: T("scenarios.data_science.share:scen.input_path").r()},
+            running_timeout_period=(
+                DS_RD_SETTING.full_timeout if self.data_type == "full" else DS_RD_SETTING.debug_timeout
+            ),
+        )
+
+        # 2) check the result and stdout after reruning the model.
+
+        # Read the content of files submission.csv and scores.csv before execution
+        submission_content_before = (
+            (implementation.workspace_path / "submission.csv").read_text()
+            if (implementation.workspace_path / "submission.csv").exists()
+            else NO_SUB
+        )
+        scores_content_before = (
+            (implementation.workspace_path / "scores.csv").read_text()
+            if (implementation.workspace_path / "scores.csv").exists()
+            else NO_SCORE
+        )
+
+        # Remove the files submission.csv and scores.csv
+        implementation.execute(env=env, entry=get_clear_ws_cmd(stage="before_inference"))
+
+        # Execute the main script
+        stdout = remove_eda_part(implementation.execute(env=env, entry="python main.py"))
+
+        # walk model_folder and list the files
+        model_folder_files = [
+            str(file.relative_to(implementation.workspace_path)) for file in model_folder.iterdir() if file.is_file()
+        ]
+
+        # this will assert the generation of necessary files
+        for f in ["submission.csv", "scores.csv"]:
+            if not (implementation.workspace_path / f).exists():
+                err_msg = f"{f} does not exist. The model is not dumped. Make sure that the required files, like submission.csv and scores.csv, are created even if you bypass the model training step by loading the saved model file directly."
+                return CoSTEERSingleFeedback(
+                    execution=err_msg,
+                    return_checking=err_msg,
+                    code=err_msg,
+                    final_decision=False,
+                )
+
+        # Check if scores contain NaN (values)
+        score_df = pd.read_csv((implementation.workspace_path / "scores.csv"), index_col=0)
+        if score_df.isnull().values.any():
+            nan_locations = score_df[score_df.isnull().any(axis=1)]
+            err_msg = f"\n[Error] The scores dataframe contains NaN values at the following locations:\n{nan_locations}"
+            return CoSTEERSingleFeedback(
+                execution=err_msg,
+                return_checking=err_msg,
+                code=err_msg,
+                final_decision=False,
+            )
+
+        submission_content_after = (
+            (implementation.workspace_path / "submission.csv").read_text()
+            if (implementation.workspace_path / "submission.csv").exists()
+            else NO_SUB
+        )
+        scores_content_after = (
+            (implementation.workspace_path / "scores.csv").read_text()
+            if (implementation.workspace_path / "scores.csv").exists()
+            else NO_SCORE
+        )
+
+        system_prompt = T(".prompts:dump_model_eval.system").r()
+        user_prompt = T(".prompts:dump_model_eval.user").r(
+            stdout=stdout.strip(),
+            code=implementation.all_codes,
+            model_folder_files=model_folder_files,
+            scores_content_before=scores_content_before,
+            scores_content_after=scores_content_after,
+        )
+
+        csfb = build_cls_from_json_with_retry(
+            CoSTEERSingleFeedback,
+            system_prompt=system_prompt,
+            user_prompt=user_prompt,
+        )
+
+        if DS_RD_SETTING.model_dump_check_level == "high":
+            # Read the content of files submission.csv and scores.csv after execution
+            # Check if the content has changed
+            # excactly same checking. But it will take more user's time
+            if scores_content_before != scores_content_after:
+                return_msg = "\n[Error] The content of scores.csv has changed. Please check the code to ensure that the model is dumped correctly, and rerun the code to use the model directly without retraining it."
+                return_msg += f"\nBefore:\n{scores_content_before}\nAfter:\n{scores_content_after}"
+                if submission_content_before != submission_content_after:
+                    # If the scores file changes, display the two contents and append it into the return_checking
+                    return_msg = "[Error] The content of submission.csv has changed. Please check the code to ensure that the model is dumped correctly, and rerun the code to use the model directly without retraining it."
+                csfb.return_checking = (csfb.return_checking or "") + return_msg
+        return csfb

rdagent/components/coder/data_science/share/prompts.yaml

@@ -0,0 +1,91 @@
+dump_model_coder:
+  guideline: |-
+    Please dump the model in a "models/" subfolder in the first running, and the script rerun performs inference without needing to retrain the model when running the code again.
+    If there are parameters generated from the training data that might be needed for inference on test data, please save them in the "models/" subfolder as well.
+    If no test set is provided, reserve a portion of the data as your test set and save the generated test files in the models/ subfolder for use in submission and inference.
+    Make sure that the required files, like submission.csv and scores.csv, are created without model training step through loading the saved model and test data file directly.
+
+dump_model_eval:
+  system: |-
+    You are a data scientist tasked with evaluating code generation. You've developed a Kaggle competition code that can produce a submission file.
+    The code should follow the guideline below:
+    {% include "components.coder.data_science.share.prompts:dump_model_coder.guideline" %}
+
+    You will receive the following information:
+    - The implemented code
+    - The stdout from running the code
+    - The file list in "models/" subfolder
+    - The scores.csv file generated during both training and inference (if it exists)
+
+    Focus on these aspects:
+    - Check if the code saves the model in the "models/" subfolder.
+    - Check if the code saves the test data in the "models/" subfolder when there is no test data specified.
+    - Ensure that when the code is rerun, it skips the training process and loads the model from the "models/" subfolder for direct inference.
+      - Verify that there is no training activity in the output.
+    - Ensure that even if you skip the model training by loading saved models, the files like scores.csv and submission.csv are still correctly created.
+    - The model's performance should remain consistent and not vary unreasonably between training and inference.
+
+    Please respond with your feedback in the following JSON format and order
+    ```json
+    {
+        "execution": "Describe whether the code executed successfully. Include any errors or issues encountered, and append all error messages and full traceback details without summarizing or omitting any information. Carefully check the stdout to ensure that when the code is rerun, it skips the training process and loads the model from the 'models/' subfolder for direct inference. Append the information that makes you think that the model is still being retrained when rerunning the code."
+        "return_checking": "Verify the generated files include necessary files. Make sure scores.csv file does not change unreasonably between training and inference",
+        "code": "The code has explicity dump the model into 'models/' subfolder; When the modes files are already in 'models/' subfolder, the code will explicity skip the training process.",
+        "final_decision": <true or false in boolean type; only return true when ensuring that the code saves the model in a 'models/' subfolder, and the script rerun performs inference without needing to retrain the model.>
+    }
+    ```
+
+  user: |-
+    ------------ The implemented code ------------ 
+    {{code}}
+
+    ------------ The stdout from running the code ------------ 
+    {{stdout}}
+
+    ------------ The file list in "models/" subfolder ------------
+    {% for f in model_folder_files %}
+    - {{ f }}
+    {% endfor %}
+
+    ------------ The scores.csv file generated ------------
+    # Training:
+    {{scores_content_before}}
+
+    # Inference:
+    {{scores_content_after}}
+
+
+docdev:
+  system: |-
+    {% include "scenarios.data_science.share:scen.role" %}  Your task is to create documentation for a data science solution.
+
+    You will be given:
+    - a list of files in the folder.
+    - content from some important files.
+
+    Please explain the trained models in the "models/" folder. The training and inference processes are detailed in the `main.py` file. The models' evaluation results are in `scores.csv`. Please respond with a markdown file that includes the following information:
+    - Explain the purpose of each model. If some models are part of a group (like those from cross-validation), describe them together.
+    - Provide key details for each model group:
+      - Important training parameters
+      - Model details
+      - Performance of each model
+
+    Be brief. Mention the file path when you introduce files.
+    Don't introduce anything other than models.
+
+    {% include "utils.agent.tpl:MarkdownOut" %}
+
+  user: |-
+    --------------- The file list in the workspace ---------------
+    {% for f in file_li %}
+    - {{ f }}
+    {% endfor %}
+
+    --------------- File content of each file ---------------
+    {% for fname, content in key_files.items() %}
+    File Path: {{fname}}
+    ```
+    {{content}}
+    ```
+    {% endfor %}
+

rdagent/components/coder/data_science/utils.py

@@ -0,0 +1,6 @@
+import re
+
+
+def remove_eda_part(stdout: str) -> str:
+    """Data Science scenario have a LLM-based EDA feature. We can remove it when current task does not involve EDA"""
+    return re.sub(r"=== Start of EDA part ===(.*)=== End of EDA part ===", "", stdout, flags=re.DOTALL)

rdagent/components/coder/data_science/workflow/__init__.py

@@ -59,7 +59,7 @@ class WorkflowMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
         # 2. code
         system_prompt = T(".prompts:workflow_coder.system").r(
             task_desc=workflow_information_str,
-            competition_info=self.scen.get_scenario_all_desc(),
+            competition_info=self.scen.get_scenario_all_desc(eda_output=workspace.file_dict.get("EDA.md", None)),
             queried_similar_successful_knowledge=queried_similar_successful_knowledge,
             queried_former_failed_knowledge=queried_former_failed_knowledge[0],
             out_spec=PythonAgentOut.get_spec(),

rdagent/components/coder/data_science/workflow/eval.py

@@ -10,9 +10,11 @@ from rdagent.components.coder.CoSTEER.evaluators import (
     CoSTEERMultiFeedback,
     CoSTEERSingleFeedback,
 )
-from rdagent.components.coder.data_science.conf import get_ds_env
+from rdagent.components.coder.data_science.conf import get_clear_ws_cmd, get_ds_env
+from rdagent.components.coder.data_science.utils import remove_eda_part
 from rdagent.core.evolving_framework import QueriedKnowledge
 from rdagent.core.experiment import FBWorkspace, Task
+from rdagent.log import rdagent_logger as logger
 from rdagent.utils.agent.tpl import T
 from rdagent.utils.agent.workflow import build_cls_from_json_with_retry
 
@@ -53,8 +55,7 @@ class WorkflowGeneralCaseSpecEvaluator(CoSTEEREvaluator):
                 final_decision=False,
             )
 
-        env = get_ds_env()
-        env.conf.extra_volumes = {f"{DS_RD_SETTING.local_data_path}/sample/{self.scen.competition}": "/kaggle/input"}
+        env = get_ds_env(extra_volumes={self.scen.debug_path: T("scenarios.data_science.share:scen.input_path").r()})
 
         # # DockerEnv for MLEBench submission validation
         # mle_de_conf = MLEBDockerConf()
@@ -65,12 +66,12 @@ class WorkflowGeneralCaseSpecEvaluator(CoSTEEREvaluator):
         # mde.prepare()
 
         # Clean the scores.csv & submission.csv.
-        implementation.execute(env=env, entry=f"rm submission.csv scores.csv")
+        implementation.execute(env=env, entry=get_clear_ws_cmd())
 
-        stdout = implementation.execute(env=env, entry=f"python main.py")
+        stdout = implementation.execute(env=env, entry=f"python -m coverage run main.py")
 
         # remove EDA part
-        stdout = re.sub(r"=== Start of EDA part ===(.*)=== End of EDA part ===", "", stdout)
+        stdout = remove_eda_part(stdout)
 
         # Check score file
         score_fp = implementation.workspace_path / "scores.csv"
@@ -79,6 +80,14 @@ class WorkflowGeneralCaseSpecEvaluator(CoSTEEREvaluator):
         if not score_fp.exists():
             score_check_text = "[Error] Metrics file (scores.csv) is not generated!"
             score_ret_code = 1
+            implementation.execute(env=env, entry="python -m coverage json -o coverage.json")
+            coverage_report_path = implementation.workspace_path / "coverage.json"
+            if coverage_report_path.exists():
+                used_files = set(json.loads(coverage_report_path.read_text())["files"].keys())
+                coverage_report_path.unlink()
+                logger.info(f"All used scripts: {used_files}")
+                if len(used_files) == 1:
+                    score_check_text += f"\n[Error] The only used script is {used_files}.\nPlease check if you have implemented entry point in 'main.py'."
         else:
             try:
                 score_df = pd.read_csv(score_fp, index_col=0)
@@ -98,21 +107,28 @@ class WorkflowGeneralCaseSpecEvaluator(CoSTEEREvaluator):
                     score_check_text += f"\n[Error] The scores dataframe does not contain the correct column names.\nCorrect columns is: ['{self.scen.metric_name}']\nBut got: {score_df.columns.tolist()}"
                     score_ret_code = 1
 
+                # Check if scores contain NaN (values)
+                if score_df.isnull().values.any():
+                    nan_locations = score_df[score_df.isnull().any(axis=1)]
+                    score_check_text += f"\n[Error] The scores dataframe contains NaN values at the following locations:\n{nan_locations}"
+                    score_ret_code = 1
+
             except Exception as e:
                 score_check_text += f"\n[Error] in checking the scores.csv file: {e}\nscores.csv's content:\n-----\n{score_fp.read_text()}\n-----"
                 score_ret_code = 1
 
         # Check submission file
-        base_check_code = (DIRNAME / "eval_tests" / "submission_format_test.txt").read_text()
+        base_check_code = T(".eval_tests.submission_format_test", ftype="txt").r()
         implementation.inject_files(**{"test/submission_format_test.py": base_check_code})
         # stdout += "----Submission Check 1-----\n"
-        submission_check_out, submission_ret_code = implementation.execute_ret_code(
-            env=env, entry="python test/submission_format_test.py"
-        )
+        submission_result = implementation.run(env=env, entry="python test/submission_format_test.py")
+        submission_check_out = submission_result.stdout
+        submission_ret_code = submission_result.exit_code
         stdout += "\n" + submission_check_out
 
         system_prompt = T(".prompts:workflow_eval.system").r(
-            scenario=self.scen.get_scenario_all_desc(),
+            # here we pass `None` to `eda_output` because we do not have nor need EDA output for workflow.
+            scenario=self.scen.get_scenario_all_desc(eda_output=None),
             task_desc=target_task.get_task_information(),
             spec=(
                 implementation.file_dict["spec/workflow.md"]

rdagent/components/coder/data_science/workflow/eval_tests/submission_format_test.txt

@@ -22,12 +22,12 @@ find . | grep -i sample | grep -i submission | grep -v sample_submission.csv | g
 """
 
 # Find sample submission file dynamically
-input_dir = Path("/kaggle/input")
+input_dir = Path("{% include "scenarios.data_science.share:scen.input_path" %}")
 # Look for common variations of sample submission filenames
 sample_submission_files = list(input_dir.glob("*sample_submission*.csv")) + \
                          list(input_dir.glob("*sampleSubmission*.csv"))
 
-assert sample_submission_files, "Error: No sample submission file found in /kaggle/input/"
+assert sample_submission_files, "Error: No sample submission file found in {% include "scenarios.data_science.share:scen.input_path" %}"
 
 # Use first matching file
 sample_submission_name = sample_submission_files[0].name

rdagent/components/coder/data_science/workflow/exp.py

@@ -10,4 +10,5 @@ from rdagent.core.utils import cache_with_pickle
 
 # Because we use isinstance to distinguish between different types of tasks, we need to use sub classes to represent different types of tasks
 class WorkflowTask(CoSTEERTask):
-    pass
+    def __init__(self, name: str = "Workflow", *args, **kwargs) -> None:
+        super().__init__(name=name, *args, **kwargs)

rdagent/components/coder/data_science/workflow/prompts.yaml

@@ -40,6 +40,7 @@ workflow_coder:
     3. The user may provide specific code organization rules and instructions. Ensure that the integration follows the given framework and structure.
     4. After predicting the output, print the shape and other information of the output to stdout to help the evaluator assess the code.
     5. You should avoid using logging module to output information in your generated code, and instead use the print() function.
+    {% include "scenarios.data_science.share:guidelines.coding" %}
 
     ## Output Format
     {% if out_spec %}
@@ -133,4 +134,4 @@ workflow_eval:
     --------- Workflow test stdout ---------
     {{ stdout }}
     --------- Workflow code generated by user ---------
-    {{ code }}
+    {{ code }}

rdagent/components/coder/factor_coder/__init__.py

@@ -26,6 +26,7 @@ class FactorCoSTEER(CoSTEER):
         try:
             exp = super().develop(exp)
         finally:
-            es = self.evolve_agent.evolving_trace[-1]
-            exp.prop_dev_feedback = es.feedback
+            if hasattr(self, "evolve_agent") and self.evolve_agent.evolving_trace:
+                es = self.evolve_agent.evolving_trace[-1]
+                exp.prop_dev_feedback = es.feedback
         return exp

rdagent/components/coder/factor_coder/config.py

@@ -15,7 +15,7 @@ class FactorCoSTEERSettings(CoSTEERSettings):
     simple_background: bool = False
     """Whether to use simple background information for code feedback"""
 
-    file_based_execution_timeout: int = 120
+    file_based_execution_timeout: int = 3600
     """Timeout in seconds for each factor implementation execution"""
 
     select_method: str = "random"

rdagent/components/coder/factor_coder/eva_utils.py

@@ -1,20 +1,16 @@
 import io
 import json
 from abc import abstractmethod
-from pathlib import Path
 from typing import Dict, Tuple
 
 import pandas as pd
-from jinja2 import Environment, StrictUndefined
 
 from rdagent.components.coder.factor_coder.config import FACTOR_COSTEER_SETTINGS
 from rdagent.components.coder.factor_coder.factor import FactorTask
 from rdagent.core.experiment import Task, Workspace
-from rdagent.core.prompts import Prompts
 from rdagent.oai.llm_conf import LLM_SETTINGS
 from rdagent.oai.llm_utils import APIBackend
-
-evaluate_prompts = Prompts(file_path=Path(__file__).parent / "prompts.yaml")
+from rdagent.utils.agent.tpl import T
 
 
 class FactorEvaluator:
@@ -81,36 +77,26 @@ class FactorCodeEvaluator(FactorEvaluator):
         factor_information = target_task.get_task_information()
         code = implementation.all_codes
 
-        system_prompt = (
-            Environment(undefined=StrictUndefined)
-            .from_string(evaluate_prompts["evaluator_code_feedback_v1_system"])
-            .render(
-                scenario=(
-                    self.scen.get_scenario_all_desc(
-                        target_task,
-                        filtered_tag="feature",
-                        simple_background=FACTOR_COSTEER_SETTINGS.simple_background,
-                    )
-                    if self.scen is not None
-                    else "No scenario description."
+        system_prompt = T(".prompts:evaluator_code_feedback_v1_system").r(
+            scenario=(
+                self.scen.get_scenario_all_desc(
+                    target_task,
+                    filtered_tag="feature",
+                    simple_background=FACTOR_COSTEER_SETTINGS.simple_background,
                 )
+                if self.scen is not None
+                else "No scenario description."
             )
         )
 
         execution_feedback_to_render = execution_feedback
         for _ in range(10):  # 10 times to split the content is enough
-            user_prompt = (
-                Environment(undefined=StrictUndefined)
-                .from_string(
-                    evaluate_prompts["evaluator_code_feedback_v1_user"],
-                )
-                .render(
-                    factor_information=factor_information,
-                    code=code,
-                    execution_feedback=execution_feedback_to_render,
-                    value_feedback=value_feedback,
-                    gt_code=gt_implementation.code if gt_implementation else None,
-                )
+            user_prompt = T(".prompts:evaluator_code_feedback_v1_user").r(
+                factor_information=factor_information,
+                code=code,
+                execution_feedback=execution_feedback_to_render,
+                value_feedback=value_feedback,
+                gt_code=gt_implementation.code if gt_implementation else None,
             )
             if (
                 APIBackend().build_messages_and_calculate_token(
@@ -189,17 +175,11 @@ class FactorOutputFormatEvaluator(FactorEvaluator):
         buffer = io.StringIO()
         gen_df.info(buf=buffer)
         gen_df_info_str = f"The user is currently working on a feature related task.\nThe output dataframe info is:\n{buffer.getvalue()}"
-        system_prompt = (
-            Environment(undefined=StrictUndefined)
-            .from_string(
-                evaluate_prompts["evaluator_output_format_system"],
-            )
-            .render(
-                scenario=(
-                    self.scen.get_scenario_all_desc(implementation.target_task, filtered_tag="feature")
-                    if self.scen is not None
-                    else "No scenario description."
-                )
+        system_prompt = T(".prompts:evaluator_output_format_system").r(
+            scenario=(
+                self.scen.get_scenario_all_desc(implementation.target_task, filtered_tag="feature")
+                if self.scen is not None
+                else "No scenario description."
             )
         )
 
@@ -504,35 +484,25 @@ class FactorFinalDecisionEvaluator(FactorEvaluator):
         code_feedback: str,
         **kwargs,
     ) -> Tuple:
-        system_prompt = (
-            Environment(undefined=StrictUndefined)
-            .from_string(evaluate_prompts["evaluator_final_decision_v1_system"])
-            .render(
-                scenario=(
-                    self.scen.get_scenario_all_desc(target_task, filtered_tag="feature")
-                    if self.scen is not None
-                    else "No scenario description."
-                )
+        system_prompt = T(".prompts:evaluator_final_decision_v1_system").r(
+            scenario=(
+                self.scen.get_scenario_all_desc(target_task, filtered_tag="feature")
+                if self.scen is not None
+                else "No scenario description."
             )
         )
         execution_feedback_to_render = execution_feedback
 
         for _ in range(10):  # 10 times to split the content is enough
-            user_prompt = (
-                Environment(undefined=StrictUndefined)
-                .from_string(
-                    evaluate_prompts["evaluator_final_decision_v1_user"],
-                )
-                .render(
-                    factor_information=target_task.get_task_information(),
-                    execution_feedback=execution_feedback_to_render,
-                    code_feedback=code_feedback,
-                    value_feedback=(
-                        value_feedback
-                        if value_feedback is not None
-                        else "No Ground Truth Value provided, so no evaluation on value is performed."
-                    ),
-                )
+            user_prompt = T(".prompts:evaluator_final_decision_v1_user").r(
+                factor_information=target_task.get_task_information(),
+                execution_feedback=execution_feedback_to_render,
+                code_feedback=code_feedback,
+                value_feedback=(
+                    value_feedback
+                    if value_feedback is not None
+                    else "No Ground Truth Value provided, so no evaluation on value is performed."
+                ),
             )
             if (
                 APIBackend().build_messages_and_calculate_token(

rdagent/components/coder/factor_coder/evolving_strategy.py

@@ -1,11 +1,9 @@
 from __future__ import annotations
 
 import json
-from pathlib import Path
+import re
 from typing import Dict
 
-from jinja2 import Environment, StrictUndefined
-
 from rdagent.components.coder.CoSTEER.evaluators import CoSTEERSingleFeedback
 from rdagent.components.coder.CoSTEER.evolving_strategy import (
     MultiProcessEvolvingStrategy,
@@ -17,11 +15,9 @@ from rdagent.components.coder.CoSTEER.knowledge_management import (
 from rdagent.components.coder.factor_coder.config import FACTOR_COSTEER_SETTINGS
 from rdagent.components.coder.factor_coder.factor import FactorFBWorkspace, FactorTask
 from rdagent.core.experiment import FBWorkspace
-from rdagent.core.prompts import Prompts
 from rdagent.oai.llm_conf import LLM_SETTINGS
 from rdagent.oai.llm_utils import APIBackend
-
-implement_prompts = Prompts(file_path=Path(__file__).parent / "prompts.yaml")
+from rdagent.utils.agent.tpl import T
 
 
 class FactorMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
@@ -36,24 +32,14 @@ class FactorMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
         queried_former_failed_knowledge_to_render: list,
         queried_similar_error_knowledge_to_render: list,
     ) -> str:
-        error_summary_system_prompt = (
-            Environment(undefined=StrictUndefined)
-            .from_string(implement_prompts["evolving_strategy_error_summary_v2_system"])
-            .render(
-                scenario=self.scen.get_scenario_all_desc(target_task),
-                factor_information_str=target_task.get_task_information(),
-                code_and_feedback=queried_former_failed_knowledge_to_render[-1].get_implementation_and_feedback_str(),
-            )
-            .strip("\n")
+        error_summary_system_prompt = T(".prompts:evolving_strategy_error_summary_v2_system").r(
+            scenario=self.scen.get_scenario_all_desc(target_task),
+            factor_information_str=target_task.get_task_information(),
+            code_and_feedback=queried_former_failed_knowledge_to_render[-1].get_implementation_and_feedback_str(),
         )
         for _ in range(10):  # max attempt to reduce the length of error_summary_user_prompt
-            error_summary_user_prompt = (
-                Environment(undefined=StrictUndefined)
-                .from_string(implement_prompts["evolving_strategy_error_summary_v2_user"])
-                .render(
-                    queried_similar_error_knowledge=queried_similar_error_knowledge_to_render,
-                )
-                .strip("\n")
+            error_summary_user_prompt = T(".prompts:evolving_strategy_error_summary_v2_user").r(
+                queried_similar_error_knowledge=queried_similar_error_knowledge_to_render,
             )
             if (
                 APIBackend().build_messages_and_calculate_token(
@@ -106,16 +92,9 @@ class FactorMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
         latest_attempt_to_latest_successful_execution = queried_knowledge.task_to_former_failed_traces[
             target_factor_task_information
         ][1]
-
-        system_prompt = (
-            Environment(undefined=StrictUndefined)
-            .from_string(
-                implement_prompts["evolving_strategy_factor_implementation_v1_system"],
-            )
-            .render(
-                scenario=self.scen.get_scenario_all_desc(target_task, filtered_tag="feature"),
-                queried_former_failed_knowledge=queried_former_failed_knowledge_to_render,
-            )
+        system_prompt = T(".prompts:evolving_strategy_factor_implementation_v1_system").r(
+            scenario=self.scen.get_scenario_all_desc(target_task, filtered_tag="feature"),
+            queried_former_failed_knowledge=queried_former_failed_knowledge_to_render,
         )
         queried_similar_successful_knowledge_to_render = queried_similar_successful_knowledge
         queried_similar_error_knowledge_to_render = queried_similar_error_knowledge
@@ -136,19 +115,12 @@ class FactorMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
             else:
                 error_summary_critics = None
             # 构建user_prompt。开始写代码
-            user_prompt = (
-                Environment(undefined=StrictUndefined)
-                .from_string(
-                    implement_prompts["evolving_strategy_factor_implementation_v2_user"],
-                )
-                .render(
-                    factor_information_str=target_factor_task_information,
-                    queried_similar_successful_knowledge=queried_similar_successful_knowledge_to_render,
-                    queried_similar_error_knowledge=queried_similar_error_knowledge_to_render,
-                    error_summary_critics=error_summary_critics,
-                    latest_attempt_to_latest_successful_execution=latest_attempt_to_latest_successful_execution,
-                )
-                .strip("\n")
+            user_prompt = T(".prompts:evolving_strategy_factor_implementation_v2_user").r(
+                factor_information_str=target_factor_task_information,
+                queried_similar_successful_knowledge=queried_similar_successful_knowledge_to_render,
+                queried_similar_error_knowledge=queried_similar_error_knowledge_to_render,
+                error_summary_critics=error_summary_critics,
+                latest_attempt_to_latest_successful_execution=latest_attempt_to_latest_successful_execution,
             )
             if (
                 APIBackend().build_messages_and_calculate_token(user_prompt=user_prompt, system_prompt=system_prompt)
@@ -165,18 +137,28 @@ class FactorMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
                 queried_similar_error_knowledge_to_render = queried_similar_error_knowledge_to_render[:-1]
         for _ in range(10):
             try:
-                code = json.loads(
-                    APIBackend(
-                        use_chat_cache=FACTOR_COSTEER_SETTINGS.coder_use_cache
-                    ).build_messages_and_create_chat_completion(
-                        user_prompt=user_prompt,
-                        system_prompt=system_prompt,
-                        json_mode=True,
-                        json_target_type=Dict[str, str],
-                    )
-                )["code"]
+                response = APIBackend(
+                    use_chat_cache=FACTOR_COSTEER_SETTINGS.coder_use_cache
+                ).build_messages_and_create_chat_completion(
+                    user_prompt=user_prompt,
+                    system_prompt=system_prompt,
+                    json_mode=True,
+                    json_target_type=Dict[str, str],
+                )
+
+                try:
+                    code = json.loads(response)["code"]
+                except json.decoder.JSONDecodeError:
+                    # extract python code block
+                    match = re.search(r"```python(.*?)```", response, re.DOTALL)
+                    if match:
+                        code = match.group(1).strip()
+                    else:
+                        raise  # continue to retry
+
                 return code
-            except json.decoder.JSONDecodeError:
+
+            except (json.decoder.JSONDecodeError, KeyError):
                 pass
         else:
             return ""  # return empty code if failed to get code after 10 attempts

rdagent/components/coder/factor_coder/factor.py

@@ -49,6 +49,12 @@ class FactorTask(CoSTEERTask):
         return f"""factor_name: {self.factor_name}
 factor_description: {self.factor_description}
 factor_formulation: {self.factor_formulation}
+variables: {str(self.variables)}"""
+
+    def get_task_brief_information(self):
+        return f"""factor_name: {self.factor_name}
+factor_description: {self.factor_description}
+factor_formulation: {self.factor_formulation}
 variables: {str(self.variables)}"""
 
     def get_task_information_and_implementation_result(self):

rdagent/components/coder/factor_coder/prompts.yaml

@@ -49,6 +49,8 @@ evolving_strategy_factor_implementation_v1_system: |-
   
   Your must write your code based on your former latest attempt below which consists of your former code and code feedback, you should read the former attempt carefully and must not modify the right part of your former code.
 
+  Notice that you should not add any other text before or after the json format.
+
   {% if queried_former_failed_knowledge|length != 0 %}
   --------------Your former latest attempt:---------------
   =====Code to the former implementation=====
@@ -116,6 +118,9 @@ evolving_strategy_error_summary_v2_system: |-
 
   You suggestion should not include any code, just some clear and short suggestions. Please point out very critical issues in your response, ignore non-important issues to avoid confusion. If no big issue found in the code, you can response "No critics found".
 
+  [NOTE]
+  1. When processing data, avoid time leakage.
+
   Please response the critic in the following format. Here is an example structure for the output:
   critic 1: The critic message to critic 1
   critic 2: The critic message to critic 2
@@ -185,7 +190,7 @@ evaluator_final_decision_v1_system: |-
   The implementation final decision is considered in the following logic:
   1. If the value and the ground truth value are exactly the same under a small tolerance, the implementation is considered correct.
   2. If the value and the ground truth value have a high correlation on ic or rank ic, the implementation is considered correct.
-  3. If no ground truth value is provided, the implementation is considered correct if the code executes successfully (assuming the data provided is correct). Any exceptions, including those actively raised, are considered faults of the code. Additionally, the code feedback must align with the scenario and factor description.
+  3. If no ground truth value is provided, the implementation is considered correct if the code executes successfully (assuming the data provided is correct). Any exceptions, including those actively raised, are considered faults of the code. Additionally, the code feedback must align with the scenario and factor description. The implementation cannot be considered correct if the code execution failed, no matter what the reason is.
 
   Please response the critic in the json format. Here is an example structure for the JSON output, please strictly follow the format:
   {

rdagent/components/coder/model_coder/conf.py

@@ -0,0 +1,13 @@
+from pydantic_settings import SettingsConfigDict
+
+from rdagent.components.coder.CoSTEER.config import CoSTEERSettings
+
+
+class ModelCoSTEERSettings(CoSTEERSettings):
+    model_config = SettingsConfigDict(env_prefix="MODEL_CoSTEER_")
+
+    env_type: str = "conda"  # or "docker"
+    """Environment to run model code in coder and runner: 'conda' for local conda env, 'docker' for Docker container"""
+
+
+MODEL_COSTEER_SETTINGS = ModelCoSTEERSettings()

rdagent/components/coder/model_coder/eva_utils.py

@@ -1,18 +1,14 @@
 import json
-from pathlib import Path
 from typing import Dict, Tuple
 
 import numpy as np
-from jinja2 import Environment, StrictUndefined
 
 from rdagent.components.coder.CoSTEER.evaluators import CoSTEEREvaluator
 from rdagent.components.coder.model_coder.model import ModelFBWorkspace, ModelTask
 from rdagent.core.experiment import Task, Workspace
-from rdagent.core.prompts import Prompts
 from rdagent.oai.llm_conf import LLM_SETTINGS
 from rdagent.oai.llm_utils import APIBackend
-
-evaluate_prompts = Prompts(file_path=Path(__file__).parent / "prompts.yaml")
+from rdagent.utils.agent.tpl import T
 
 
 # This shape evaluator is also used in data_science
@@ -70,32 +66,21 @@ class ModelCodeEvaluator(CoSTEEREvaluator):
         model_task_information = target_task.get_task_information()
         code = implementation.all_codes
 
-        system_prompt = (
-            Environment(undefined=StrictUndefined)
-            .from_string(evaluate_prompts["evaluator_code_feedback"]["system"])
-            .render(
-                scenario=(
-                    self.scen.get_scenario_all_desc(target_task, filtered_tag=target_task.model_type)
-                    if self.scen is not None
-                    else "No scenario description."
-                )
+        system_prompt = T(".prompts:evaluator_code_feedback.system").r(
+            scenario=(
+                self.scen.get_scenario_all_desc(target_task, filtered_tag=target_task.model_type)
+                if self.scen is not None
+                else "No scenario description."
             )
         )
-
         execution_feedback_to_render = model_execution_feedback
         for _ in range(10):  # 10 times to split the content is enough
-            user_prompt = (
-                Environment(undefined=StrictUndefined)
-                .from_string(
-                    evaluate_prompts["evaluator_code_feedback"]["user"],
-                )
-                .render(
-                    model_information=model_task_information,
-                    code=code,
-                    model_execution_feedback=execution_feedback_to_render,
-                    model_value_feedback=model_value_feedback,
-                    gt_code=gt_implementation.all_codes if gt_implementation else None,
-                )
+            user_prompt = T(".prompts:evaluator_code_feedback.user").r(
+                model_information=model_task_information,
+                code=code,
+                model_execution_feedback=execution_feedback_to_render,
+                model_value_feedback=model_value_feedback,
+                gt_code=gt_implementation.all_codes if gt_implementation else None,
             )
             if (
                 APIBackend().build_messages_and_calculate_token(
@@ -133,34 +118,25 @@ class ModelFinalEvaluator(CoSTEEREvaluator):
         if gt_implementation is not None:
             assert isinstance(gt_implementation, ModelFBWorkspace)
 
-        system_prompt = (
-            Environment(undefined=StrictUndefined)
-            .from_string(evaluate_prompts["evaluator_final_feedback"]["system"])
-            .render(
-                scenario=(
-                    self.scen.get_scenario_all_desc(target_task, filtered_tag=target_task.model_type)
-                    if self.scen is not None
-                    else "No scenario description."
-                )
+        system_prompt = T(".prompts:evaluator_final_feedback.system").r(
+            scenario=(
+                self.scen.get_scenario_all_desc(target_task, filtered_tag=target_task.model_type)
+                if self.scen is not None
+                else "No scenario description."
             )
         )
 
         execution_feedback_to_render = model_execution_feedback
 
         for _ in range(10):  # 10 times to split the content is enough
-            user_prompt = (
-                Environment(undefined=StrictUndefined)
-                .from_string(
-                    evaluate_prompts["evaluator_final_feedback"]["user"],
-                )
-                .render(
-                    model_information=target_task.get_task_information(),
-                    model_execution_feedback=execution_feedback_to_render,
-                    model_shape_feedback=model_shape_feedback,
-                    model_code_feedback=model_code_feedback,
-                    model_value_feedback=model_value_feedback,
-                )
+            user_prompt = T(".prompts:evaluator_final_feedback.user").r(
+                model_information=target_task.get_task_information(),
+                model_execution_feedback=execution_feedback_to_render,
+                model_shape_feedback=model_shape_feedback,
+                model_code_feedback=model_code_feedback,
+                model_value_feedback=model_value_feedback,
             )
+
             if (
                 APIBackend().build_messages_and_calculate_token(
                     user_prompt=user_prompt,

rdagent/components/coder/model_coder/evolving_strategy.py

@@ -1,9 +1,6 @@
 import json
-from pathlib import Path
 from typing import Dict
 
-from jinja2 import Environment, StrictUndefined
-
 from rdagent.components.coder.CoSTEER.config import CoSTEER_SETTINGS
 from rdagent.components.coder.CoSTEER.evaluators import CoSTEERSingleFeedback
 from rdagent.components.coder.CoSTEER.evolving_strategy import (
@@ -13,17 +10,11 @@ from rdagent.components.coder.CoSTEER.knowledge_management import (
     CoSTEERQueriedKnowledge,
     CoSTEERQueriedKnowledgeV2,
 )
-from rdagent.components.coder.model_coder.model import (
-    ModelExperiment,
-    ModelFBWorkspace,
-    ModelTask,
-)
+from rdagent.components.coder.model_coder.model import ModelFBWorkspace, ModelTask
 from rdagent.core.experiment import FBWorkspace
-from rdagent.core.prompts import Prompts
 from rdagent.oai.llm_conf import LLM_SETTINGS
 from rdagent.oai.llm_utils import APIBackend
-
-coder_prompts = Prompts(file_path=Path(__file__).parent / "prompts.yaml")
+from rdagent.utils.agent.tpl import T
 
 
 class ModelMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
@@ -52,32 +43,18 @@ class ModelMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
             if isinstance(queried_knowledge, CoSTEERQueriedKnowledgeV2)
             else queried_former_failed_knowledge
         )
-
-        system_prompt = (
-            Environment(undefined=StrictUndefined)
-            .from_string(
-                coder_prompts["evolving_strategy_model_coder"]["system"],
-            )
-            .render(
-                scenario=self.scen.get_scenario_all_desc(filtered_tag=target_task.model_type),
-                queried_former_failed_knowledge=queried_former_failed_knowledge_to_render,
-                current_code=target_task.base_code,
-            )
+        system_prompt = T(".prompts:evolving_strategy_model_coder.system").r(
+            scenario=self.scen.get_scenario_all_desc(filtered_tag="model"),
+            queried_former_failed_knowledge=queried_former_failed_knowledge_to_render,
+            current_code=workspace.file_dict.get("model.py"),
         )
 
         queried_similar_successful_knowledge_to_render = queried_similar_successful_knowledge
         for _ in range(10):  # max attempt to reduce the length of user_prompt
-            user_prompt = (
-                Environment(undefined=StrictUndefined)
-                .from_string(
-                    coder_prompts["evolving_strategy_model_coder"]["user"],
-                )
-                .render(
-                    model_information_str=model_information_str,
-                    queried_similar_successful_knowledge=queried_similar_successful_knowledge_to_render,
-                    queried_former_failed_knowledge=queried_former_failed_knowledge_to_render,
-                )
-                .strip("\n")
+            user_prompt = T(".prompts:evolving_strategy_model_coder.user").r(
+                model_information_str=model_information_str,
+                queried_similar_successful_knowledge=queried_similar_successful_knowledge_to_render,
+                queried_former_failed_knowledge=queried_former_failed_knowledge_to_render,
             )
             if (
                 APIBackend().build_messages_and_calculate_token(

rdagent/components/coder/model_coder/model.py

@@ -5,10 +5,11 @@ from pathlib import Path
 from typing import Dict, Optional
 
 from rdagent.components.coder.CoSTEER.task import CoSTEERTask
+from rdagent.components.coder.model_coder.conf import MODEL_COSTEER_SETTINGS
 from rdagent.core.experiment import Experiment, FBWorkspace
 from rdagent.core.utils import cache_with_pickle
 from rdagent.oai.llm_utils import md5_hash
-from rdagent.utils.env import KGDockerEnv, QTDockerEnv
+from rdagent.utils.env import KGDockerEnv, QlibCondaConf, QlibCondaEnv, QTDockerEnv
 
 
 class ModelTask(CoSTEERTask):
@@ -19,6 +20,7 @@ class ModelTask(CoSTEERTask):
         architecture: str,
         *args,
         hyperparameters: Dict[str, str],
+        training_hyperparameters: Dict[str, str],
         formulation: str = None,
         variables: Dict[str, str] = None,
         model_type: Optional[str] = None,
@@ -28,6 +30,7 @@ class ModelTask(CoSTEERTask):
         self.architecture: str = architecture
         self.variables: str = variables
         self.hyperparameters: str = hyperparameters
+        self.training_hyperparameters: str = training_hyperparameters
         self.model_type: str = (
             model_type  # Tabular for tabular model, TimesSeries for time series model, Graph for graph model, XGBoost for XGBoost model
         )
@@ -41,6 +44,17 @@ description: {self.description}
         task_desc += f"architecture: {self.architecture}\n"
         task_desc += f"variables: {self.variables}\n" if self.variables else ""
         task_desc += f"hyperparameters: {self.hyperparameters}\n"
+        task_desc += f"training_hyperparameters: {self.training_hyperparameters}\n"
+        task_desc += f"model_type: {self.model_type}\n"
+        return task_desc
+
+    def get_task_brief_information(self):
+        task_desc = f"""name: {self.name}
+description: {self.description}
+"""
+        task_desc += f"architecture: {self.architecture}\n"
+        task_desc += f"hyperparameters: {self.hyperparameters}\n"
+        task_desc += f"training_hyperparameters: {self.training_hyperparameters}\n"
         task_desc += f"model_type: {self.model_type}\n"
         return task_desc
 
@@ -99,7 +113,15 @@ class ModelFBWorkspace(FBWorkspace):
     ):
         self.before_execute()
         try:
-            qtde = QTDockerEnv() if self.target_task.version == 1 else KGDockerEnv()
+            if self.target_task.version == 1:
+                if MODEL_COSTEER_SETTINGS.env_type == "docker":
+                    qtde = QTDockerEnv()
+                elif MODEL_COSTEER_SETTINGS.env_type == "conda":
+                    qtde = QlibCondaEnv(conf=QlibCondaConf())
+                else:
+                    raise ValueError(f"Unknown env_type: {MODEL_COSTEER_SETTINGS.env_type}")
+            else:
+                qtde = KGDockerEnv()
             qtde.prepare()
 
             if self.target_task.version == 1:

rdagent/components/coder/model_coder/model_execute_template_v1.txt

@@ -16,8 +16,8 @@ if MODEL_TYPE == "Tabular":
     m = model_cls(num_features=input_shape[1])
     data = torch.full(input_shape, INPUT_VALUE)
 elif MODEL_TYPE == "TimeSeries":
-    input_shape = (BATCH_SIZE, NUM_FEATURES, NUM_TIMESTEPS)
-    m = model_cls(num_features=input_shape[1], num_timesteps=input_shape[2])
+    input_shape = (BATCH_SIZE, NUM_TIMESTEPS, NUM_FEATURES)
+    m = model_cls(num_features=input_shape[2], num_timesteps=input_shape[1])
     data = torch.full(input_shape, INPUT_VALUE)
 elif MODEL_TYPE == "Graph":
     node_feature = torch.randn(BATCH_SIZE, NUM_FEATURES)

rdagent/components/coder/model_coder/one_shot/__init__.py

@@ -1,12 +1,10 @@
 import re
 from pathlib import Path
 
-from jinja2 import Environment, StrictUndefined
-
 from rdagent.components.coder.model_coder.model import ModelExperiment, ModelFBWorkspace
 from rdagent.core.developer import Developer
-from rdagent.core.prompts import Prompts
 from rdagent.oai.llm_utils import APIBackend
+from rdagent.utils.agent.tpl import T
 
 DIRNAME = Path(__file__).absolute().resolve().parent
 
@@ -17,18 +15,14 @@ class ModelCodeWriter(Developer[ModelExperiment]):
         for t in exp.sub_tasks:
             mti = ModelFBWorkspace(t)
             mti.prepare()
-            pr = Prompts(file_path=DIRNAME / "prompt.yaml")
 
-            user_prompt_tpl = Environment(undefined=StrictUndefined).from_string(pr["code_implement_user"])
-            sys_prompt_tpl = Environment(undefined=StrictUndefined).from_string(pr["code_implement_sys"])
-
-            user_prompt = user_prompt_tpl.render(
+            user_prompt = T(".prompts:code_implement_user").r(
                 name=t.name,
                 description=t.description,
                 formulation=t.formulation,
                 variables=t.variables,
             )
-            system_prompt = sys_prompt_tpl.render()
+            system_prompt = T(".prompts:code_implement_sys").r()
 
             resp = APIBackend().build_messages_and_create_chat_completion(user_prompt, system_prompt)
 

rdagent/components/coder/model_coder/prompts.yaml

@@ -1,41 +1,29 @@
 extract_model_formulation_system: |-
     offer description of the proposed model in this paper, write a latex formula with variable as well as the architecture of the model. the format should be like 
     {
-    "model_name (The name of the model)": {
-        "description": "A detailed description of the model",
-        "formulation": "A LaTeX formula representing the model's formulation",
-        "architecture": "A detailed description of the model's architecture, e.g., neural network layers or tree structures",
-        "variables": {
-            "\\hat{y}_u": "The predicted output for node u",
-            "variable_name_2": "Description of variable 2",
-            "variable_name_3": "Description of variable 3"
-        },
-        "hyperparameters": {
-            "hyperparameter_name_1": "value of hyperparameter 1",
-            "hyperparameter_name_2": "value of hyperparameter 2",
-            "hyperparameter_name_3": "value of hyperparameter 3"
-        },
-        "model_type": "Tabular or TimeSeries or Graph or XGBoost"  # Should be one of "Tabular", "TimeSeries", "Graph", or "XGBoost"
-    }
-    }
-    Eg. 
-    {
-    "ABC Model": {
-        "description": "A detailed description of the model",
-        "formulation": "A LaTeX formula representing the model's formulation",
-        "architecture": "A detailed description of the model's architecture, e.g., neural network layers or tree structures",
-        "variables": {
-            "\\hat{y}_u": "The predicted output for node u",
-            "variable_name_2": "Description of variable 2",
-            "variable_name_3": "Description of variable 3"
-        },
-        "hyperparameters": {
-            "hyperparameter_name_1": "value of hyperparameter 1",
-            "hyperparameter_name_2": "value of hyperparameter 2",
-            "hyperparameter_name_3": "value of hyperparameter 3"
-        },
-        "model_type": "Tabular or TimeSeries or Graph or RandomForest or XGBoost"  # If torch & Neural network models are required, the choice should be one of "Tabular", "TimeSeries", or "Graph" 
-    }
+        "model_name (The name of the model)": {
+            "description": "A detailed description of the model",
+            "formulation": "A LaTeX formula representing the model's formulation",
+            "architecture": "A detailed description of the model's architecture, e.g., neural network layers or tree structures",
+            "variables": {
+                "\\hat{y}_u": "The predicted output for node u",
+                "variable_name_2": "Description of variable 2",
+                "variable_name_3": "Description of variable 3"
+            },
+            "hyperparameters": {
+                "hyperparameter_name_1": "value of hyperparameter 1",
+                "hyperparameter_name_2": "value of hyperparameter 2",
+                "hyperparameter_name_3": "value of hyperparameter 3"
+            },
+            "training_hyperparameters" {  # All values are for reference; you can set them yourself
+                "n_epochs": "100",
+                "lr": "1e-3",
+                "early_stop": 10,
+                "batch_size": 256,
+                "weight_decay": 1e-4,
+            }
+            "model_type": "Tabular or TimeSeries or Graph or XGBoost"  # Should be one of "Tabular", "TimeSeries", "Graph", or "XGBoost"
+        }
     }
     such format content should be begin with ```json and end with ``` and the content should be in json format.
 

rdagent/components/coder/model_coder/task_loader.py

@@ -2,19 +2,19 @@ from __future__ import annotations
 
 import json
 import re
-from pathlib import Path
 
-from rdagent.components.coder.model_coder.model import ModelExperiment, ModelTask
+from pydantic import BaseModel, Field
+
+from rdagent.components.coder.model_coder.model import ModelTask
 from rdagent.components.document_reader.document_reader import (
     load_and_process_pdfs_by_langchain,
 )
 from rdagent.components.loader.task_loader import ModelTaskLoader
-from rdagent.core.prompts import Prompts
 from rdagent.log import rdagent_logger as logger
 from rdagent.oai.llm_utils import APIBackend
 from rdagent.scenarios.qlib.experiment.model_experiment import QlibModelExperiment
-
-document_process_prompts = Prompts(file_path=Path(__file__).parent / "prompts.yaml")
+from rdagent.utils.agent.tpl import T
+from rdagent.utils.workflow import wait_retry
 
 
 def extract_model_from_doc(doc_content: str) -> dict:
@@ -32,7 +32,7 @@ def extract_model_from_doc(doc_content: str) -> dict:
         {model_name: dict{description, formulation, variables}}
     """
     session = APIBackend().build_chat_session(
-        session_system_prompt=document_process_prompts["extract_model_formulation_system"],
+        session_system_prompt=T(".prompts:extract_model_formulation_system").r(),
     )
     current_user_prompt = doc_content
 
@@ -97,7 +97,7 @@ def extract_model_from_docs(docs_dict):
 
 
 class ModelExperimentLoaderFromDict(ModelTaskLoader):
-    def load(self, model_dict: dict) -> list:
+    def load(self, model_dict: dict) -> QlibModelExperiment:
         """Load data from a dict."""
         task_l = []
         for model_name, model_data in model_dict.items():
@@ -108,6 +108,7 @@ class ModelExperimentLoaderFromDict(ModelTaskLoader):
                 architecture=model_data["architecture"],
                 variables=model_data["variables"],
                 hyperparameters=model_data["hyperparameters"],
+                training_hyperparameters=model_data["training_hyperparameters"],
                 model_type=model_data["model_type"],
             )
             task_l.append(task)
@@ -115,7 +116,8 @@ class ModelExperimentLoaderFromDict(ModelTaskLoader):
 
 
 class ModelExperimentLoaderFromPDFfiles(ModelTaskLoader):
-    def load(self, file_or_folder_path: str) -> dict:
+    @wait_retry(retry_n=5)
+    def load(self, file_or_folder_path: str) -> QlibModelExperiment:
         docs_dict = load_and_process_pdfs_by_langchain(file_or_folder_path)  # dict{file_path:content}
         model_dict = extract_model_from_docs(
             docs_dict

rdagent/components/knowledge_management/graph.py

@@ -13,15 +13,17 @@ from rdagent.components.knowledge_management.vector_base import (
     cosine,
 )
 from rdagent.core.knowledge_base import KnowledgeBase
+from rdagent.log import rdagent_logger as logger
 from rdagent.oai.llm_utils import APIBackend
 
 Node = KnowledgeMetaData
 
 
 class UndirectedNode(Node):
-    def __init__(self, content: str = "", label: str = "", embedding: Any = None) -> None:
+    def __init__(self, content: str = "", label: str = "", embedding: Any = None, appendix: Any = None) -> None:
         super().__init__(content, label, embedding)
         self.neighbors: set[UndirectedNode] = set()
+        self.appendix = appendix  # appendix stores any additional information
         assert isinstance(content, str), "content must be a string"
 
     def add_neighbor(self, node: UndirectedNode) -> None:
@@ -86,6 +88,10 @@ class Graph(KnowledgeBase):
         size = 16
         embeddings = []
         for i in range(0, len(contents), size):
+            logger.info(
+                f"Creating embedding for index {i} to {i + size} with {len(contents)} contents",
+                tag="batch embedding",
+            )
             embeddings.extend(
                 APIBackend().create_embedding(input_content=contents[i : i + size]),
             )
@@ -270,7 +276,7 @@ class UndirectedGraph(Graph):
         self,
         node: UndirectedNode | str,
         similarity_threshold: float = 0.0,
-        topk_k: int = 5,
+        topk_k: int = None,
         constraint_labels: list[str] | None = None,
     ) -> list[UndirectedNode]:
         """

rdagent/components/knowledge_management/vector_base.py

@@ -87,7 +87,7 @@ class VectorBase(KnowledgeBase):
         """
         pass
 
-    def search(self, content: str, topk_k: int = 5, similarity_threshold: float = 0) -> List[Document]:
+    def search(self, content: str, topk_k: int | None = None, similarity_threshold: float = 0) -> List[Document]:
         """
         search vector_df by node
         Parameters
@@ -156,7 +156,11 @@ class PDVectorBase(VectorBase):
                 self.add(document=doc)
 
     def search(
-        self, content: str, topk_k: int = 5, similarity_threshold: float = 0, constraint_labels: list[str] | None = None
+        self,
+        content: str,
+        topk_k: int | None = None,
+        similarity_threshold: float = 0,
+        constraint_labels: list[str] | None = None,
     ) -> Tuple[List[Document], List]:
         """
         Search vector by node's embedding.
@@ -192,7 +196,9 @@ class PDVectorBase(VectorBase):
             lambda x: 1 - cosine(x, document.embedding)
         )  # cosine is cosine distance, 1-similarity
 
-        searched_similarities = similarities[similarities > similarity_threshold].nlargest(topk_k)
+        searched_similarities = similarities[similarities > similarity_threshold]
+        if topk_k is not None:
+            searched_similarities = searched_similarities.nlargest(topk_k)
         most_similar_docs = filtered_df.loc[searched_similarities.index]
 
         docs = []

rdagent/components/proposal/__init__.py

@@ -1,11 +1,7 @@
 from abc import abstractmethod
-from pathlib import Path
 from typing import Tuple
 
-from jinja2 import Environment, StrictUndefined
-
 from rdagent.core.experiment import Experiment
-from rdagent.core.prompts import Prompts
 from rdagent.core.proposal import (
     Hypothesis,
     Hypothesis2Experiment,
@@ -14,8 +10,8 @@ from rdagent.core.proposal import (
     Trace,
 )
 from rdagent.oai.llm_utils import APIBackend
-
-prompt_dict = Prompts(file_path=Path(__file__).parent / "prompts.yaml")
+from rdagent.utils.agent.tpl import T
+from rdagent.utils.workflow import wait_retry
 
 
 class LLMHypothesisGen(HypothesisGen):
@@ -32,27 +28,31 @@ class LLMHypothesisGen(HypothesisGen):
     def gen(self, trace: Trace) -> Hypothesis:
         context_dict, json_flag = self.prepare_context(trace)
 
-        system_prompt = (
-            Environment(undefined=StrictUndefined)
-            .from_string(prompt_dict["hypothesis_gen"]["system_prompt"])
-            .render(
-                targets=self.targets,
-                scenario=self.scen.get_scenario_all_desc(filtered_tag="hypothesis_and_experiment"),
-                hypothesis_output_format=context_dict["hypothesis_output_format"],
-                hypothesis_specification=context_dict["hypothesis_specification"],
-            )
+        system_prompt = T(".prompts:hypothesis_gen.system_prompt").r(
+            targets=self.targets,
+            scenario=(
+                self.scen.get_scenario_all_desc(filtered_tag=self.targets)
+                if self.targets in ["factor", "model"]
+                else self.scen.get_scenario_all_desc(filtered_tag="hypothesis_and_experiment")
+            ),
+            hypothesis_output_format=context_dict["hypothesis_output_format"],
+            hypothesis_specification=context_dict["hypothesis_specification"],
         )
-        user_prompt = (
-            Environment(undefined=StrictUndefined)
-            .from_string(prompt_dict["hypothesis_gen"]["user_prompt"])
-            .render(
-                targets=self.targets,
-                hypothesis_and_feedback=context_dict["hypothesis_and_feedback"],
-                RAG=context_dict["RAG"],
-            )
+        user_prompt = T(".prompts:hypothesis_gen.user_prompt").r(
+            targets=self.targets,
+            hypothesis_and_feedback=context_dict["hypothesis_and_feedback"],
+            last_hypothesis_and_feedback=(
+                context_dict["last_hypothesis_and_feedback"] if "last_hypothesis_and_feedback" in context_dict else ""
+            ),
+            sota_hypothesis_and_feedback=(
+                context_dict["sota_hypothesis_and_feedback"] if "sota_hypothesis_and_feedback" in context_dict else ""
+            ),
+            RAG=context_dict["RAG"],
         )
 
-        resp = APIBackend().build_messages_and_create_chat_completion(user_prompt, system_prompt, json_mode=json_flag)
+        resp = APIBackend().build_messages_and_create_chat_completion(
+            user_prompt, system_prompt, json_mode=json_flag, json_target_type=dict[str, str]
+        )
 
         hypothesis = self.convert_response(resp)
 
@@ -84,30 +84,33 @@ class LLMHypothesis2Experiment(Hypothesis2Experiment[Experiment]):
     @abstractmethod
     def convert_response(self, response: str, hypothesis: Hypothesis, trace: Trace) -> Experiment: ...
 
+    @wait_retry(retry_n=5)
     def convert(self, hypothesis: Hypothesis, trace: Trace) -> Experiment:
         context, json_flag = self.prepare_context(hypothesis, trace)
-        system_prompt = (
-            Environment(undefined=StrictUndefined)
-            .from_string(prompt_dict["hypothesis2experiment"]["system_prompt"])
-            .render(
-                targets=self.targets,
-                scenario=trace.scen.get_scenario_all_desc(filtered_tag="hypothesis_and_experiment"),
-                experiment_output_format=context["experiment_output_format"],
-            )
+        system_prompt = T(".prompts:hypothesis2experiment.system_prompt").r(
+            targets=self.targets,
+            scenario=trace.scen.get_scenario_all_desc(filtered_tag=self.targets),
+            experiment_output_format=context["experiment_output_format"],
         )
-        user_prompt = (
-            Environment(undefined=StrictUndefined)
-            .from_string(prompt_dict["hypothesis2experiment"]["user_prompt"])
-            .render(
-                targets=self.targets,
-                target_hypothesis=context["target_hypothesis"],
-                hypothesis_and_feedback=context["hypothesis_and_feedback"],
-                target_list=context["target_list"],
-                RAG=context["RAG"],
-            )
+        user_prompt = T(".prompts:hypothesis2experiment.user_prompt").r(
+            targets=self.targets,
+            target_hypothesis=context["target_hypothesis"],
+            hypothesis_and_feedback=(
+                context["hypothesis_and_feedback"] if "hypothesis_and_feedback" in context else ""
+            ),
+            last_hypothesis_and_feedback=(
+                context["last_hypothesis_and_feedback"] if "last_hypothesis_and_feedback" in context else ""
+            ),
+            sota_hypothesis_and_feedback=(
+                context["sota_hypothesis_and_feedback"] if "sota_hypothesis_and_feedback" in context else ""
+            ),
+            target_list=context["target_list"],
+            RAG=context["RAG"],
         )
 
-        resp = APIBackend().build_messages_and_create_chat_completion(user_prompt, system_prompt, json_mode=json_flag)
+        resp = APIBackend().build_messages_and_create_chat_completion(
+            user_prompt, system_prompt, json_mode=json_flag, json_target_type=dict[str, dict[str, str | dict]]
+        )
 
         return self.convert_response(resp, hypothesis, trace)
 

rdagent/components/proposal/prompts.yaml

@@ -1,48 +1,64 @@
 hypothesis_gen:
   system_prompt: |-
-    The user is working on generating new hypotheses for the {{targets}} in a data-driven research and development process. 
-    The {{targets}} are used in the following scenario:
-    {{scenario}}
-    The user has already proposed several hypotheses and conducted evaluations on them. This information will be provided to you. Your task is to check whether a similar hypothesis has already been generated. 
+    The user is working on generating new hypotheses for the {{ targets }} in a data-driven research and development process. 
+    The {{ targets }} are used in the following scenario:
+    {{ scenario }}
+    The user has already proposed several hypotheses and conducted evaluations on them. This information will be provided to you. Your task is to analyze previous experiments, reflect on the decision made in each experiment, and consider why experiments with a decision of true were successful while those with a decision of false failed. Then, think about how to improve further — either by refining the existing approach or by exploring an entirely new direction.
     If one exists and you agree with it, feel free to use it. If you disagree, please generate an improved version.
     {% if hypothesis_specification %}
-    To assist you in formulating new hypotheses, the user has provided some additional information: {{hypothesis_specification}}.
+    To assist you in formulating new hypotheses, the user has provided some additional information: {{ hypothesis_specification }}.
     **Important:** If the hypothesis_specification outlines the next steps you need to follow, ensure you adhere to those instructions.
     {% endif %}
     Please generate the output using the following format and specifications:
     {{ hypothesis_output_format }}
 
   user_prompt: |-
-    {% if hypothesis_and_feedback|length == 0 %}It is the first round of hypothesis generation. The user has no hypothesis on this scenario yet.
-    {% else %}It is not the first round, the user has made several hypothesis on this scenario and did several evaluation on them.
-    The former hypothesis and the corresponding feedbacks are as follows (focus on the last one & the new hypothesis that it provides and reasoning to see if you agree):
+    {% if hypothesis_and_feedback|length == 0 %}
+    It is the first round of hypothesis generation. The user has no hypothesis on this scenario yet.
+    {% else %}
+    The former hypothesis and the corresponding feedbacks are as follows:
     {{ hypothesis_and_feedback }}
     {% endif %}
-    {% if RAG %}
-    To assist you in generating new {{targets}}, we have provided the following information: {{RAG}}.
-    **Note:** The provided RAG is for reference only. 
-    You must carefully assess whether the RAG aligns with the {{targets}}. 
-    If it does not, it should not be used. Exercise caution and make your own judgment.
+    {% if last_hypothesis_and_feedback %}
+    Here is the last trial's hypothesis and the corresponding feedback (The main feedback contains a new hypothesis for your reference only. You need to evaluate the complete trace chain to decide whether to adopt it or propose a more appropriate hypothesis):
+    {{ last_hypothesis_and_feedback }}
+    {% endif %}
+    {% if sota_hypothesis_and_feedback != "" %}
+    Here is the SOTA trail's hypothesis and the corresponding feedback:
+    {{ sota_hypothesis_and_feedback }}
+    {% endif %}
+    {% if RAG %}
+    To assist you in generating new {{ targets }}, we have provided the following information: {{ RAG }}.
     {% endif %}
-    Also generate the relevant keys for the reasoning and the distilled knowledge that follows. For those keys, in particular for knowledge, explain in the context of the specific scenario to build up domain knowledge in the specific field rather than general knowledge.
 
 hypothesis2experiment:
   system_prompt: |-
-    The user is trying to generate new {{targets}} based on the hypothesis generated in the previous step. 
-    The {{targets}} are used in certain scenario, the scenario is as follows:
+    The user is trying to generate new {{ targets }} based on the hypothesis generated in the previous step. 
+    The {{ targets }} are used in certain scenario, the scenario is as follows:
     {{ scenario }}
-    The user will use the {{targets}} generated to do some experiments. The user will provide this information to you:
-    1. The target hypothesis you are targeting to generate {{targets}} for.
+    The user will use the {{ targets }} generated to do some experiments. The user will provide this information to you:
+    1. The target hypothesis you are targeting to generate {{ targets }} for.
     2. The hypothesis generated in the previous steps and their corresponding feedbacks.
-    3. Former proposed {{targets}} on similar hypothesis.
-    4. Some additional information to help you generate new {{targets}}.
+    3. Former proposed {{ targets }} on similar hypothesis.
+    4. Some additional information to help you generate new {{ targets }}.
     Please generate the output following the format below:
     {{ experiment_output_format }}
     
   user_prompt: |-
     The user has made several hypothesis on this scenario and did several evaluation on them.
-    The target hypothesis you are targeting to generate {{targets}} for is as follows:
+    The target hypothesis you are targeting to generate {{ targets }} for is as follows:
     {{ target_hypothesis }}
+    {% if hypothesis_and_feedback %}
     The former hypothesis and the corresponding feedbacks are as follows:
     {{ hypothesis_and_feedback }}
-    Please generate the new {{targets}} based on the information above.
+    {% endif %}
+    {% if last_hypothesis_and_feedback %}
+    The latest hypothesis and the corresponding feedback are as follows:
+    {{ last_hypothesis_and_feedback }}
+    {% endif %}
+    {% if sota_hypothesis_and_feedback %}
+    The SOTA hypothesis and the corresponding feedback are as follows:
+    {{ sota_hypothesis_and_feedback }}
+    {% endif %}
+
+    Please generate the new {{ targets }} based on the information above.

rdagent/components/workflow/rd_loop.py

@@ -3,9 +3,11 @@ Model workflow with session control
 It is from `rdagent/app/qlib_rd_loop/model.py` and try to replace `rdagent/app/qlib_rd_loop/RDAgent.py`
 """
 
+import asyncio
 from typing import Any
 
 from rdagent.components.workflow.conf import BasePropSetting
+from rdagent.core.conf import RD_AGENT_SETTINGS
 from rdagent.core.developer import Developer
 from rdagent.core.proposal import (
     Experiment2Feedback,
@@ -24,25 +26,20 @@ from rdagent.utils.workflow import LoopBase, LoopMeta
 class RDLoop(LoopBase, metaclass=LoopMeta):
 
     def __init__(self, PROP_SETTING: BasePropSetting):
-        with logger.tag("init"):
-            scen: Scenario = import_class(PROP_SETTING.scen)()
-            logger.log_object(scen, tag="scenario")
+        scen: Scenario = import_class(PROP_SETTING.scen)()
+        logger.log_object(scen, tag="scenario")
+        logger.log_object(PROP_SETTING.model_dump(), tag="RDLOOP_SETTINGS")
+        logger.log_object(RD_AGENT_SETTINGS.model_dump(), tag="RD_AGENT_SETTINGS")
+        self.hypothesis_gen: HypothesisGen = import_class(PROP_SETTING.hypothesis_gen)(scen)
 
-            self.hypothesis_gen: HypothesisGen = import_class(PROP_SETTING.hypothesis_gen)(scen)
-            logger.log_object(self.hypothesis_gen, tag="hypothesis generator")
+        self.hypothesis2experiment: Hypothesis2Experiment = import_class(PROP_SETTING.hypothesis2experiment)()
 
-            self.hypothesis2experiment: Hypothesis2Experiment = import_class(PROP_SETTING.hypothesis2experiment)()
-            logger.log_object(self.hypothesis2experiment, tag="hypothesis2experiment")
+        self.coder: Developer = import_class(PROP_SETTING.coder)(scen)
+        self.runner: Developer = import_class(PROP_SETTING.runner)(scen)
 
-            self.coder: Developer = import_class(PROP_SETTING.coder)(scen)
-            logger.log_object(self.coder, tag="coder")
-            self.runner: Developer = import_class(PROP_SETTING.runner)(scen)
-            logger.log_object(self.runner, tag="runner")
-
-            self.summarizer: Experiment2Feedback = import_class(PROP_SETTING.summarizer)(scen)
-            logger.log_object(self.summarizer, tag="summarizer")
-            self.trace = Trace(scen=scen)
-            super().__init__()
+        self.summarizer: Experiment2Feedback = import_class(PROP_SETTING.summarizer)(scen)
+        self.trace = Trace(scen=scen)
+        super().__init__()
 
     # excluded steps
     def _propose(self):
@@ -56,37 +53,37 @@ class RDLoop(LoopBase, metaclass=LoopMeta):
         return exp
 
     # included steps
-    def direct_exp_gen(self, prev_out: dict[str, Any]):
-        with logger.tag("r"):  # research
-            hypo = self._propose()
-            exp = self._exp_gen(hypo)
-        return {"propose": hypo, "exp_gen": exp}
+    async def direct_exp_gen(self, prev_out: dict[str, Any]):
+        while True:
+            if self.get_unfinished_loop_cnt(self.loop_idx) < RD_AGENT_SETTINGS.get_max_parallel():
+                hypo = self._propose()
+                exp = self._exp_gen(hypo)
+                return {"propose": hypo, "exp_gen": exp}
+            await asyncio.sleep(1)
 
     def coding(self, prev_out: dict[str, Any]):
-        with logger.tag("d"):  # develop
-            exp = self.coder.develop(prev_out["direct_exp_gen"]["exp_gen"])
-            logger.log_object(exp.sub_workspace_list, tag="coder result")
+        exp = self.coder.develop(prev_out["direct_exp_gen"]["exp_gen"])
+        logger.log_object(exp.sub_workspace_list, tag="coder result")
         return exp
 
     def running(self, prev_out: dict[str, Any]):
-        with logger.tag("ef"):  # evaluate and feedback
-            exp = self.runner.develop(prev_out["coding"])
-            logger.log_object(exp, tag="runner result")
+        exp = self.runner.develop(prev_out["coding"])
+        logger.log_object(exp, tag="runner result")
         return exp
 
     def feedback(self, prev_out: dict[str, Any]):
         e = prev_out.get(self.EXCEPTION_KEY, None)
         if e is not None:
             feedback = HypothesisFeedback(
-                observations="Error occurred in loop, skip this loop",
+                observations=str(e),
                 hypothesis_evaluation="",
                 new_hypothesis="",
                 reason="",
                 decision=False,
             )
+            logger.log_object(feedback, tag="feedback")
             self.trace.hist.append((prev_out["direct_exp_gen"]["exp_gen"], feedback))
         else:
             feedback = self.summarizer.generate_feedback(prev_out["running"], self.trace)
-            with logger.tag("ef"):  # evaluate and feedback
-                logger.log_object(feedback, tag="feedback")
+            logger.log_object(feedback, tag="feedback")
             self.trace.hist.append((prev_out["running"], feedback))

rdagent/core/conf.py

@@ -1,6 +1,5 @@
 from __future__ import annotations
 
-# TODO: use pydantic for other modules in Qlib
 from pathlib import Path
 from typing import cast
 
@@ -39,17 +38,12 @@ class ExtendedBaseSettings(BaseSettings):
                 env_prefix=base_cls.model_config.get("env_prefix"),
                 env_nested_delimiter=base_cls.model_config.get("env_nested_delimiter"),
             )
-            for base_cls in base_iter(cast(type[ExtendedBaseSettings], settings_cls))
+            for base_cls in base_iter(cast("type[ExtendedBaseSettings]", settings_cls))
         ]
         return init_settings, env_settings, *parent_env_settings, dotenv_settings, file_secret_settings
 
 
 class RDAgentSettings(ExtendedBaseSettings):
-    # TODO: (xiao) I think LLMSetting may be a better name.
-    # TODO: (xiao) I think most of the config should be in oai.config
-    # Log configs
-    # TODO: (xiao) think it can be a separate config.
-    log_trace_path: str | None = None
 
     # azure document intelligence configs
     azure_document_intelligence_key: str = ""
@@ -80,5 +74,27 @@ class RDAgentSettings(ExtendedBaseSettings):
     stdout_context_len: int = 400
     stdout_line_len: int = 10000
 
+    enable_mlflow: bool = False
+
+    initial_fator_library_size: int = 20
+
+    # parallel loop
+    step_semaphore: int | dict[str, int] = 1
+    """the semaphore for each step;  you can specify a overall semaphore
+    or a step-wise semaphore like {"coding": 3, "running": 2}"""
+
+    def get_max_parallel(self) -> int:
+        """Based on the setting of semaphore, return the maximum number of parallel loops"""
+        if isinstance(self.step_semaphore, int):
+            return self.step_semaphore
+        return max(self.step_semaphore.values())
+
+    # NOTE: for debug
+    # the following function only serves as debugging and is necessary in main logic.
+    subproc_step: bool = False
+
+    def is_force_subproc(self) -> bool:
+        return self.subproc_step or self.get_max_parallel() > 1
+
 
 RD_AGENT_SETTINGS = RDAgentSettings()

rdagent/core/exception.py

@@ -57,3 +57,9 @@ class KaggleError(Exception):
     """
     Exceptions raised when calling Kaggle API
     """
+
+
+class PolicyError(Exception):
+    """
+    Exceptions raised due to content management policy
+    """

rdagent/core/experiment.py

@@ -9,12 +9,17 @@ import uuid
 from abc import ABC, abstractmethod
 from collections.abc import Sequence
 from copy import deepcopy
+from dataclasses import dataclass
 from pathlib import Path
-from typing import Any, Generic, Literal, TypeVar
+from typing import TYPE_CHECKING, Any, Generic, TypeVar
 
 from rdagent.core.conf import RD_AGENT_SETTINGS
 from rdagent.core.evaluation import Feedback
 from rdagent.utils import filter_redundant_text
+
+if TYPE_CHECKING:
+    from rdagent.utils.env import EnvResult
+
 from rdagent.utils.fmt import shrink_text
 
 if typing.TYPE_CHECKING:
@@ -59,6 +64,12 @@ ASpecificTask = TypeVar("ASpecificTask", bound=Task)
 ASpecificFeedback = TypeVar("ASpecificFeedback", bound=Feedback)
 
 
+@dataclass
+class RunningInfo:
+    result: object = None  # The result of the experiment, can be different types in different scenarios.
+    running_time: float | None = None
+
+
 class Workspace(ABC, Generic[ASpecificTask, ASpecificFeedback]):
     """
     A workspace is a place to store the task implementation. It evolves as the developer implements the task.
@@ -68,6 +79,7 @@ class Workspace(ABC, Generic[ASpecificTask, ASpecificFeedback]):
     def __init__(self, target_task: ASpecificTask | None = None) -> None:
         self.target_task: ASpecificTask | None = target_task
         self.feedback: ASpecificFeedback | None = None
+        self.running_info: RunningInfo = RunningInfo()
 
     @abstractmethod
     def execute(self, *args: Any, **kwargs: Any) -> object | None:
@@ -250,26 +262,25 @@ class FBWorkspace(Workspace):
         """
         Before each execution, make sure to prepare and inject code.
         """
-        stdout, _ = self.execute_ret_code(env, entry)
-        return stdout
+        result = self.run(env, entry)
+        return result.stdout
 
-    def execute_ret_code(self, env: Env, entry: str) -> tuple[str, int]:
+    def run(self, env: Env, entry: str) -> EnvResult:
         """
-        Execute the code in the environment and return both the stdout and the exit code.
+        Execute the code in the environment and return an EnvResult object (stdout, exit_code, running_time).
 
         Before each execution, make sure to prepare and inject code.
         """
         self.prepare()
         self.inject_files(**self.file_dict)
-        stdout, return_code = env.run_ret_code(entry, str(self.workspace_path))
-        return (
-            shrink_text(
-                filter_redundant_text(stdout),
-                context_lines=RD_AGENT_SETTINGS.stdout_context_len,
-                line_len=RD_AGENT_SETTINGS.stdout_line_len,
-            ),
-            return_code,
+        result = env.run(entry, str(self.workspace_path), env={"PYTHONPATH": "./"})
+        # result is EnvResult
+        result.stdout = shrink_text(
+            filter_redundant_text(result.stdout),
+            context_lines=RD_AGENT_SETTINGS.stdout_context_len,
+            line_len=RD_AGENT_SETTINGS.stdout_line_len,
         )
+        return result
 
     def __str__(self) -> str:
         return f"Workspace[{self.workspace_path=}" + (
@@ -319,11 +330,22 @@ class Experiment(
         # NOTE: Assumption
         # - only runner will assign this variable
         # - We will always create a new Experiment without copying previous results when we goto the next new loop.
-        self.result: object = None  # The result of the experiment, can be different types in different scenarios.
+        self.running_info = RunningInfo()
         self.sub_results: dict[str, float] = (
             {}
         )  # TODO: in Kaggle, now sub results are all saved in self.result, remove this in the future.
 
+        # For parallel multi-trace support
+        self.local_selection: tuple[int, ...] | None = None
+
+    @property
+    def result(self) -> object:
+        return self.running_info.result
+
+    @result.setter
+    def result(self, value: object) -> None:
+        self.running_info.result = value
+
 
 ASpecificExp = TypeVar("ASpecificExp", bound=Experiment)
 

rdagent/core/proposal.py

@@ -1,19 +1,19 @@
-""" """
+# TODO: remove `self.scen` if traces will be passed into the instance.
 
 from __future__ import annotations
 
+import asyncio
 from abc import ABC, abstractmethod
 from typing import TYPE_CHECKING, Generic, TypeVar
 
+from rdagent.core.conf import RD_AGENT_SETTINGS
 from rdagent.core.evaluation import Feedback
 from rdagent.core.experiment import ASpecificExp, Experiment
 from rdagent.core.knowledge_base import KnowledgeBase
 from rdagent.core.scenario import Scenario
 
 if TYPE_CHECKING:
-    from rdagent.core.prompts import Prompts
-
-# class data_ana: XXX
+    from rdagent.utils.workflow.loop import LoopBase
 
 
 class Hypothesis:
@@ -42,12 +42,7 @@ class Hypothesis:
 
     def __str__(self) -> str:
         return f"""Hypothesis: {self.hypothesis}
-                Reason: {self.reason}
-                Concise Reason & Knowledge: {self.concise_reason}
-                Concise Observation: {self.concise_observation}
-                Concise Justification: {self.concise_justification}
-                Concise Knowledge: {self.concise_knowledge}
-                """
+Reason: {self.reason}"""
 
     # source: data_ana | model_nan = None
 
@@ -60,6 +55,7 @@ class ExperimentFeedback(Feedback):
         self,
         reason: str,
         *,
+        code_change_summary: str | None = None,
         decision: bool,
         exception: Exception | None = None,
     ) -> None:
@@ -70,12 +66,17 @@ class ExperimentFeedback(Feedback):
         self.exception: Exception | None = (
             exception  # if the experiment raises exception, it will be integrated into part of the feedback.
         )
+        self.code_change_summary = code_change_summary
 
     def __bool__(self) -> bool:
         return self.decision
 
     def __str__(self) -> str:
-        return f"Decision: {self.decision}\nReason: {self.reason}"
+        res = f"Decision: {self.decision}\nReason: {self.reason}"
+        code_change_summary = getattr(self, "code_change_summary", None)
+        if code_change_summary is not None:
+            res += "\nCode Change Summary: " + code_change_summary
+        return res
 
     @classmethod
     def from_exception(cls, e: Exception) -> ExperimentFeedback:
@@ -93,9 +94,10 @@ class HypothesisFeedback(ExperimentFeedback):
         new_hypothesis: str,
         reason: str,
         *,
+        code_change_summary: str | None = None,
         decision: bool,
     ) -> None:
-        super().__init__(reason, decision=decision)
+        super().__init__(reason, decision=decision, code_change_summary=code_change_summary)
         self.observations = observations
         self.hypothesis_evaluation = hypothesis_evaluation
         self.new_hypothesis = new_hypothesis
@@ -112,11 +114,20 @@ ASpecificKB = TypeVar("ASpecificKB", bound=KnowledgeBase)
 
 
 class Trace(Generic[ASpecificScen, ASpecificKB]):
+    NodeType = tuple[Experiment, ExperimentFeedback]  # Define NodeType as a new type representing the tuple
+    NEW_ROOT: tuple = ()
+
     def __init__(self, scen: ASpecificScen, knowledge_base: ASpecificKB | None = None) -> None:
         self.scen: ASpecificScen = scen
-        self.hist: list[tuple[Experiment, ExperimentFeedback]] = []
+        self.hist: list[Trace.NodeType] = (
+            []
+        )  # List of tuples containing experiments and their feedback, organized over time.
+        self.dag_parent: list[tuple[int, ...]] = []  # List of tuples representing parent indices in the DAG structure.
+        # (,) represents no parent; (1,) presents one parent; (1, 2) represents two parents.
+
         # TODO: self.hist is 2-tuple now, remove hypothesis from it, change old code for this later.
         self.knowledge_base: ASpecificKB | None = knowledge_base
+        self.current_selection: tuple[int, ...] = (-1,)
 
     def get_sota_hypothesis_and_experiment(self) -> tuple[Hypothesis | None, Experiment | None]:
         """Access the last experiment result, sub-task, and the corresponding hypothesis."""
@@ -127,6 +138,105 @@ class Trace(Generic[ASpecificScen, ASpecificKB]):
 
         return None, None
 
+    def is_selection_new_tree(self, selection: tuple[int, ...] | None = None) -> bool:
+        """
+        Check if the current trace is a new tree.
+        - selection maybe (-1,) when the dag_parent is empty.
+        """
+        if selection is None:
+            selection = self.get_current_selection()
+
+        return selection == self.NEW_ROOT or len(self.dag_parent) == 0
+
+    def get_current_selection(self) -> tuple[int, ...]:
+        return self.current_selection
+
+    def set_current_selection(self, selection: tuple[int, ...]) -> None:
+        self.current_selection = selection
+
+    def get_parent_exps(
+        self,
+        selection: tuple[int, ...] | None = None,
+    ) -> list[Trace.NodeType]:
+        """
+        Collect all ancestors of the given selection.
+        The return list follows the order of [root->...->parent->current_node].
+        """
+        if selection is None:
+            selection = self.get_current_selection()
+
+        if self.is_selection_new_tree(selection):
+            return []
+
+        return [self.hist[i] for i in self.get_parents(selection[0])]
+
+    def exp2idx(self, exp: Experiment | list[Experiment]) -> int | list[int] | None:
+        if isinstance(exp, list):
+            exps: list[Experiment] = exp
+
+            # keep the order
+            exp_to_index: dict[Experiment, int] = {_exp: i for i, (_exp, _) in enumerate(self.hist)}
+            return [exp_to_index[_exp] for _exp in exps]
+        for i, (_exp, _) in enumerate(self.hist):
+            if _exp == exp:
+                return i
+        return None
+
+    def idx2exp(self, idx: int | list[int]) -> Experiment | list[Experiment]:
+        if isinstance(idx, list):
+            idxs: list[int] = idx
+            return [self.hist[_idx][0] for _idx in idxs]
+        return self.hist[idx][0]
+
+    def is_parent(self, parent_idx: int, child_idx: int) -> bool:
+        ancestors = self.get_parents(child_idx)
+        return parent_idx in ancestors
+
+    def get_parents(self, child_idx: int) -> list[int]:
+        if self.is_selection_new_tree((child_idx,)):
+            return []
+
+        ancestors: list[int] = []
+        curr = child_idx
+        while True:
+            ancestors.insert(0, curr)
+            parent_tuple = self.dag_parent[curr]
+            if not parent_tuple or parent_tuple[0] == curr:
+                break
+            curr = parent_tuple[0]
+
+        return ancestors
+
+
+class CheckpointSelector:
+    """
+    In the trace, we may start from any check point (we'll represent it as a variable `from_checkpoint_idx`)
+    """
+
+    @abstractmethod
+    def get_selection(self, trace: Trace) -> tuple[int, ...] | None:
+        """
+        checkpoint_idx represents the place where we want to create a new node.
+        the return value should be the idx of target node (the parent of the new generating node).
+        - `(-1, )` represents starting from the latest trial in the trace - default value
+        - `(idx, )` represents starting from the `idx`-th trial in the trace.
+        - `None` represents starting from scratch (start a new trace)
+
+        - More advanced selection strategies in `select.py`
+        """
+
+
+class SOTAexpSelector:
+    """
+    Select the SOTA experiment from the trace to submit
+    """
+
+    @abstractmethod
+    def get_sota_exp_to_submit(self, trace: Trace) -> Experiment | None:
+        """
+        Select the SOTA experiment from the trace to submit
+        """
+
 
 class ExpGen(ABC):
 
@@ -148,13 +258,19 @@ class ExpGen(ABC):
             )
         """
 
+    async def async_gen(self, trace: Trace, loop: LoopBase) -> Experiment:
+        """
+        generate the experiment and decide whether to stop yield generation and give up control to other routines.
+        """
+        # we give a default implementation here.
+        # The proposal is set to try best to generate the experiment in max-parallel level.
+        while True:
+            if loop.get_unfinished_loop_cnt(loop.loop_idx) < RD_AGENT_SETTINGS.get_max_parallel():
+                return self.gen(trace)
+            await asyncio.sleep(1)
+
 
 class HypothesisGen(ABC):
-    # NOTE: the design is a little wierd
-    # - Sometimes we want accurate access the prompts in a specific level
-    #   - It renders the prompt to a specific abstract level
-    # - Sometimes we want to access the most recent level prompts
-    prompts: Prompts  # this is a class level prompt.
 
     def __init__(self, scen: Scenario) -> None:
         self.scen = scen

rdagent/core/utils.py

@@ -69,7 +69,7 @@ def similarity(text1: str, text2: str) -> int:
     text2 = text2 if isinstance(text2, str) else ""
 
     # Maybe we can use other similarity algorithm such as tfidf
-    return cast(int, fuzz.ratio(text1, text2))  # mypy does not regard it as int
+    return cast("int", fuzz.ratio(text1, text2))  # mypy does not regard it as int
 
 
 def import_class(class_path: str) -> Any:

rdagent/log/base.py

@@ -5,7 +5,7 @@ from collections.abc import Generator
 from dataclasses import dataclass
 from datetime import datetime
 from pathlib import Path
-from typing import Literal, Optional, Union
+from typing import Literal, Optional
 
 
 @dataclass
@@ -43,10 +43,8 @@ class Storage:
     def log(
         self,
         obj: object,
-        name: str = "",
-        save_type: Literal["json", "text", "pkl"] = "text",
+        tag: str = "",
         timestamp: datetime | None = None,
-        **kwargs: dict,
     ) -> str | Path:
         """
 
@@ -66,15 +64,22 @@ class Storage:
         ...
 
     @abstractmethod
-    def iter_msg(self, watch: bool = False) -> Generator[Message, None, None]:
+    def iter_msg(self) -> Generator[Message, None, None]:
         """
-        Parameters
-        ----------
-        watch : bool
-            should we watch the new content and display them
+        Iterate the message in the storage.
         """
         ...
 
+    @abstractmethod
+    def truncate(self, time: datetime) -> None:
+        """
+        Remove all log entries after the specified time.
+        """
+        ...
+
+    def __str__(self) -> str:
+        return self.__class__.__name__
+
 
 class View:
     """

rdagent/log/conf.py

@@ -0,0 +1,27 @@
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+from pydantic_settings import SettingsConfigDict
+
+from rdagent.core.conf import ExtendedBaseSettings
+
+
+class LogSettings(ExtendedBaseSettings):
+    model_config = SettingsConfigDict(env_prefix="LOG_", protected_namespaces=())
+
+    trace_path: str = str(Path.cwd() / "log" / datetime.now(timezone.utc).strftime("%Y-%m-%d_%H-%M-%S-%f"))
+
+    format_console: str | None = None
+    """"If it is None, leave it as the default"""
+
+    ui_server_port: int | None = None
+
+    storages: dict[str, list[int | str]] = {}
+
+    def model_post_init(self, _context: Any, /) -> None:
+        if self.ui_server_port is not None:
+            self.storages["rdagent.log.ui.storage.WebStorage"] = [self.ui_server_port, self.trace_path]
+
+
+LOG_SETTINGS = LogSettings()

rdagent/log/logger.py

@@ -1,28 +1,26 @@
-import json
 import os
-import pickle
 import sys
 from contextlib import contextmanager
-from datetime import datetime, timezone
-from functools import partial
-from logging import LogRecord
-from multiprocessing import Pipe
-from multiprocessing.connection import Connection
+from contextvars import ContextVar
+from datetime import datetime
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Dict, Generator, Union
+from typing import Generator
 
 from loguru import logger
 
-if TYPE_CHECKING:
-    from loguru import Record
+from .conf import LOG_SETTINGS
+
+if LOG_SETTINGS.format_console is not None:
+    logger.remove()
+    logger.add(sys.stdout, format=LOG_SETTINGS.format_console)
 
 from psutil import Process
 
-from rdagent.core.conf import RD_AGENT_SETTINGS
-from rdagent.core.utils import SingletonBaseClass
+from rdagent.core.utils import SingletonBaseClass, import_class
 
+from .base import Storage
 from .storage import FileStorage
-from .utils import LogColors, get_caller_info
+from .utils import get_caller_info
 
 
 class RDAgentLog(SingletonBaseClass):
@@ -48,45 +46,49 @@ class RDAgentLog(SingletonBaseClass):
 
     """
 
-    # TODO: Simplify it to introduce less concepts ( We may merge RDAgentLog, Storage &)
-    # Solution:  Storage => PipeLog, View => PipeLogView, RDAgentLog is an instance of PipeLogger
-    # PipeLogger.info(...) ,  PipeLogger.get_resp() to get feedback from frontend.
-    # def f():
-    #   logger = PipeLog()
-    #   logger.info("<code>")
-    #   feedback = logger.get_reps()
-    _tag: str = ""
+    # Thread-/coroutine-local tag;  In Linux forked subprocess, it will be copied to the subprocess.
+    _tag_ctx: ContextVar[str] = ContextVar("_tag_ctx", default="")
 
-    def __init__(self, log_trace_path: Union[str, None] = RD_AGENT_SETTINGS.log_trace_path) -> None:
-        if log_trace_path is None:
-            timestamp = datetime.now(timezone.utc).strftime("%Y-%m-%d_%H-%M-%S-%f")
-            self.log_trace_path = Path.cwd() / "log" / timestamp
-        else:
-            self.log_trace_path = Path(log_trace_path)
+    @property
+    def _tag(self) -> str:  # Get current tag
+        return self._tag_ctx.get()
 
-        self.log_trace_path.mkdir(parents=True, exist_ok=True)
+    @_tag.setter  # Set current tag
+    def _tag(self, value: str) -> None:
+        self._tag_ctx.set(value)
 
-        self.storage = FileStorage(self.log_trace_path)
+    def __init__(self) -> None:
+        self.storage = FileStorage(LOG_SETTINGS.trace_path)
+        self.other_storages: list[Storage] = []
+        for storage, args in LOG_SETTINGS.storages.items():
+            storage_cls = import_class(storage)
+            self.other_storages.append(storage_cls(*args))
 
         self.main_pid = os.getpid()
 
-    def set_trace_path(self, log_trace_path: str | Path) -> None:
-        self.log_trace_path = Path(log_trace_path)
-        self.storage = FileStorage(log_trace_path)
-
     @contextmanager
     def tag(self, tag: str) -> Generator[None, None, None]:
         if tag.strip() == "":
             raise ValueError("Tag cannot be empty.")
-        if self._tag != "":
-            tag = "." + tag
-
-        # TODO: It may result in error in mutithreading or co-routine
-        self._tag = self._tag + tag
+        # Generate a new complete tag
+        current_tag = self._tag_ctx.get()
+        new_tag = tag if current_tag == "" else f"{current_tag}.{tag}"
+        # Set and save token for later restore
+        token = self._tag_ctx.set(new_tag)
         try:
             yield
         finally:
-            self._tag = self._tag[: -len(tag)]
+            # Restore previous tag (thread/coroutine safe)
+            self._tag_ctx.reset(token)
+
+    def set_storages_path(self, path: str | Path) -> None:
+        for storage in [self.storage] + self.other_storages:
+            if hasattr(storage, "path"):
+                storage.path = path
+
+    def truncate_storages(self, time: datetime) -> None:
+        for storage in [self.storage] + self.other_storages:
+            storage.truncate(time=time)
 
     def get_pids(self) -> str:
         """
@@ -103,81 +105,32 @@ class RDAgentLog(SingletonBaseClass):
             process = parent_process
         return pid_chain
 
-    def file_format(self, record: "Record", raw: bool = False) -> str:
-        # FIXME: the formmat is tightly coupled with the message reading in storage.
-        record["message"] = LogColors.remove_ansi_codes(record["message"])
-        if raw:
-            return "{message}"
-        return "{time:YYYY-MM-DD HH:mm:ss.SSS} | {level: <8} | {name}:{function}:{line} - {message}\n"
-
     def log_object(self, obj: object, *, tag: str = "") -> None:
-        # TODO: I think we can merge the log_object function with other normal log methods to make the interface simpler.
-        caller_info = get_caller_info()
         tag = f"{self._tag}.{tag}.{self.get_pids()}".strip(".")
 
-        # FIXME: it looks like a hacking... We should redesign it...
-        if "debug_" in tag:
-            debug_log_path = self.log_trace_path / "debug_llm.pkl"
-            debug_data = {"tag": tag, "obj": obj}
-            if debug_log_path.exists():
-                with debug_log_path.open("rb") as f:
-                    existing_data = pickle.load(f)
-                existing_data.append(debug_data)
-                with debug_log_path.open("wb") as f:
-                    pickle.dump(existing_data, f)
-            else:
-                with debug_log_path.open("wb") as f:
-                    pickle.dump([debug_data], f)
-            return
+        for storage in [self.storage] + self.other_storages:
+            storage.log(obj, tag=tag)
 
-        logp = self.storage.log(obj, name=tag, save_type="pkl")
+    def _log(self, level: str, msg: str, *, tag: str = "", raw: bool = False) -> None:
+        caller_info = get_caller_info(level=3)
+        tag = f"{self._tag}.{tag}.{self.get_pids()}".strip(".")
 
-        file_handler_id = logger.add(
-            self.log_trace_path / tag.replace(".", "/") / "common_logs.log", format=self.file_format
-        )
-        logger.patch(lambda r: r.update(caller_info)).info(f"Logging object in {Path(logp).absolute()}")
-        logger.remove(file_handler_id)
-
-    def info(self, msg: str, *, tag: str = "", raw: bool = False) -> None:
-        # TODO: too much duplicated. due to we have no logger with stream context;
-        caller_info = get_caller_info()
         if raw:
             logger.remove()
             logger.add(sys.stderr, format=lambda r: "{message}")
 
-        tag = f"{self._tag}.{tag}.{self.get_pids()}".strip(".")
-        log_file_path = self.log_trace_path / tag.replace(".", "/") / "common_logs.log"
-        if raw:
-            file_handler_id = logger.add(log_file_path, format=partial(self.file_format, raw=True))
-        else:
-            file_handler_id = logger.add(log_file_path, format=self.file_format)
-
-        logger.patch(lambda r: r.update(caller_info)).info(msg)
-        logger.remove(file_handler_id)
+        log_func = getattr(logger.patch(lambda r: r.update(caller_info)), level)
+        log_func(msg)
 
         if raw:
             logger.remove()
             logger.add(sys.stderr)
 
-    def warning(self, msg: str, *, tag: str = "") -> None:
-        # TODO: reuse code
-        # _log(self, msg: str, *, tag: str = "", level=Literal["warning", "error", ..]) -> None:
-        # getattr(logger.patch(lambda r: r.update(caller_info)), level)(msg)
-        caller_info = get_caller_info()
+    def info(self, msg: str, *, tag: str = "", raw: bool = False) -> None:
+        self._log("info", msg, tag=tag, raw=raw)
 
-        tag = f"{self._tag}.{tag}.{self.get_pids()}".strip(".")
-        file_handler_id = logger.add(
-            self.log_trace_path / tag.replace(".", "/") / "common_logs.log", format=self.file_format
-        )
-        logger.patch(lambda r: r.update(caller_info)).warning(msg)
-        logger.remove(file_handler_id)
+    def warning(self, msg: str, *, tag: str = "", raw: bool = False) -> None:
+        self._log("warning", msg, tag=tag, raw=raw)
 
-    def error(self, msg: str, *, tag: str = "") -> None:
-        caller_info = get_caller_info()
-
-        tag = f"{self._tag}.{tag}.{self.get_pids()}".strip(".")
-        file_handler_id = logger.add(
-            self.log_trace_path / tag.replace(".", "/") / "common_logs.log", format=self.file_format
-        )
-        logger.patch(lambda r: r.update(caller_info)).error(msg)
-        logger.remove(file_handler_id)
+    def error(self, msg: str, *, tag: str = "", raw: bool = False) -> None:
+        self._log("error", msg, tag=tag, raw=raw)

rdagent/log/mle_summary.py

@@ -1,36 +1,28 @@
-import json
-import re
+import pickle
+import traceback
 from collections import defaultdict
 from pathlib import Path
 
 import fire
 import pandas as pd
 
-from rdagent.app.data_science.conf import DS_RD_SETTING
-from rdagent.components.coder.data_science.conf import get_ds_env
 from rdagent.core.experiment import FBWorkspace
 from rdagent.core.proposal import ExperimentFeedback
 from rdagent.log.storage import FileStorage
+from rdagent.log.utils import extract_json, extract_loopid_func_name, is_valid_session
+from rdagent.log.utils.folder import get_first_session_file_after_duration
 from rdagent.scenarios.data_science.experiment.experiment import DSExperiment
+from rdagent.scenarios.data_science.test_eval import (
+    MLETestEval,
+    NoTestEvalError,
+    get_test_eval,
+)
 from rdagent.scenarios.kaggle.kaggle_crawler import score_rank
-from rdagent.utils.env import DockerEnv, MLEBDockerConf
+from rdagent.utils.workflow import LoopBase
 
-de = get_ds_env("mlebench")
-de.conf.extra_volumes = {f"{DS_RD_SETTING.local_data_path}/zip_files": "/mle/data"}
-de.prepare()
+test_eval = get_test_eval()
 
-
-def extract_mle_json(log_content: str) -> dict | None:
-    match = re.search(r"\{.*\}", log_content, re.DOTALL)
-    if match:
-        return json.loads(match.group(0))
-    return None
-
-
-def extract_loopid_func_name(tag):
-    """提取 Loop ID 和函数名称"""
-    match = re.search(r"Loop_(\d+)\.([^.]+)", tag)
-    return match.groups() if match else (None, None)
+is_mle = isinstance(test_eval, MLETestEval)
 
 
 def save_grade_info(log_trace_path: Path):
@@ -41,27 +33,53 @@ def save_grade_info(log_trace_path: Path):
 
         if "running" in msg.tag:
             if isinstance(msg.content, DSExperiment):
-                mle_score_str = msg.content.experiment_workspace.execute(
-                    env=de,
-                    entry=f"mlebench grade-sample submission.csv {competition} --data-dir /mle/data | tee mle_score.txt",
-                )
-                msg.content.experiment_workspace.execute(env=de, entry="chmod 777 mle_score.txt")
-                trace_storage.log(
-                    mle_score_str, name=f"{msg.tag}.mle_score.pid", save_type="pkl", timestamp=msg.timestamp
-                )
-
-
-def is_valid_session(p: Path) -> bool:
-    return p.is_dir() and p.joinpath("__session__").exists()
+                # TODO:  mle_score.txt is not a general name now.
+                # Please use a more general name like test_score.txt
+                try:
+                    mle_score_str = test_eval.eval(competition, msg.content.experiment_workspace)
+                    trace_storage.log(
+                        mle_score_str, tag=f"{msg.tag}.mle_score.pid", save_type="pkl", timestamp=msg.timestamp
+                    )
+                except Exception as e:
+                    print(f"Error in {log_trace_path}: {e}", traceback.format_exc())
 
 
 def save_all_grade_info(log_folder):
-    for log_trace_path in log_folder.iterdir():
+    for log_trace_path in Path(log_folder).iterdir():
         if is_valid_session(log_trace_path):
-            save_grade_info(log_trace_path)
+            try:
+                save_grade_info(log_trace_path)
+            except NoTestEvalError as e:
+                print(f"Error in {log_trace_path}: {e}", traceback.format_exc())
 
 
-def summarize_folder(log_folder: Path):
+def _get_loop_and_fn_after_hours(log_folder: Path, hours: int):
+    stop_session_fp = get_first_session_file_after_duration(log_folder, f"{hours}h")
+
+    with stop_session_fp.open("rb") as f:
+        session_obj: LoopBase = pickle.load(f)
+
+    loop_trace = session_obj.loop_trace
+    stop_li = max(loop_trace.keys())
+    last_loop = loop_trace[stop_li]
+    last_step = last_loop[-1]
+    stop_fn = session_obj.steps[last_step.step_idx]
+    print(f"Stop Loop: {stop_li=}, {stop_fn=}")
+    files = sorted(
+        (log_folder / "__session__").glob("*/*_*"), key=lambda f: (int(f.parent.name), int(f.name.split("_")[0]))
+    )
+
+    print(f"Max Session: {files[-1:]=}")
+    return stop_li, stop_fn
+
+
+def summarize_folder(log_folder: Path, hours: int | None = None):
+    """
+    Summarize the log folder and save the summary as a pickle file.
+    Args:
+        log_folder (Path): The path to the log folder (contains many log traces).
+        hours (int | None): The number of hours to stat. If None, stat all.
+    """
     log_folder = Path(log_folder)
     stat = defaultdict(dict)
     for log_trace_path in log_folder.iterdir():  # One log trace
@@ -88,44 +106,50 @@ def summarize_folder(log_folder: Path):
         sota_exp_score = None
         sota_exp_rank = None
         grade_output = None
-        for msg in FileStorage(log_trace_path).iter_msg():  # messages in log trace
+
+        if hours:
+            stop_li, stop_fn = _get_loop_and_fn_after_hours(log_trace_path, hours)
+        msgs = [(msg, extract_loopid_func_name(msg.tag)) for msg in FileStorage(log_trace_path).iter_msg()]
+        msgs = [(msg, int(loop_id) if loop_id else loop_id, fn) for msg, (loop_id, fn) in msgs]
+        msgs.sort(key=lambda m: m[1] if m[1] else -1)  # sort by loop id
+        for msg, loop_id, fn in msgs:  # messages in log trace
+            if loop_id:
+                loop_num = max(loop_id + 1, loop_num)
+            if hours and loop_id == stop_li and fn == stop_fn:
+                break
             if msg.tag and "llm" not in msg.tag and "session" not in msg.tag:
                 if "competition" in msg.tag:
                     stat[log_trace_path.name]["competition"] = msg.content
 
                     # get threshold scores
                     workflowexp = FBWorkspace()
-                    stdout = workflowexp.execute(
-                        env=de,
-                        entry=f"mlebench grade-sample None {stat[log_trace_path.name]['competition']} --data-dir /mle/data",
-                    )
-                    grade_output = extract_mle_json(stdout)
-                    if grade_output:
-                        bronze_threshold = grade_output["bronze_threshold"]
-                        silver_threshold = grade_output["silver_threshold"]
-                        gold_threshold = grade_output["gold_threshold"]
-                        median_threshold = grade_output["median_threshold"]
-
-                if "direct_exp_gen" in msg.tag and isinstance(msg.content, DSExperiment):
-                    loop_num += 1
+                    if is_mle:
+                        stdout = workflowexp.execute(
+                            env=test_eval.env,
+                            entry=f"mlebench grade-sample None {stat[log_trace_path.name]['competition']} --data-dir /mle/data",
+                        )
+                        grade_output = extract_json(stdout)
+                        if grade_output:
+                            bronze_threshold = grade_output["bronze_threshold"]
+                            silver_threshold = grade_output["silver_threshold"]
+                            gold_threshold = grade_output["gold_threshold"]
+                            median_threshold = grade_output["median_threshold"]
 
                 if "running" in msg.tag:
                     if isinstance(msg.content, DSExperiment):
-                        submission_path = msg.content.experiment_workspace.workspace_path / "submission.csv"
-                        if submission_path.exists():
-                            made_submission_num += 1
-                            scores_path = msg.content.experiment_workspace.workspace_path / "scores.csv"
-                            valid_scores[loop_num - 1] = pd.read_csv(scores_path, index_col=0)
+                        if msg.content.result is not None:
+                            valid_scores[loop_id] = msg.content.result
                     elif "mle_score" in msg.tag:
-                        loop_id, _ = extract_loopid_func_name(msg.tag)
-                        loop_id = int(loop_id)
-                        grade_output = extract_mle_json(msg.content)
+                        grade_output = extract_json(msg.content)
                         if grade_output:
+                            if grade_output["submission_exists"]:
+                                made_submission_num += 1
                             if grade_output["score"] is not None:
-                                test_scores[loop_id + 1] = grade_output["score"]
-                                _, test_ranks[loop_id + 1] = score_rank(
-                                    stat[log_trace_path.name]["competition"], grade_output["score"]
-                                )
+                                test_scores[loop_id] = grade_output["score"]
+                                if is_mle:
+                                    _, test_ranks[loop_id] = score_rank(
+                                        stat[log_trace_path.name]["competition"], grade_output["score"]
+                                    )
                             if grade_output["valid_submission"]:
                                 valid_submission_num += 1
                             if grade_output["above_median"]:
@@ -158,9 +182,10 @@ def summarize_folder(log_folder: Path):
                                 sota_exp_stat = "made_submission"
                             if grade_output["score"] is not None:
                                 sota_exp_score = grade_output["score"]
-                                _, sota_exp_rank = score_rank(
-                                    stat[log_trace_path.name]["competition"], grade_output["score"]
-                                )
+                                if is_mle:
+                                    _, sota_exp_rank = score_rank(
+                                        stat[log_trace_path.name]["competition"], grade_output["score"]
+                                    )
 
         stat[log_trace_path.name].update(
             {
@@ -185,10 +210,14 @@ def summarize_folder(log_folder: Path):
                 "median_threshold": median_threshold,
             }
         )
-    if (log_folder / "summary.pkl").exists():
-        (log_folder / "summary.pkl").unlink()
-        print("Old summary file removed.")
-    pd.to_pickle(stat, log_folder / "summary.pkl")
+
+    # Save the summary
+    save_name = f"summary_{hours}h.pkl" if hours else "summary.pkl"
+    save_p = log_folder / save_name
+    if save_p.exists():
+        save_p.unlink()
+        print(f"Old {save_name} removed.")
+    pd.to_pickle(stat, save_p)
 
 
 # {

rdagent/log/server/README.md

@@ -0,0 +1,253 @@
+# API
+
+## A. Controls
+
+### 1. /upload [POST]
+
+#### Request
+
+- "scenario": one of six values
+    1. "Finance Data Building"
+    2. "Finance Data Building (Reports)"
+    3. "Finance Model Implementation"
+    4. "General Model Implementation"
+    5. "Medical Model Implementation"
+    6. "Data Science"
+- "files": **2** scenarios need this
+    1. in "Finance Data Building (Reports)" Scenario, one or more pdf files.
+    2. in "General Model Implementation" Scenario, one pdf file or one pdf link like `https://arxiv.org/pdf/2210.09789`
+- "competition": **Data Science** Scenario need this, one of 75 competitions.
+- "loops": Number of loops after which RD-Agent will automatically stop (optional; if not set, it will not stop automatically and must be stopped manually).
+- "all_duration": Total duration (in hours) for which the RD-Agent should run before stopping automatically. If not set, the agent will continue running until stopped manually or by the "loops" parameter.
+
+#### Response
+
+- "id": a unique identifier string, such as `/home/rdagent_log/data_science/competition_A/trace_1` or `/home/rdagent_log/finance/trace_1`, used to mark the series of logs generated by this RD-Agent run.
+
+### 2. /control [POST]
+
+#### Request
+
+- "id": identifier
+- "action": one of three values
+    1. "pause"
+    2. "resume"
+    3. "stop"
+
+#### Response
+
+- "status": "success" / "error: ..."
+
+### 3. /trace [POST]
+
+Returns the sequence of Messages generated for the current id on the backend that **have not yet been returned to the frontend**.
+
+#### Request
+
+- "id": identifier
+- "all": True / False. True means all Messages not yet provided to the frontend will be returned; False returns a random 1 to 10 Messages. In most cases, this should be True.
+- "reset": True / False. Reset means the pointer for "not yet returned to the frontend" will be set back to the first Message generated for this id, i.e., return from the beginning. In most cases, this should be False.
+
+#### Response
+
+- a list of [Messages](#b-messages)
+
+## B. Messages
+
+### Research
+
+Only **2** Message in one loop
+
+1. hypothesis
+
+```json
+{
+    "tag": "research.hypothesis",
+    "timestamp": "<isoformat>",
+    "loop_id": "1",
+    "content": {
+        "hypothesis": "...",
+        "reason": "...",
+        "component": "...", // only exists in Data Science Scenario
+        "concise_reason": "...",
+        "concise_justification": "...",
+        "concise_observation": "...",
+        "concise_knowledge": "...",
+    }
+}
+```
+
+2. tasks
+
+```json
+{
+    "tag": "research.tasks",
+    "timestamp": "<isoformat>",
+    "loop_id": "1",
+    "content": [    // list of tasks
+        {
+            "name": "...",
+            "description": "...",
+            "model_type": "...", // only exists in "Finance Model Implementation", "General Model Implementation", "Medical Model Implementation", or some tasks of "Data Science"
+            "architecture": "...", // same as above
+            "hyperparameters": "...", // same as above
+        },
+        {
+
+        }
+        //... same as above
+    ]
+}
+```
+
+### evolving
+
+- 1 to 10 pairs of Messages (codes & feedbacks), each identified by an "evo_id" indicating the evolving round.
+- In the **Data Science** scenario, each evolving round contains only **one task**, but the "codes" for that task may include **multiple code files**.
+- In other scenarios, each evolving round may contain **multiple tasks**, but each task's "codes" will include only **one code file**.
+
+1. codes
+
+```json
+{
+    "tag": "evolving.codes",
+    "timestamp": "<isoformat>",
+    "loop_id": "1",
+    "evo_id": "0",
+    "content": [ // list of task_name & codes
+        {
+            "evo_id": "0",
+            "target_task_name": "task_1",
+            "workspace": { // one or more codes
+                "a.py": "...<python codes>",
+                "b.py": "...<python codes>",
+                //...
+            }
+        },
+        {
+            "evo_id": "0",
+            "target_task_name": "task_2",
+            "workspace": {
+                "a.py": "...<python codes>",
+                //...
+            }
+        }
+        //... same as above
+    ]
+}
+
+{
+    "tag": "evolving.codes",
+    "timestamp": "<isoformat>",
+    "loop_id": "1",
+    "evo_id": "1",
+    "content": [
+        //... same as above
+    ]
+}
+```
+
+2. feedbacks
+
+```json
+{
+    "tag": "evolving.feedbacks",
+    "timestamp": "<isoformat>",
+    "loop_id": "1",
+    "evo_id": "0",
+    "content": [ // list of feedbacks
+        {
+            "evo_id": "0",
+            "final_decision": "True", // True or False
+            "execution": "...",
+            "code": "...",
+            "return_checking": "..."
+        },
+        //... same as above
+    ]
+}
+
+{
+    "tag": "evolving.codes",
+    "timestamp": "<isoformat>",
+    "loop_id": "1",
+    "evo_id": "1",
+    "content": [
+        //... same as above
+    ]
+}
+```
+
+### feedback
+
+Each tag below appears only once per loop.
+
+1. config (only exists in "Finance Data Building"/"Finance Data Building (Reports)"/"Finance Model Implementation")
+
+```json
+{
+    "tag": "feedback.config",
+    "timestamp": "<isoformat>",
+    "loop_id": "1",
+    "content": {
+        "config": "a markdown string",
+    }
+}
+```
+
+2. return_chart (only exists in "Finance Data Building"/"Finance Data Building (Reports)"/"Finance Model Implementation")
+
+```json
+{
+    "tag": "feedback.return_chart",
+    "timestamp": "<isoformat>",
+    "loop_id": "1",
+    "content": {
+        "chart_html": "chart html codes string",
+    }
+}
+```
+
+3. metric
+
+```json
+{
+    "tag": "feedback.metric",
+    "timestamp": "<isoformat>",
+    "loop_id": "1",
+    "content": {
+        "result": "{ \"<metric_name>\": <value>, ... }" // A JSON string containing metric names and their corresponding values.
+    }
+}
+```
+
+4. hypothesis_feedback
+
+```json
+{
+    "tag": "feedback.hypothesis_feedback",
+    "timestamp": "<isoformat>",
+    "loop_id": "1",
+    "content": {
+        "decision": "True",
+        "reason": "...",
+        "exception": "...",
+        "observations": "...", // may not exists
+        "hypothesis_evaluation": "...", // may not existsc
+        "new_hypothesis": "...", // may not exists
+    }
+}
+```
+
+# TODO
+
+## Session
+
+- How to continue.
+- show & copy trace_id(name)?
+- 
+
+## Page
+
+1. remove Medical, add Finance Whole Pipeline
+2. 

rdagent/log/server/app.py

@@ -0,0 +1,260 @@
+import os
+import random
+import signal
+import subprocess
+from collections import defaultdict
+from datetime import datetime, timezone
+from pathlib import Path
+
+import randomname
+import typer
+from flask import Flask, jsonify, request, send_from_directory
+from flask_cors import CORS
+
+from rdagent.log.storage import FileStorage
+from rdagent.log.ui.conf import UI_SETTING
+from rdagent.log.ui.storage import WebStorage
+from rdagent.log.utils import is_valid_session
+
+app = Flask(__name__, static_folder=UI_SETTING.static_path)
+CORS(app)
+
+rdagent_processes = defaultdict()
+server_port = 19899
+log_folder_path = Path(UI_SETTING.trace_folder).absolute()
+
+
+@app.route("/favicon.ico")
+def favicon():
+    return send_from_directory(app.static_folder, "favicon.ico", mimetype="image/vnd.microsoft.icon")
+
+
+msgs_for_frontend = defaultdict(list)
+pointers = defaultdict(lambda: defaultdict(int))  # pointers[trace_id][user_ip]
+
+
+def read_trace(log_path: Path, id: str = "") -> None:
+    fs = FileStorage(log_path)
+    ws = WebStorage(port=1, path=log_path)
+    msgs_for_frontend[id] = []
+    last_timestamp = None
+    for msg in fs.iter_msg():
+        data = ws._obj_to_json(obj=msg.content, tag=msg.tag, id=id, timestamp=msg.timestamp.isoformat())
+        if data:
+            if isinstance(data, list):
+                for d in data:
+                    msgs_for_frontend[id].append(d["msg"])
+                    last_timestamp = msg.timestamp
+            else:
+                msgs_for_frontend[id].append(data["msg"])
+                last_timestamp = msg.timestamp
+
+    now = datetime.now(timezone.utc)
+    if last_timestamp and (now - last_timestamp).total_seconds() > 1800:
+        msgs_for_frontend[id].append({"tag": "END", "timestamp": now.isoformat(), "content": {}})
+
+
+# load all traces from the log folder
+for p in log_folder_path.glob("*/*/"):
+    if is_valid_session(p):
+        read_trace(p, id=str(p))
+
+
+@app.route("/trace", methods=["POST"])
+def update_trace():
+    global pointers, msgs_for_frontend
+    data = request.get_json()
+    trace_id = data.get("id")
+    return_all = data.get("all")
+    reset = data.get("reset")
+    msg_num = random.randint(1, 10)
+    app.logger.info(data)
+    log_folder_path = Path(UI_SETTING.trace_folder).absolute()
+    if not trace_id:
+        return jsonify({"error": "Trace ID is required"}), 400
+    trace_id = str(log_folder_path / trace_id)
+
+    user_ip = request.remote_addr
+
+    if reset:
+        pointers[trace_id][user_ip] = 0
+
+    start_pointer = pointers[trace_id][user_ip]
+    end_pointer = start_pointer + msg_num
+    if end_pointer > len(msgs_for_frontend[trace_id]) or return_all:
+        end_pointer = len(msgs_for_frontend[trace_id])
+
+    returned_msgs = msgs_for_frontend[trace_id][start_pointer:end_pointer]
+
+    pointers[trace_id][user_ip] = end_pointer
+    if returned_msgs:
+        app.logger.info([msg["tag"] for msg in returned_msgs])
+    return jsonify(returned_msgs), 200
+
+
+@app.route("/upload", methods=["POST"])
+def upload_file():
+    # 获取请求体中的字段
+    global rdagent_processes, server_port
+    scenario = request.form.get("scenario")
+    files = request.files.getlist("files")
+    competition = request.form.get("competition")
+    loop_n = request.form.get("loops")
+    all_duration = request.form.get("all_duration")
+
+    # scenario = "Data Science Loop"
+    if scenario == "Data Science":
+        competition = competition[10:]  # Eg. MLE-Bench:aerial-cactus-competition
+        trace_name = f"{competition}-{randomname.get_name()}"
+    else:
+        trace_name = randomname.get_name()
+    trace_files_path = log_folder_path / scenario / "uploads" / trace_name
+
+    log_trace_path = (log_folder_path / scenario / trace_name).absolute()
+    stdout_path = log_folder_path / scenario / f"{trace_name}.stdout"
+    if not stdout_path.exists():
+        stdout_path.parent.mkdir(parents=True, exist_ok=True)
+
+    # save files
+    for file in files:
+        if file:
+            p = log_folder_path / scenario / "uploads" / trace_name
+            if not p.exists():
+                p.mkdir(parents=True, exist_ok=True)
+            file.save(p / file.filename)
+
+    if scenario == "Finance Data Building":
+        cmds = ["rdagent", "fin_factor"]
+    if scenario == "Finance Data Building (Reports)":
+        cmds = ["rdagent", "fin_factor_report", "--report_folder", str(trace_files_path)]
+    if scenario == "Finance Model Implementation":
+        cmds = ["rdagent", "fin_model"]
+    if scenario == "General Model Implementation":
+        if len(files) == 0:  # files is one link
+            rfp = request.form.get("files")[0]
+        else:  # one file is uploaded
+            rfp = str(trace_files_path / files[0].filename)
+        cmds = ["rdagent", "general_model", "--report_file_path", rfp]
+    if scenario == "Finance Whole Pipeline":
+        cmds = ["rdagent", "fin_quant"]
+    if scenario == "Data Science":
+        cmds = ["rdagent", "data_science", "--competition", competition]
+
+    # time control parameters
+    if scenario != "Finance Data Building (Reports)":
+        if loop_n:
+            cmds += ["--loop_n", loop_n]
+    if all_duration:
+        cmds += ["--timeout", f"{all_duration}h"]
+
+    app.logger.info(f"Started process for {log_trace_path} with parameters: {cmds}")
+    with stdout_path.open("w") as log_file:
+        rdagent_processes[str(log_trace_path)] = subprocess.Popen(
+            cmds,
+            stdout=log_file,
+            stderr=log_file,
+            env={
+                **os.environ,
+                "LOG_TRACE_PATH": str(log_trace_path),
+                "LOG_UI_SERVER_PORT": str(server_port),
+            },
+        )
+    return (
+        jsonify(
+            {
+                "id": f"{scenario}/{trace_name}",
+            }
+        ),
+        200,
+    )
+
+
+@app.route("/receive", methods=["POST"])
+def receive_msgs():
+    try:
+        data = request.get_json()
+        # app.logger.info(data["msg"]["tag"])
+        if not data:
+            return jsonify({"error": "No JSON data received"}), 400
+    except Exception as e:
+        return jsonify({"error": "Internal Server Error"}), 500
+
+    if isinstance(data, list):
+        for d in data:
+            msgs_for_frontend[d["id"]].append(d["msg"])
+    else:
+        msgs_for_frontend[data["id"]].append(data["msg"])
+
+    return jsonify({"status": "success"}), 200
+
+
+@app.route("/control", methods=["POST"])
+def control_process():
+    global rdagent_processes, msgs_for_frontend
+    data = request.get_json()
+    app.logger.info(data)
+    if not data or "id" not in data or "action" not in data:
+        return jsonify({"error": "Missing 'id' or 'action' in request"}), 400
+
+    id = str(log_folder_path / data["id"])
+    action = data["action"]
+
+    if id not in rdagent_processes or rdagent_processes[id] is None:
+        return jsonify({"error": "No running process for given id"}), 400
+
+    process = rdagent_processes[id]
+
+    if process.poll() is not None:
+        msgs_for_frontend[id].append({"tag": "END", "timestamp": datetime.now(timezone.utc).isoformat(), "content": {}})
+        return jsonify({"error": "Process has already terminated"}), 400
+
+    try:
+        if action == "pause":
+            os.kill(process.pid, signal.SIGSTOP)
+            return jsonify({"status": "paused"}), 200
+        elif action == "resume":
+            os.kill(process.pid, signal.SIGCONT)
+            return jsonify({"status": "resumed"}), 200
+        elif action == "stop":
+            process.terminate()
+            process.wait()
+            del rdagent_processes[id]
+            msgs_for_frontend[id].append(
+                {"tag": "END", "timestamp": datetime.now(timezone.utc).isoformat(), "content": {}}
+            )
+            return jsonify({"status": "stopped"}), 200
+        else:
+            return jsonify({"error": "Unknown action"}), 400
+    except Exception as e:
+        return jsonify({"error": f"Failed to {action} process, {e}"}), 500
+
+
+@app.route("/test", methods=["GET"])
+def test():
+    # return 'Hello, World!'
+    global msgs_for_frontend, pointers
+    msgs = {k: [i["tag"] for i in v] for k, v in msgs_for_frontend.items()}
+    pointers = pointers
+    return jsonify({"msgs": msgs, "pointers": pointers}), 200
+
+
+@app.route("/", methods=["GET"])
+def index():
+    # return 'Hello, World!'
+    # return {k: [i["tag"] for i in v] for k, v in msgs_for_frontend.items()}
+    return send_from_directory(app.static_folder, "index.html")
+
+
+@app.route("/<path:fn>", methods=["GET"])
+def server_static_files(fn):
+    return send_from_directory(app.static_folder, fn)
+
+
+def main(port: int = 19899):
+    global server_port
+    server_port = port
+    app.run(debug=False, host="0.0.0.0", port=port)
+
+
+if __name__ == "__main__":
+    typer.run(main)