feat: Added loop code for Kaggle scene. (#211)

* fuse all code into one commit

* remove container auto

* change remove method

* add kaggle env start

* change kaggle api

* change structure

* add crawler

* add requirements

* refeact the code

* delete mistaken codes

* merge docker settings and crawler

* add chrome install README for crawler usage

* Connect scen with Kaggle to download data

* Reformat some files to pass CI.

* fix some ci errors

* fix a ci error

* fix a ci error

---------

Co-authored-by: Bowen Xian <xianbowen@outlook.com>

rdagent/app/kaggle/conf.py

@@ -0,0 +1,42 @@
+from pathlib import Path
+
+from pydantic_settings import BaseSettings
+
+from rdagent.components.workflow.conf import BasePropSetting
+
+
+class PropSetting(BasePropSetting):
+    class Config:
+        env_prefix = "KG_"
+        """Use `KG_` as prefix for environment variables"""
+        protected_namespaces = ()
+        """Add 'model_' to the protected namespaces"""
+
+    # 1) overriding the default
+    scen: str = "rdagent.scenarios.kaggle.experiment.model_experiment.KGModelScenario"
+    """Scenario class for data mining model"""
+
+    hypothesis_gen: str = "rdagent.scenarios.kaggle.proposal.model_proposal.KGModelHypothesisGen"
+    """Hypothesis generation class"""
+
+    hypothesis2experiment: str = "rdagent.scenarios.kaggle.proposal.model_proposal.KGModelHypothesis2Experiment"
+    """Hypothesis to experiment class"""
+
+    coder: str = "rdagent.scenarios.kaggle.developer.model_coder.KGModelCoSTEER"
+    """Coder class"""
+
+    runner: str = "rdagent.scenarios.kaggle.developer.model_runner.KGModelRunner"
+    """Runner class"""
+
+    summarizer: str = "rdagent.scenarios.kaggle.developer.feedback.KGModelHypothesisExperiment2Feedback"
+    """Summarizer class"""
+
+    evolving_n: int = 10
+    """Number of evolutions"""
+
+    evolving_n: int = 10
+
+    competition: str = ""
+
+
+PROP_SETTING = PropSetting()

rdagent/app/kaggle/model.py

@@ -0,0 +1,65 @@
+from collections import defaultdict
+
+import fire
+
+from rdagent.app.kaggle.conf import PROP_SETTING
+from rdagent.components.workflow.conf import BasePropSetting
+from rdagent.components.workflow.rd_loop import RDLoop
+from rdagent.core.exception import ModelEmptyError
+from rdagent.core.proposal import (
+    Hypothesis2Experiment,
+    HypothesisExperiment2Feedback,
+    HypothesisGen,
+    Trace,
+)
+from rdagent.core.utils import import_class
+from rdagent.log import rdagent_logger as logger
+
+
+class ModelRDLoop(RDLoop):
+    def __init__(self, PROP_SETTING: BasePropSetting):
+        with logger.tag("init"):
+            scen: Scenario = import_class(PROP_SETTING.scen)(PROP_SETTING.competition)
+            logger.log_object(scen, tag="scenario")
+
+            self.hypothesis_gen: HypothesisGen = import_class(PROP_SETTING.hypothesis_gen)(scen)
+            logger.log_object(self.hypothesis_gen, tag="hypothesis generator")
+
+            self.hypothesis2experiment: Hypothesis2Experiment = import_class(PROP_SETTING.hypothesis2experiment)()
+            logger.log_object(self.hypothesis2experiment, tag="hypothesis2experiment")
+
+            self.coder: Developer = import_class(PROP_SETTING.coder)(scen)
+            logger.log_object(self.coder, tag="coder")
+            self.runner: Developer = import_class(PROP_SETTING.runner)(scen)
+            logger.log_object(self.runner, tag="runner")
+
+            self.summarizer: HypothesisExperiment2Feedback = import_class(PROP_SETTING.summarizer)(scen)
+            logger.log_object(self.summarizer, tag="summarizer")
+            self.trace = Trace(scen=scen)
+            super(RDLoop, self).__init__()
+
+    skip_loop_error = (ModelEmptyError,)
+
+
+def main(path=None, step_n=None, competition=None):
+    """
+    Auto R&D Evolving loop for models in a kaggle{} scenario.
+
+    You can continue running session by
+
+    .. code-block:: python
+
+        dotenv run -- python rdagent/app/kaggle/model.py [--competition titanic] $LOG_PATH/__session__/1/0_propose  --step_n 1   # `step_n` is a optional paramter
+
+    """
+    if competition:
+        PROP_SETTING.competition = competition
+    if path is None:
+        model_loop = ModelRDLoop(PROP_SETTING)
+    else:
+        model_loop = ModelRDLoop.load(path)
+    model_loop.run(step_n=step_n)
+
+
+if __name__ == "__main__":
+    fire.Fire(main)

rdagent/scenarios/kaggle/README.md

@@ -0,0 +1,20 @@
+# Kaggle Crawler
+
+## Install chrome & chromedriver for Linux
+
+In one folder
+```shell
+# install chrome
+wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb
+sudo apt install ./google-chrome-stable_current_amd64.deb
+google-chrome --version
+
+# install chromedriver
+wget https://storage.googleapis.com/chrome-for-testing-public/<chrome-version>/linux64/chromedriver-linux64.zip
+unzip chromedriver-linux64.zip
+cd chromedriver-linux64
+sudo mv chromedriver /usr/local/bin
+sudo chmod +x /usr/local/bin/chromedriver
+
+chromedriver --version
+```

rdagent/scenarios/kaggle/developer/feedback.py

@@ -0,0 +1,68 @@
+import json
+from pathlib import Path
+
+from jinja2 import Environment, StrictUndefined
+
+from rdagent.core.experiment import Experiment
+from rdagent.core.prompts import Prompts
+from rdagent.core.proposal import (
+    Hypothesis,
+    HypothesisExperiment2Feedback,
+    HypothesisFeedback,
+    Trace,
+)
+from rdagent.log import rdagent_logger as logger
+from rdagent.oai.llm_utils import APIBackend
+from rdagent.utils import convert2bool
+
+feedback_prompts = Prompts(file_path=Path(__file__).parent.parent.parent / "qlib" / "prompts.yaml")
+DIRNAME = Path(__file__).absolute().resolve().parent
+
+
+class KGModelHypothesisExperiment2Feedback(HypothesisExperiment2Feedback):
+    """Generated feedbacks on the hypothesis from **Executed** Implementations of different tasks & their comparisons with previous performances"""
+
+    def generate_feedback(self, exp: Experiment, hypothesis: Hypothesis, trace: Trace) -> HypothesisFeedback:
+        """
+        The `ti` should be executed and the results should be included, as well as the comparison between previous results (done by LLM).
+        For example: `mlflow` of Qlib will be included.
+        """
+
+        logger.info("Generating feedback...")
+        # Define the system prompt for hypothesis feedback
+        system_prompt = feedback_prompts["model_feedback_generation"]["system"]
+
+        # Define the user prompt for hypothesis feedback
+        context = trace.scen
+        SOTA_hypothesis, SOTA_experiment = trace.get_sota_hypothesis_and_experiment()
+
+        user_prompt = (
+            Environment(undefined=StrictUndefined)
+            .from_string(feedback_prompts["model_feedback_generation"]["user"])
+            .render(
+                context=context,
+                last_hypothesis=SOTA_hypothesis,
+                last_task=SOTA_experiment.sub_tasks[0].get_task_information() if SOTA_hypothesis else None,
+                last_code=SOTA_experiment.sub_workspace_list[0].code_dict.get("model.py") if SOTA_hypothesis else None,
+                last_result=SOTA_experiment.result if SOTA_hypothesis else None,
+                hypothesis=hypothesis,
+                exp=exp,
+            )
+        )
+
+        # Call the APIBackend to generate the response for hypothesis feedback
+        response_hypothesis = APIBackend().build_messages_and_create_chat_completion(
+            user_prompt=user_prompt,
+            system_prompt=system_prompt,
+            json_mode=True,
+        )
+
+        # Parse the JSON response to extract the feedback
+        response_json_hypothesis = json.loads(response_hypothesis)
+        return HypothesisFeedback(
+            observations=response_json_hypothesis.get("Observations", "No observations provided"),
+            hypothesis_evaluation=response_json_hypothesis.get("Feedback for Hypothesis", "No feedback provided"),
+            new_hypothesis=response_json_hypothesis.get("New Hypothesis", "No new hypothesis provided"),
+            reason=response_json_hypothesis.get("Reasoning", "No reasoning provided"),
+            decision=convert2bool(response_json_hypothesis.get("Decision", "false")),
+        )

rdagent/scenarios/kaggle/developer/model_coder.py

@@ -0,0 +1,3 @@
+from rdagent.components.coder.model_coder.CoSTEER import ModelCoSTEER
+
+KGModelCoSTEER = ModelCoSTEER

rdagent/scenarios/kaggle/developer/model_runner.py

@@ -0,0 +1,38 @@
+import shutil
+import uuid
+from pathlib import Path
+
+import pandas as pd
+
+from rdagent.components.coder.model_coder.model import ModelExperiment, ModelFBWorkspace
+from rdagent.components.runner import CachedRunner
+from rdagent.components.runner.conf import RUNNER_SETTINGS
+from rdagent.core.developer import Developer
+from rdagent.core.exception import ModelEmptyError
+from rdagent.log import rdagent_logger as logger
+from rdagent.scenarios.kaggle.experiment.model_experiment import KGModelExperiment
+from rdagent.utils.env import KGDockerEnv
+
+
+class KGModelRunner(CachedRunner[KGModelExperiment]):
+    def develop(self, exp: KGModelExperiment) -> KGModelExperiment:
+        if RUNNER_SETTINGS.cache_result:
+            cache_hit, result = self.get_cache_result(exp)
+            if cache_hit:
+                exp.result = result
+                return exp
+
+        if exp.sub_workspace_list[0].code_dict.get("model.py") is None:
+            raise ModelEmptyError("model.py is empty")
+        # to replace & inject code
+        exp.experiment_workspace.inject_code(**{"model.py": exp.sub_workspace_list[0].code_dict["model.py"]})
+
+        env_to_use = {"PYTHONPATH": "./"}
+
+        result = exp.experiment_workspace.execute(run_env=env_to_use)
+
+        exp.result = result
+        if RUNNER_SETTINGS.cache_result:
+            self.dump_cache_result(exp, result)
+
+        return exp

rdagent/scenarios/kaggle/docker/Dockerfile

@@ -0,0 +1,25 @@
+FROM pytorch/pytorch:latest
+# For GPU support, please choose the proper tag from https://hub.docker.com/r/pytorch/pytorch/tags
+
+RUN apt-get clean && apt-get update && apt-get install -y \  
+    curl \  
+    vim \  
+    git \  
+    build-essential \
+    && rm -rf /var/lib/apt/lists/* 
+
+WORKDIR /workspace
+
+RUN python -m pip install numpy
+RUN python -m pip install --upgrade cython
+# RUN python -m pip install -e .
+
+RUN python -m pip install pandas
+# RUN pip install pyg_lib torch_scatter torch_sparse torch_cluster -f https://data.pyg.org/whl/torch-2.3.0%2Bcu121.html
+RUN pip install torch_geometric
+RUN pip install ogb
+RUN pip install networkx
+RUN pip install scikit-learn
+RUN pip install catboost
+RUN pip install xgboost
+RUN pip install sparse

rdagent/scenarios/kaggle/experiment/model_experiment.py

@@ -0,0 +1,113 @@
+import json
+from pathlib import Path
+
+from jinja2 import Environment, StrictUndefined
+
+from rdagent.components.coder.model_coder.model import (
+    ModelExperiment,
+    ModelFBWorkspace,
+    ModelTask,
+)
+from rdagent.core.prompts import Prompts
+from rdagent.core.scenario import Scenario
+from rdagent.oai.llm_utils import APIBackend
+from rdagent.scenarios.kaggle.experiment.workspace import KGFBWorkspace
+from rdagent.scenarios.kaggle.kaggle_crawler import crawl_descriptions
+
+prompt_dict = Prompts(file_path=Path(__file__).parent / "prompts.yaml")
+
+
+class KGModelExperiment(ModelExperiment[ModelTask, KGFBWorkspace, ModelFBWorkspace]):
+    def __init__(self, *args, **kwargs) -> None:
+        super().__init__(*args, **kwargs)
+        self.experiment_workspace = KGFBWorkspace(template_folder_path=Path(__file__).parent / "model_template")
+
+
+class KGModelScenario(Scenario):
+    def __init__(self, competition: str) -> None:
+        super().__init__()
+        self.competition = competition
+        self.competition_descriptions = crawl_descriptions(competition)
+        self.competition_type = None
+        self.competition_description = None
+        self.target_description = None
+        self.competition_features = None
+        self._analysis_competition_description()
+
+    def _analysis_competition_description(self):
+        # TODO: use gpt to analyze the competition description
+
+        sys_prompt = (
+            Environment(undefined=StrictUndefined)
+            .from_string(prompt_dict["kg_description_template"]["system"])
+            .render()
+        )
+
+        user_prompt = (
+            Environment(undefined=StrictUndefined)
+            .from_string(prompt_dict["kg_description_template"]["user"])
+            .render(
+                competition_descriptions=self.competition_descriptions,
+            )
+        )
+
+        response_analysis = APIBackend().build_messages_and_create_chat_completion(
+            user_prompt=user_prompt,
+            system_prompt=sys_prompt,
+            json_mode=True,
+        )
+
+        response_json_analysis = json.loads(response_analysis)
+        self.competition_type = response_json_analysis.get("Competition Type", "No type provided")
+        self.competition_description = response_json_analysis.get("Competition Description", "No description provided")
+        self.target_description = response_json_analysis.get("Target Description", "No target provided")
+        self.competition_features = response_json_analysis.get("Competition Features", "No features provided")
+
+    @property
+    def background(self) -> str:
+        background_template = prompt_dict["kg_model_background"]
+
+        background_prompt = (
+            Environment(undefined=StrictUndefined)
+            .from_string(background_template)
+            .render(
+                competition_type=self.competition_type,
+                competition_description=self.competition_description,
+                target_description=self.target_description,
+                competition_features=self.competition_features,
+            )
+        )
+
+        return background_prompt
+
+    @property
+    def source_data(self) -> str:
+        raise NotImplementedError("source_data is not implemented")
+
+    @property
+    def output_format(self) -> str:
+        return prompt_dict["kg_model_output_format"]
+
+    @property
+    def interface(self) -> str:
+        return prompt_dict["kg_model_interface"]
+
+    @property
+    def simulator(self) -> str:
+        return prompt_dict["kg_model_simulator"]
+
+    @property
+    def rich_style_description(self) -> str:
+        return """
+kaggle scen """
+
+    def get_scenario_all_desc(self) -> str:
+        return f"""Background of the scenario:
+{self.background}
+The interface you should follow to write the runnable code:
+{self.interface}
+The output of your code should be in the format:
+{self.output_format}
+The simulator user can use to test your model:
+{self.simulator}
+"""

rdagent/scenarios/kaggle/experiment/model_template/README.md

@@ -0,0 +1,3 @@
+## This folder is a template to be copied from for each model implementation & running process. 
+
+Components: Dummy model.py, versatile conf.yaml, and a result reader. 

rdagent/scenarios/kaggle/experiment/model_template/train.py

@@ -0,0 +1,137 @@
+import random
+
+import numpy as np
+import pandas as pd
+import torch
+import torch.nn as nn
+from model import model_cls
+from sklearn.compose import ColumnTransformer
+from sklearn.impute import SimpleImputer
+from sklearn.metrics import accuracy_score
+from sklearn.model_selection import train_test_split
+from sklearn.pipeline import Pipeline
+from sklearn.preprocessing import OneHotEncoder
+
+# Set random seed for reproducibility
+SEED = 42
+random.seed(SEED)
+torch.manual_seed(SEED)
+np.random.seed(SEED)
+
+
+def compute_metrics_for_classification(y_true, y_pred):
+    """Compute accuracy metric for classification."""
+    accuracy = accuracy_score(y_true, y_pred)
+    return accuracy
+
+
+def train_model(X_train, y_train, X_valid, y_valid):
+    """Define and train the model."""
+    X_train_dense = X_train.toarray() if hasattr(X_train, "toarray") else X_train
+    X_valid_dense = X_valid.toarray() if hasattr(X_valid, "toarray") else X_valid
+
+    X_train_tensor = torch.tensor(X_train_dense, dtype=torch.float32)
+    y_train_tensor = torch.tensor(y_train, dtype=torch.float32).unsqueeze(1)
+    X_valid_tensor = torch.tensor(X_valid_dense, dtype=torch.float32)
+    y_valid_tensor = torch.tensor(y_valid, dtype=torch.float32).unsqueeze(1)
+
+    # Define the model
+    model = model_cls(num_features=X_train.shape[1])
+
+    # Define loss function and optimizer
+    criterion = nn.BCELoss()  # Binary cross entropy loss
+    optimizer = torch.optim.Adam(model.parameters(), lr=0.001)
+
+    # Train the model
+    num_epochs = 150  # Number of epochs
+    for epoch in range(num_epochs):
+        model.train()
+        optimizer.zero_grad()
+        y_train_pred = model(X_train_tensor)
+        loss = criterion(y_train_pred, y_train_tensor)
+        loss.backward()
+        optimizer.step()
+
+        # Evaluate model on validation set after each epoch
+        model.eval()
+        with torch.no_grad():
+            y_valid_pred = model(X_valid_tensor)
+            valid_loss = criterion(y_valid_pred, y_valid_tensor)
+        print(f"Epoch {epoch+1}/{num_epochs}, Loss: {loss.item()}, Validation Loss: {valid_loss.item()}")
+
+    return model
+
+
+def predict(model, X):
+    """Make predictions using the trained model."""
+    X_dense = X.toarray() if hasattr(X, "toarray") else X
+    X_tensor = torch.tensor(X_dense, dtype=torch.float32)
+    model.eval()
+
+    with torch.no_grad():
+        y_pred = model(X_tensor)
+    y_pred = y_pred.numpy().flatten()
+    return y_pred > 0.5  # Apply threshold to get boolean predictions
+
+
+if __name__ == "__main__":
+    # Load and preprocess the data
+    data_df = pd.read_csv("/root/.data/train.csv")
+    data_df = data_df.drop(["PassengerId", "Name"], axis=1)
+
+    X = data_df.drop(["Transported"], axis=1)
+    y = data_df.Transported.to_numpy()
+
+    # Identify numerical and categorical features
+    numerical_cols = [cname for cname in X.columns if X[cname].dtype in ["int64", "float64"]]
+    categorical_cols = [cname for cname in X.columns if X[cname].dtype == "object"]
+
+    # Define preprocessors for numerical and categorical features
+    categorical_transformer = Pipeline(
+        steps=[("imputer", SimpleImputer(strategy="most_frequent")), ("onehot", OneHotEncoder(handle_unknown="ignore"))]
+    )
+
+    numerical_transformer = Pipeline(steps=[("imputer", SimpleImputer(strategy="mean"))])
+
+    # Combine preprocessing steps
+    preprocessor = ColumnTransformer(
+        transformers=[
+            ("cat", categorical_transformer, categorical_cols),
+            ("num", numerical_transformer, numerical_cols),
+        ]
+    )
+
+    # Split the data into training and validation sets
+    X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=0.10, random_state=SEED)
+
+    # Fit the preprocessor on the training data and transform both training and validation data
+    preprocessor.fit(X_train)
+    X_train = preprocessor.transform(X_train)
+    X_valid = preprocessor.transform(X_valid)
+
+    # Train the model
+    model = train_model(X_train, y_train, X_valid, y_valid)
+
+    # Evaluate the model on the validation set
+    y_valid_pred = predict(model, X_valid)
+    accuracy = compute_metrics_for_classification(y_valid, y_valid_pred)
+    print("Final Accuracy on validation set: ", accuracy)
+
+    # Save the validation accuracy
+    pd.Series(data=[accuracy], index=["ACC"]).to_csv("./submission.csv")
+
+    # Load and preprocess the test set
+    submission_df = pd.read_csv("/root/.data/test.csv")
+    submission_df = submission_df.drop(["PassengerId", "Name"], axis=1)
+    X_test = preprocessor.transform(submission_df)
+
+    # Make predictions on the test set and save them
+    y_test_pred = predict(model, X_test)
+    pd.Series(y_test_pred).to_csv("./submission_update.csv", index=False)
+
+    # submit predictions for the test set
+    submission_df = pd.read_csv("/root/.data/test.csv")
+    submission_df = submission_df.drop(["PassengerId", "Name"], axis=1)
+    X_test = preprocessor.transform(submission_df)
+    y_test_pred = predict(model, X_test)
+    y_test_pred.to_csv("./submission_update.csv")

rdagent/scenarios/kaggle/experiment/prompts.yaml

@@ -0,0 +1,74 @@
+kg_description_template:
+  system: |-
+    You are an assistant that extracts structured information from unstructured text.
+
+  user: |-
+    Based on the following competition description, please extract the following details:
+    1. Competition Type
+    2. Competition Description
+    3. Target Description
+    4. Competition Features
+
+    Competition Description: {{ competition_descriptions }}
+
+kg_model_background: |-
+  You are solving this data science tasks of {{ competition_type }}: 
+  {{competition_description}}
+  
+  We provide an overall pipeline in train.py. Now fill in the provided train.py script to train a  {{ competition_type }} model to get a good performance on this task.
+  
+  The model is a machine learning or deep learning structure designed to predict {{ target_description }}. 
+  The data is extracted from the competition dataset, focusing on passenger attributes like {{ competition_features }}.
+  
+  The model is defined in the following parts:
+  - Name: The name of the model.
+  - Description: A description of the model.
+  - Architecture: The detailed architecture of the model, such as neural network layers or tree structures.
+  
+  The model should provide clear and detailed documentation of its architecture and hyperparameters.
+
+kg_model_interface: |-
+  Your python code should follow the interface to better interact with the user's system.
+  You code should contain several parts:
+  1. The import part: import the necessary libraries.
+  2. A class that is a subclass of pytorch.nn.Module. This class should have an __init__ function and a forward function, which inputs a tensor and outputs a tensor.
+  3. Set a variable called "model_cls" to the class you defined.
+
+  The user will save your code into a python file called "model.py". Then the user imports model_cls in file "model.py" after setting the cwd into the directory:
+  ```python
+  from model import model_cls
+  ```
+  So your python code should follow the pattern:
+  ```python
+  class XXXModel(torch.nn.Module):
+      ...
+  model_cls = XXXModel
+  ```
+
+  The model has one types, "Tabular" for tabular model. The input shape to a tabular model is (batch_size, num_features). 
+  The output shape of the model should be (batch_size, 1).
+  The "batch_size" is a dynamic value which is determined by the input of forward function.
+  The "num_features" is static which will be provided to the model through init function.
+  User will initialize the tabular model with the following code:
+  ```python
+  model = model_cls(num_features=num_features)
+  ```
+  User will initialize the tabular model with the following code:
+  ```python
+  model = model_cls(num_features=num_features)
+  ```
+  No other parameters will be passed to the model so give other parameters a default value or just make them static.
+
+  Don't write any try-except block in your python code. The user will catch the exception message and provide the feedback to you. Also, don't write main function in your python code. The user will call the forward method in the model_cls to get the output tensor.
+
+  Please notice that your model should only use current features as input. The user will provide the input tensor to the model's forward function.
+
+
+kg_model_output_format: |-
+  Your output should be a tensor with shape (batch_size, 1). 
+  The output tensor should be saved in a file named "output.pth" in the same directory as your python file.
+  The user will evaluate the shape of the output tensor so the tensor read from "output.pth" should be 8 numbers.
+
+kg_model_simulator: |-
+  The models will be trained on the Spaceship Titanic dataset and evaluated on their ability to predict whether passengers were transported using metrics like accuracy and AUC-ROC. 
+  Model performance will be iteratively improved based on feedback from evaluation results.

rdagent/scenarios/kaggle/experiment/workspace.py

@@ -0,0 +1,33 @@
+import subprocess
+import zipfile
+from pathlib import Path
+
+import pandas as pd
+
+from rdagent.app.kaggle.conf import PROP_SETTING
+from rdagent.core.experiment import FBWorkspace
+from rdagent.log import rdagent_logger as logger
+from rdagent.utils.env import DockerEnv, KGDockerEnv
+
+
+class KGFBWorkspace(FBWorkspace):
+    def __init__(self, template_folder_path: Path, *args, **kwargs) -> None:
+        super().__init__(*args, **kwargs)
+        self.inject_code_from_folder(template_folder_path)
+
+    def execute(self, run_env: dict = {}, *args, **kwargs) -> str:
+        qtde = KGDockerEnv(PROP_SETTING.competition)
+        qtde.prepare()
+
+        execute_log = qtde.run(
+            local_path=str(self.workspace_path),
+            entry=f"python train.py",
+            env=run_env,
+        )
+
+        csv_path = self.workspace_path / "submission.csv"
+
+        if not csv_path.exists():
+            logger.error(f"File {csv_path} does not exist.")
+            return None
+        return pd.read_csv(csv_path, index_col=0).iloc[:, 0]

rdagent/scenarios/kaggle/kaggle_crawler.py

@@ -0,0 +1,77 @@
+import json
+import time
+from pathlib import Path
+
+from selenium import webdriver
+from selenium.webdriver.chrome.service import Service
+from selenium.webdriver.common.by import By
+
+LOCAL_PATH = "/data/userdata/share/kaggle_competition_descriptions"
+
+options = webdriver.ChromeOptions()
+options.add_argument("--no-sandbox")
+options.add_argument("--disable-dev-shm-usage")
+options.add_argument("--headless")
+
+service = Service("/usr/local/bin/chromedriver")
+
+
+def crawl_descriptions(competition: str, wait: float = 3.0, force: bool = False) -> dict[str, str]:
+    if (fp := Path(f"{LOCAL_PATH}/{competition}.json")).exists() and not force:
+        print(f"Found {competition}.json, loading from local file.")
+        with fp.open("r") as f:
+            return json.load(f)
+
+    driver = webdriver.Chrome(options=options, service=service)
+    overview_url = f"https://www.kaggle.com/competitions/{competition}/overview"
+    driver.get(overview_url)
+    time.sleep(wait)
+    site_body = driver.find_element(By.ID, "site-content")
+    descriptions = {}
+
+    # Get the subtitles
+    elements = site_body.find_elements(By.CSS_SELECTOR, f"a[href^='/competitions/{competition}/overview/']")
+    subtitles = []
+    for e in elements:
+        inner_text = ""
+        for child in e.find_elements(By.XPATH, ".//*"):
+            inner_text += child.get_attribute("innerHTML").strip()
+        subtitles.append(inner_text)
+
+    # Get main contents
+    contents = []
+    elements = site_body.find_elements(By.CSS_SELECTOR, ".sc-iWlrxG.cMAZdc")
+    for e in elements:
+        content = e.get_attribute("innerHTML")
+        contents.append(content)
+
+    assert len(subtitles) == len(contents) + 1 and subtitles[-1] == "Citation"
+    for i in range(len(subtitles) - 1):
+        descriptions[subtitles[i]] = contents[i]
+
+    # Get the citation
+    element = site_body.find_element(By.CSS_SELECTOR, ".sc-ifyrTC.sc-fyziuY")
+    citation = element.get_attribute("innerHTML")
+    descriptions[subtitles[-1]] = citation
+
+    data_url = f"https://www.kaggle.com/competitions/{competition}/data"
+    driver.get(data_url)
+    time.sleep(wait)
+    data_element = driver.find_element(By.CSS_SELECTOR, ".sc-iWlrxG.cMAZdc")
+    descriptions["Data Description"] = data_element.get_attribute("innerHTML")
+
+    driver.quit()
+    with open(f"{LOCAL_PATH}/{competition}.json", "w") as f:
+        json.dump(descriptions, f)
+    return descriptions
+
+
+if __name__ == "__main__":
+    from kaggle.api.kaggle_api_extended import KaggleApi
+
+    api = KaggleApi()
+    api.authenticate()
+    cs = api.competitions_list()
+    for c in cs:
+        name = c.ref.split("/")[-1]
+        crawl_descriptions(name)

rdagent/scenarios/kaggle/proposal/model_proposal.py

@@ -0,0 +1,105 @@
+import json
+from pathlib import Path
+from typing import List, Tuple
+
+from jinja2 import Environment, StrictUndefined
+
+from rdagent.components.coder.model_coder.model import ModelExperiment, ModelTask
+from rdagent.components.proposal.model_proposal import (
+    ModelHypothesis,
+    ModelHypothesis2Experiment,
+    ModelHypothesisGen,
+)
+from rdagent.core.prompts import Prompts
+from rdagent.core.proposal import Hypothesis, Scenario, Trace
+from rdagent.scenarios.kaggle.experiment.model_experiment import KGModelExperiment
+
+prompt_dict = Prompts(file_path=Path(__file__).parent.parent.parent / "qlib" / "prompts.yaml")
+
+KGModelHypothesis = ModelHypothesis
+
+
+class KGModelHypothesisGen(ModelHypothesisGen):
+    """
+    # NOTE: we can share this class across different data mining scenarios
+    # It may better to move the class into components folder like `rdagent/components/proposal/model_proposal.py`
+    # Here is the use case:
+
+    .. code-block:: python
+
+        class XXXDMModelHypothesisGen(DMModelHypothesisGen):
+            prompts: Prompts = a_specifc_prompt_dict
+    """
+
+    def __init__(self, scen: Scenario) -> Tuple[dict, bool]:
+        super().__init__(scen)
+
+    def prepare_context(self, trace: Trace) -> Tuple[dict, bool]:
+        hypothesis_feedback = (
+            Environment(undefined=StrictUndefined)
+            .from_string(prompt_dict["hypothesis_and_feedback"])
+            .render(trace=trace)
+        )
+        context_dict = {
+            "hypothesis_and_feedback": hypothesis_feedback,
+            "RAG": "",
+            "hypothesis_output_format": prompt_dict["hypothesis_output_format"],
+            "hypothesis_specification": "...",
+        }
+        return context_dict, True
+
+    def convert_response(self, response: str) -> ModelHypothesis:
+        response_dict = json.loads(response)
+        hypothesis = KGModelHypothesis(
+            hypothesis=response_dict["hypothesis"],
+            reason=response_dict["reason"],
+            concise_reason=response_dict["concise_reason"],
+            concise_observation=response_dict["concise_observation"],
+            concise_justification=response_dict["concise_justification"],
+            concise_knowledge=response_dict["concise_knowledge"],
+        )
+        return hypothesis
+
+
+class KGModelHypothesis2Experiment(ModelHypothesis2Experiment):
+    def prepare_context(self, hypothesis: Hypothesis, trace: Trace) -> Tuple[dict, bool]:
+        scenario = trace.scen.get_scenario_all_desc()
+        experiment_output_format = prompt_dict["model_experiment_output_format"]
+
+        hypothesis_and_feedback = (
+            Environment(undefined=StrictUndefined)
+            .from_string(prompt_dict["hypothesis_and_feedback"])
+            .render(trace=trace)
+        )
+
+        experiment_list: List[ModelExperiment] = [t[1] for t in trace.hist]
+
+        model_list = []
+        for experiment in experiment_list:
+            model_list.extend(experiment.sub_tasks)
+
+        return {
+            "target_hypothesis": str(hypothesis),
+            "scenario": scenario,
+            "hypothesis_and_feedback": hypothesis_and_feedback,
+            "experiment_output_format": experiment_output_format,
+            "target_list": model_list,
+            "RAG": ...,
+        }, True
+
+    def convert_response(self, response: str, trace: Trace) -> ModelExperiment:
+        response_dict = json.loads(response)
+        tasks = []
+        for model_name in response_dict:
+            description = response_dict[model_name]["description"]
+            formulation = response_dict[model_name]["formulation"]
+            architecture = response_dict[model_name]["architecture"]
+            variables = response_dict[model_name]["variables"]
+            hyperparameters = response_dict[model_name]["hyperparameters"]
+            model_type = response_dict[model_name]["model_type"]
+            tasks.append(
+                ModelTask(model_name, description, formulation, architecture, variables, hyperparameters, model_type)
+            )
+        exp = KGModelExperiment(tasks)
+        exp.based_experiments = [t[1] for t in trace.hist if t[2]]
+        return exp

rdagent/utils/env.py

@@ -9,6 +9,7 @@ Tries to create uniform environment for the agent to run;
 import os
 import subprocess
 import sys
+import zipfile
 from abc import abstractmethod
 from pathlib import Path
 from typing import Dict, Generic, Optional, TypeVar
@@ -159,6 +160,24 @@ class DMDockerConf(DockerConf):
     shm_size: str | None = "16g"
 
 
+class KGDockerConf(DockerConf):
+    class Config:
+        env_prefix = "KG_DOCKER_"
+
+    build_from_dockerfile: bool = True
+    dockerfile_folder_path: Path = Path(__file__).parent.parent / "scenarios" / "kaggle" / "docker"
+    image: str = "local_kg:latest"
+    # image: str = "gcr.io/kaggle-gpu-images/python:latest"
+    mount_path: str = "/workspace/kg_workspace/"
+    default_entry: str = "python train.py"
+    extra_volumes: dict = {
+        # TODO connect to the place where the data is stored
+        Path("git_ignore_folder/data").resolve(): "/root/.data/"
+    }
+
+    share_data_path: str = "/data/userdata/share/kaggle"
+
+
 # physionet.org/files/mimic-eicu-fiddle-feature/1.0.0/FIDDLE_mimic3
 class DockerEnv(Env[DockerConf]):
     # TODO: Save the output into a specific file
@@ -284,3 +303,25 @@ class DMDockerEnv(DockerEnv):
             os.system(cmd)
         else:
             logger.info("Data already exists. Download skipped.")
+
+
+class KGDockerEnv(DockerEnv):
+    """Qlib Torch Docker"""
+
+    def __init__(self, competition: str, conf: DockerConf = KGDockerConf()):
+        super().__init__(conf)
+        self.competition = competition
+
+    def prepare(self):
+        """
+        Download image & data if it doesn't exist
+        """
+        super().prepare()
+
+        # download data
+        data_path = f"{self.conf.share_data_path}/{self.competition}"
+        subprocess.run(["kaggle", "competitions", "download", "-c", self.competition, "-p", data_path])
+
+        # unzip data
+        with zipfile.ZipFile(f"{data_path}/{self.competition}.zip", "r") as zip_ref:
+            zip_ref.extractall(data_path)

requirements.txt

@@ -65,3 +65,7 @@ docker
 streamlit
 plotly
 st-theme
+
+# kaggle crawler
+selenium
+kaggle