diff --git a/constraints/3.10.txt b/constraints/3.10.txt index 52c0872f..864aac11 100644 --- a/constraints/3.10.txt +++ b/constraints/3.10.txt @@ -1,3 +1,4 @@ +azure-identity==1.17.1 dill==0.3.9 pillow==10.4.0 psutil==6.1.0 diff --git a/constraints/3.11.txt b/constraints/3.11.txt index 52c0872f..864aac11 100644 --- a/constraints/3.11.txt +++ b/constraints/3.11.txt @@ -1,3 +1,4 @@ +azure-identity==1.17.1 dill==0.3.9 pillow==10.4.0 psutil==6.1.0 diff --git a/rdagent/app/kaggle/conf.py b/rdagent/app/kaggle/conf.py index 13a4d5db..f8f36df1 100644 --- a/rdagent/app/kaggle/conf.py +++ b/rdagent/app/kaggle/conf.py @@ -44,7 +44,7 @@ class KaggleBasePropSetting(BasePropSetting): competition: str = "" """Kaggle competition name, e.g., 'sf-crime'""" - local_data_path: str = "/data/userdata/share/kaggle" + local_data_path: str = "" """Folder storing Kaggle competition data""" if_action_choosing_based_on_UCB: bool = False diff --git a/rdagent/core/exception.py b/rdagent/core/exception.py index 20ebc6dc..2167ab9d 100644 --- a/rdagent/core/exception.py +++ b/rdagent/core/exception.py @@ -42,3 +42,9 @@ class ModelEmptyError(Exception): """ Exceptions raised when no model is generated correctly """ + + +class KaggleError(Exception): + """ + Exceptions raised when calling Kaggle API + """ diff --git a/rdagent/scenarios/kaggle/kaggle_crawler.py b/rdagent/scenarios/kaggle/kaggle_crawler.py index 38ec0947..f35edb5a 100644 --- a/rdagent/scenarios/kaggle/kaggle_crawler.py +++ b/rdagent/scenarios/kaggle/kaggle_crawler.py @@ -15,6 +15,7 @@ from selenium.webdriver.chrome.service import Service from selenium.webdriver.common.by import By from rdagent.app.kaggle.conf import KAGGLE_IMPLEMENT_SETTING +from rdagent.core.exception import KaggleError from rdagent.core.prompts import Prompts from rdagent.log import rdagent_logger as logger from rdagent.oai.llm_utils import APIBackend @@ -99,11 +100,28 @@ def crawl_descriptions(competition: str, wait: float = 3.0, force: bool = False) def download_data(competition: str, local_path: str = KAGGLE_IMPLEMENT_SETTING.local_data_path) -> None: data_path = f"{local_path}/{competition}" if not Path(data_path).exists(): - subprocess.run(["kaggle", "competitions", "download", "-c", competition, "-p", data_path]) + try: + subprocess.run( + ["kaggle", "competitions", "download", "-c", competition, "-p", data_path], + check=True, + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, + ) + except subprocess.CalledProcessError as e: + logger.error(f"Download failed: {e}, stderr: {e.stderr}, stdout: {e.stdout}") + raise KaggleError(f"Download failed: {e}, stderr: {e.stderr}, stdout: {e.stdout}") - # unzip data - with zipfile.ZipFile(f"{data_path}/{competition}.zip", "r") as zip_ref: - zip_ref.extractall(data_path) + # unzip data + unzip_path = f"{local_path}/{competition}" + if not Path(unzip_path).exists(): + unzip_data(unzip_file_path=f"{data_path}/{competition}.zip", unzip_target_path=unzip_path) + for sub_zip_file in Path(unzip_path).rglob("*.zip"): + unzip_data(sub_zip_file, unzip_target_path=unzip_path) + + +def unzip_data(unzip_file_path: str, unzip_target_path: str) -> None: + with zipfile.ZipFile(unzip_file_path, "r") as zip_ref: + zip_ref.extractall(unzip_target_path) def leaderboard_scores(competition: str) -> list[float]: