gavindiaz/NexQuant
1
0
Fork 0
mirror of https://github.com/NicolasBohn/NexQuant.git synced 2026-07-27 23:47:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: create Jupyter notebook pipeline file based on main.py file (#1134)

Browse Source 
* First commit

* isort

* black

* tweak prompt

* fix for argparse

* fix typo

* add e2e

* Add test files, clean

* fix black settings

* revert

* fix trailing

* remove extra

* comment

* small fix, updated prompt

* Fix argparse

* small improvements

* fix for merge

* fix for merge
This commit is contained in:
Paul
2025-08-05 03:06:57 -07:00
committed by GitHub
parent 56ed919b2e
commit f03b1b918d
19 changed files with 5920 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/notebook/test_notebook_converter.py
+186
View File
@@ -0,0 +1,186 @@
import json
import os
import unittest
from rdagent.components.coder.data_science.share.notebook import NotebookConverter
test_files_dir = os.path.join(os.path.dirname(__file__), "testfiles")
def normalize_nb_json_for_comparison(nb_json_str):
nb_json = json.loads(nb_json_str)
for cell in nb_json["cells"]:
if "id" in cell:
cell.pop("id", None)
return json.dumps(nb_json, indent=4)
class TestNotebookConverter(unittest.TestCase):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.converter = NotebookConverter()
self.maxDiff = None
def test_validation_pass(self):
with open(os.path.join(test_files_dir, "main.py"), "r") as f:
code = f.read()
result = self.converter.validate_code_format(code)
self.assertIsNone(result, "Code format should be valid")
def test_validation_missing_main_fn(self):
with open(os.path.join(test_files_dir, "main_missing_main_fn.py"), "r") as f:
code = f.read()
result = self.converter.validate_code_format(code)
self.assertEqual(
result,
"[Error] No main function found in the code. Please ensure that the main function is defined and contains the necessary print statements to divide sections.",
)
def test_validation_missing_sections(self):
with open(os.path.join(test_files_dir, "main_missing_sections.py"), "r") as f:
code = f.read()
result = self.converter.validate_code_format(code)
self.assertEqual(
result,
"[Error] No sections found in the code. Expected to see 'print(\"Section: <section name>\")' as section dividers. Also make sure that they are actually run and not just comments.",
)
def test_argparse_happy_path(self):
code = """import argparse
parser = argparse.ArgumentParser(description='Test script')
parser.add_argument('--debug', action='store_true', help='Enable debug mode')
args = parser.parse_args()
def main():
print(args.debug)
print("Section: Data Loading")
# Load dataset from CSV into a DataFrame
load_data()
if __name__ == "__main__":
main()"""
notebookJson = json.loads(
self.converter.convert(
task=None,
code=code,
stdout="",
use_debug_flag=True,
)
)
self.assertEqual(
"".join(notebookJson["cells"][0]["source"]),
"""import sys
# hack to allow argparse to work in notebook
sys.argv = ["main.py", "--debug"]
import argparse
parser = argparse.ArgumentParser(description='Test script')
parser.add_argument('--debug', action='store_true', help='Enable debug mode')
args = parser.parse_args()
print(args.debug)""",
)
self.assertEqual(
"".join(notebookJson["cells"][1]["source"]),
"""## Data Loading
Load dataset from CSV into a DataFrame
""",
)
self.assertEqual(
"".join(notebookJson["cells"][2]["source"]),
"""print("Section: Data Loading")
load_data()""",
)
def test_argparse_with_dupe_sys(self):
code = """import argparse
import sys
parser = argparse.ArgumentParser(description='Test script')
parser.add_argument('--debug', action='store_true', help='Enable debug mode')
args = parser.parse_args()
print(sys)
def main():
print(args.debug)
print("Section: Data Loading")
# Load dataset from CSV into a DataFrame
load_data()
if __name__ == "__main__":
main()"""
notebookJson = json.loads(
self.converter.convert(
task=None,
code=code,
stdout="",
use_debug_flag=True,
)
)
self.assertEqual(
"".join(notebookJson["cells"][0]["source"]),
"""import sys
# hack to allow argparse to work in notebook
sys.argv = ["main.py", "--debug"]
import argparse
parser = argparse.ArgumentParser(description='Test script')
parser.add_argument('--debug', action='store_true', help='Enable debug mode')
args = parser.parse_args()
print(sys)
print(args.debug)""",
)
self.assertEqual(
"".join(notebookJson["cells"][1]["source"]),
"""## Data Loading
Load dataset from CSV into a DataFrame
""",
)
self.assertEqual(
"".join(notebookJson["cells"][2]["source"]),
"""print("Section: Data Loading")
load_data()""",
)
def test_convert(self):
with open(os.path.join(test_files_dir, "main.py"), "r") as f:
code = f.read()
notebookJson = self.converter.convert(
task=None,
code=code,
stdout="",
# outfile=os.path.join(test_files_dir, "main.ipynb"), # Uncomment this to save to the file
)
with open(os.path.join(test_files_dir, "main.ipynb"), "r") as f:
expected_notebook = f.read()
self.assertEqual(
normalize_nb_json_for_comparison(notebookJson),
normalize_nb_json_for_comparison(expected_notebook),
"Converted notebook should match expected output",
)
def test_convert_2(self):
with open(os.path.join(test_files_dir, "main2.py"), "r") as f:
code = f.read()
notebookJson = self.converter.convert(
task=None,
code=code,
stdout="",
# outfile=os.path.join(test_files_dir, "main2.ipynb"), # Uncomment this to save to the file
)
with open(os.path.join(test_files_dir, "main2.ipynb"), "r") as f:
expected_notebook = f.read()
self.assertEqual(
normalize_nb_json_for_comparison(notebookJson),
normalize_nb_json_for_comparison(expected_notebook),
"Converted notebook should match expected output",
)
if __name__ == "__main__":
unittest.main()
# pytest test/notebook/test_notebook_converter.py
test/notebook/test_util.py
+1922
View File
File diff suppressed because it is too large Load Diff
test/notebook/testfiles/main.ipynb
+608
View File
@@ -0,0 +1,608 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": null,
"id": "ebeca6b7",
"metadata": {},
"outputs": [],
"source": [
"import sys\n",
"# hack to allow argparse to work in notebook\n",
"sys.argv = [\"main.py\"]\n",
"\n",
"import os\n",
"import time\n",
"import random\n",
"import numpy as np\n",
"import pandas as pd\n",
"\n",
"import torch\n",
"import torch.nn as nn\n",
"import torch.optim as optim\n",
"from torch.utils.data import Dataset, DataLoader\n",
"\n",
"import timm\n",
"import albumentations as A\n",
"from albumentations.pytorch import ToTensorV2\n",
"\n",
"from sklearn.model_selection import StratifiedKFold\n",
"from sklearn.metrics import roc_auc_score, confusion_matrix\n",
"\n",
"import cv2\n",
"import argparse\n",
"\n",
"parser = argparse.ArgumentParser()\n",
"parser.add_argument('--debug', action='store_true', help='Run in debug mode')\n",
"args = parser.parse_args()\n",
"DEBUG = args.debug\n",
"\n",
"SEED = 2024\n",
"np.random.seed(SEED)\n",
"random.seed(SEED)\n",
"torch.manual_seed(SEED)\n",
"torch.cuda.manual_seed_all(SEED)\n",
"\n",
"DEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
"TRAIN_DIR = './workspace_input/train/'\n",
"TEST_DIR = './workspace_input/test/'\n",
"TRAIN_CSV = './workspace_input/train.csv'\n",
"SAMPLE_SUB_PATH = './workspace_input/sample_submission.csv'\n",
"MODEL_DIR = 'models/'\n",
"os.makedirs(MODEL_DIR, exist_ok=True)\n",
"\n",
"class CactusDataset(Dataset):\n",
" def __init__(self, image_ids, labels=None, id2path=None, transforms=None):\n",
" self.image_ids = image_ids\n",
" self.labels = labels\n",
" self.id2path = id2path\n",
" self.transforms = transforms\n",
"\n",
" def __len__(self):\n",
" return len(self.image_ids)\n",
"\n",
" def __getitem__(self, idx):\n",
" img_id = self.image_ids[idx]\n",
" img_path = self.id2path[img_id]\n",
" image = cv2.imread(img_path)\n",
" if image is None:\n",
" raise RuntimeError(f\"Cannot read image at {img_path}\")\n",
" image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n",
" if self.transforms:\n",
" augmented = self.transforms(image=image)\n",
" image = augmented[\"image\"]\n",
" if self.labels is not None:\n",
" label = self.labels[idx]\n",
" return image, label, img_id\n",
" else:\n",
" return image, img_id\n",
"\n"
]
},
{
"cell_type": "markdown",
"id": "9086e8dc",
"metadata": {},
"source": [
"## Data Loading and Preprocessing\n",
"This section loads the train and test data, performs EDA, and prepares the dataset.\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "05509a31",
"metadata": {},
"outputs": [],
"source": [
"def compute_class_weight(y):\n",
" counts = np.bincount(y)\n",
" if len(counts) < 2:\n",
" counts = np.pad(counts, (0, 2-len(counts)), constant_values=0)\n",
" n_pos, n_neg = counts[1], counts[0]\n",
" total = n_pos + n_neg\n",
" minority, majority = min(n_pos, n_neg), max(n_pos, n_neg)\n",
" ratio = majority / (minority + 1e-10)\n",
" need_weights = ratio > 2\n",
" weights = None\n",
" if need_weights:\n",
" inv_freq = [1 / (n_neg + 1e-10), 1 / (n_pos + 1e-10)]\n",
" s = sum(inv_freq)\n",
" weights = [w / s * 2 for w in inv_freq]\n",
" return weights, n_pos, n_neg, ratio, need_weights\n",
"\n",
"def print_eda(train_df):\n",
" print(\"=== Start of EDA part ===\")\n",
" print(\"Shape of train.csv:\", train_df.shape)\n",
" print(\"First 5 rows:\\n\", train_df.head())\n",
" print(\"Column data types:\\n\", train_df.dtypes)\n",
" print(\"Missing values per column:\\n\", train_df.isnull().sum())\n",
" print(\"Unique values per column:\")\n",
" for col in train_df.columns:\n",
" print(f\" - {col}: {train_df[col].nunique()}\")\n",
" label_counts = train_df['has_cactus'].value_counts()\n",
" print(\"Label distribution (has_cactus):\")\n",
" print(label_counts)\n",
" pos, neg = label_counts.get(1, 0), label_counts.get(0, 0)\n",
" total = pos + neg\n",
" if total > 0:\n",
" print(f\" Positive:Negative ratio: {pos}:{neg} ({pos/total:.3f}:{neg/total:.3f})\")\n",
" print(f\" Percentage positive: {pos/total*100:.2f}%\")\n",
" else:\n",
" print(\" No data found.\")\n",
" print(\"Image filename examples:\", train_df['id'].unique()[:5])\n",
" print(\"=== End of EDA part ===\")\n",
"\n",
"print(\"Section: Data Loading and Preprocessing\")\n",
"try:\n",
" train_df = pd.read_csv(TRAIN_CSV)\n",
"except Exception as e:\n",
" print(f\"Failed to load train.csv: {e}\")\n",
" sys.exit(1)\n",
"print_eda(train_df)\n",
"\n",
"train_id2path = {img_id: os.path.join(TRAIN_DIR, img_id) for img_id in train_df['id']}\n",
"try:\n",
" sample_sub = pd.read_csv(SAMPLE_SUB_PATH)\n",
"except Exception as e:\n",
" print(f\"Failed to load sample_submission.csv: {e}\")\n",
" sys.exit(1)\n",
"test_img_ids = list(sample_sub['id'])\n",
"test_id2path = {img_id: os.path.join(TEST_DIR, img_id) for img_id in test_img_ids}\n",
"print(f\"Loaded {len(train_id2path)} train images, {len(test_id2path)} test images.\")\n",
"\n",
"y_train = train_df['has_cactus'].values\n",
"class_weights, n_pos, n_neg, imbalance_ratio, need_weights = compute_class_weight(y_train)\n",
"print(f\"Class stats: Pos={n_pos}, Neg={n_neg}, Imbalance Ratio(majority/minority)={imbalance_ratio:.3f}\")\n",
"print(f\"Use class weights: {need_weights}, Class weights: {class_weights if class_weights is not None else '[1.0,1.0]'}\")\n",
"if class_weights is not None:\n",
" np.save(os.path.join(MODEL_DIR, \"class_weights.npy\"), class_weights)"
]
},
{
"cell_type": "markdown",
"id": "b201cd3f",
"metadata": {},
"source": [
"## Feature Engineering\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "d7d4697e",
"metadata": {},
"outputs": [],
"source": [
"print(\"Section: Feature Engineering\")\n",
"train_df = train_df.copy()\n",
"cv_fold = 5\n",
"skf = StratifiedKFold(n_splits=cv_fold, shuffle=True, random_state=SEED)\n",
"folds = np.zeros(len(train_df), dtype=np.int32)\n",
"for idx, (_, val_idx) in enumerate(skf.split(train_df['id'], train_df['has_cactus'])):\n",
" folds[val_idx] = idx\n",
"train_df['fold'] = folds\n",
"print(f\"Assigned stratified {cv_fold}-fold indices. Fold sample counts:\")\n",
"for f in range(cv_fold):\n",
" dist = train_df.loc[train_df['fold'] == f, 'has_cactus'].value_counts().to_dict()\n",
" print(f\" Fold {f}: n={len(train_df[train_df['fold'] == f])} class dist={dist}\")"
]
},
{
"cell_type": "markdown",
"id": "23e606da",
"metadata": {},
"source": [
"## Model Training and Evaluation\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "853b0c24",
"metadata": {},
"outputs": [],
"source": [
"def inference_and_submission(train_df, train_id2path, test_img_ids, test_id2path, dropout_rate, class_weights, need_weights,\n",
" BATCH_SIZE, N_WORKERS, cv_fold):\n",
" oof_true, oof_pred, fold_scores, fold_val_ids = [], [], [], []\n",
" for fold in range(cv_fold):\n",
" df_val = train_df[train_df['fold'] == fold].reset_index(drop=True)\n",
" val_img_ids = df_val['id'].tolist()\n",
" val_labels = df_val['has_cactus'].values\n",
" val_ds = CactusDataset(val_img_ids, val_labels, id2path=train_id2path, transforms=get_transforms(\"val\"))\n",
" val_loader = get_dataloader(val_ds, BATCH_SIZE, shuffle=False, num_workers=N_WORKERS)\n",
" fold_model_path = os.path.join(MODEL_DIR, f\"efficientnet_b3_fold{fold}.pt\")\n",
" model = get_efficientnet_b3(dropout_rate=dropout_rate)\n",
" model.load_state_dict(torch.load(fold_model_path, map_location='cpu'))\n",
" model.to(DEVICE)\n",
" model.eval()\n",
" fold_class_weights = class_weights if need_weights else None\n",
" if fold_class_weights is not None:\n",
" fold_class_weights = torch.tensor(fold_class_weights).float().to(DEVICE)\n",
" loss_fn = nn.BCEWithLogitsLoss(reduction='none')\n",
" _, val_true, val_pred = eval_model(model, loss_fn, val_loader, DEVICE, fold_class_weights)\n",
" val_auc = roc_auc_score(val_true, val_pred)\n",
" oof_true.append(val_true)\n",
" oof_pred.append(val_pred)\n",
" fold_val_ids.append(val_img_ids)\n",
" fold_scores.append(val_auc)\n",
" print(f\"Reloaded fold {fold}, OOF Validation AUC={val_auc:.5f}\")\n",
"\n",
" all_oof_true = np.concatenate(oof_true)\n",
" all_oof_pred = np.concatenate(oof_pred)\n",
" oof_auc = roc_auc_score(all_oof_true, all_oof_pred)\n",
" oof_cm = confusion_info(all_oof_true, all_oof_pred)\n",
" print(f\"OOF ROC-AUC (from loaded models): {oof_auc:.5f}\")\n",
" print(f\"OOF Confusion Matrix:\\n{oof_cm}\")\n",
"\n",
" test_ds = CactusDataset(\n",
" test_img_ids, labels=None,\n",
" id2path=test_id2path,\n",
" transforms=get_transforms(\"val\")\n",
" )\n",
" test_loader = get_dataloader(test_ds, BATCH_SIZE, shuffle=False, num_workers=N_WORKERS)\n",
" test_pred_list = []\n",
" for fold in range(cv_fold):\n",
" fold_model_path = os.path.join(MODEL_DIR, f\"efficientnet_b3_fold{fold}.pt\")\n",
" model = get_efficientnet_b3(dropout_rate=dropout_rate)\n",
" model.load_state_dict(torch.load(fold_model_path, map_location='cpu'))\n",
" model.to(DEVICE)\n",
" model.eval()\n",
" preds = []\n",
" with torch.no_grad():\n",
" for batch in test_loader:\n",
" images, img_ids = batch\n",
" images = images.to(DEVICE)\n",
" logits = model(images)\n",
" probs = torch.sigmoid(logits).cpu().numpy().reshape(-1)\n",
" preds.append(probs)\n",
" fold_test_pred = np.concatenate(preds)\n",
" test_pred_list.append(fold_test_pred)\n",
" print(f\"Loaded fold {fold} for test prediction.\")\n",
" test_probs = np.mean(test_pred_list, axis=0)\n",
"\n",
" submission = pd.read_csv(SAMPLE_SUB_PATH)\n",
" submission['has_cactus'] = test_probs\n",
" submission.to_csv('submission.csv', index=False)\n",
" print(f\"Saved submission.csv in required format with {len(submission)} rows.\")\n",
"\n",
" scores_df = pd.DataFrame({\n",
" 'Model': [f\"efficientnet_b3_fold{f}\" for f in range(cv_fold)] + ['ensemble'],\n",
" 'ROC-AUC': list(fold_scores) + [oof_auc]\n",
" })\n",
" scores_df.set_index('Model', inplace=True)\n",
" scores_df.to_csv(\"scores.csv\")\n",
" print(f\"Saved cross-validation scores to scores.csv\")\n",
"\n",
"def confusion_info(y_true, y_pred, threshold=0.5):\n",
" preds = (y_pred > threshold).astype(int)\n",
" cm = confusion_matrix(y_true, preds)\n",
" return cm\n",
"\n",
"@torch.no_grad()\n",
"def eval_model(model, loss_fn, dataloader, device, class_weights):\n",
" model.eval()\n",
" y_true, y_pred = [], []\n",
" total_loss = 0.0\n",
" total_samples = 0\n",
" for batch in dataloader:\n",
" images, labels, _ = batch\n",
" images = images.to(device)\n",
" labels = labels.float().unsqueeze(1).to(device)\n",
" logits = model(images)\n",
" probs = torch.sigmoid(logits)\n",
" y_true.append(labels.cpu().numpy())\n",
" y_pred.append(probs.cpu().numpy())\n",
" if class_weights is not None:\n",
" weight = labels * class_weights[1] + (1 - labels) * class_weights[0]\n",
" loss = loss_fn(logits, labels)\n",
" loss = (loss * weight).mean()\n",
" else:\n",
" loss = loss_fn(logits, labels)\n",
" total_loss += loss.item() * labels.size(0)\n",
" total_samples += labels.size(0)\n",
" y_true = np.vstack(y_true).reshape(-1)\n",
" y_pred = np.vstack(y_pred).reshape(-1)\n",
" avg_loss = total_loss / total_samples\n",
" return avg_loss, y_true, y_pred\n",
"\n",
"def train_one_epoch(model, loss_fn, optimizer, scheduler, dataloader, device, class_weights):\n",
" model.train()\n",
" total_loss = 0.0\n",
" total_samples = 0\n",
" for batch in dataloader:\n",
" images, labels, _ = batch\n",
" images = images.to(device)\n",
" labels = labels.float().unsqueeze(1).to(device)\n",
" logits = model(images)\n",
" if class_weights is not None:\n",
" weight = labels * class_weights[1] + (1 - labels) * class_weights[0]\n",
" loss = loss_fn(logits, labels)\n",
" loss = (loss * weight).mean()\n",
" else:\n",
" loss = loss_fn(logits, labels)\n",
" optimizer.zero_grad()\n",
" loss.backward()\n",
" optimizer.step()\n",
" if scheduler is not None:\n",
" scheduler.step()\n",
" total_loss += loss.item() * labels.size(0)\n",
" total_samples += labels.size(0)\n",
" avg_loss = total_loss / total_samples\n",
" return avg_loss\n",
"\n",
"def get_efficientnet_b3(dropout_rate=0.3):\n",
" model = timm.create_model('efficientnet_b3', pretrained=True)\n",
" n_in = model.classifier.in_features if hasattr(model, \"classifier\") else model.fc.in_features\n",
" model.classifier = nn.Sequential(\n",
" nn.Dropout(dropout_rate),\n",
" nn.Linear(n_in, 1)\n",
" )\n",
" return model\n",
"\n",
"def get_dataloader(dataset, batch_size, shuffle=False, num_workers=4, pin_memory=True):\n",
" return DataLoader(\n",
" dataset,\n",
" batch_size=batch_size,\n",
" shuffle=shuffle,\n",
" num_workers=num_workers,\n",
" pin_memory=pin_memory\n",
" )\n",
"\n",
"def get_transforms(mode='train'):\n",
" # Correct Cutout: Albumentations v1.4.15 provides 'Cutout' as a class, but not always in the root.\n",
" # Defensive import; fallback to the most robust method for v1.4.15\n",
" imagenet_mean = [0.485, 0.456, 0.406]\n",
" imagenet_std = [0.229, 0.224, 0.225]\n",
" if mode == 'train':\n",
" min_frac, max_frac = 0.05, 0.2\n",
" min_cut = int(300 * min_frac)\n",
" max_cut = int(300 * max_frac)\n",
" # There is no A.Cutout in v1.4.15 root, but A.augmentations.transforms.Cutout exists.\n",
" try:\n",
" from albumentations.augmentations.transforms import Cutout\n",
" have_cutout = True\n",
" except ImportError:\n",
" have_cutout = False\n",
" this_cut_h = random.randint(min_cut, max_cut)\n",
" this_cut_w = random.randint(min_cut, max_cut)\n",
" cutout_fill = [int(255 * m) for m in imagenet_mean]\n",
" tforms = [\n",
" A.RandomResizedCrop(300, 300, scale=(0.7, 1.0), ratio=(0.8, 1.2), p=1.0),\n",
" A.Rotate(limit=30, p=0.8),\n",
" ]\n",
" if have_cutout:\n",
" tforms.append(\n",
" Cutout(\n",
" num_holes=1,\n",
" max_h_size=this_cut_h,\n",
" max_w_size=this_cut_w,\n",
" fill_value=cutout_fill, # RGB image in albumentations requires [R,G,B]\n",
" always_apply=False,\n",
" p=0.7\n",
" )\n",
" )\n",
" else:\n",
" # No available Cutout, so fallback to no cutout but emit warning\n",
" print(\"WARNING: albumentations.Cutout not found, continuing without Cutout augmentation\")\n",
" tforms.extend([\n",
" A.RandomContrast(limit=0.2, p=0.5),\n",
" A.ColorJitter(brightness=0.2, contrast=0.2, saturation=0.2, hue=0.1, p=0.1),\n",
" A.Normalize(mean=imagenet_mean, std=imagenet_std, max_pixel_value=255.0),\n",
" ToTensorV2()\n",
" ])\n",
" return A.Compose(tforms)\n",
" else:\n",
" return A.Compose([\n",
" A.Resize(300, 300),\n",
" A.Normalize(mean=imagenet_mean, std=imagenet_std, max_pixel_value=255.0),\n",
" ToTensorV2()\n",
" ])\n",
"\n",
"print(\"Section: Model Training and Evaluation\")\n",
"dropout_rate = round(random.uniform(0.2, 0.5), 2)\n",
"print(f\"Model config: EfficientNet-B3, Image size 300, Head dropout={dropout_rate}\")\n",
"\n",
"if DEBUG:\n",
" print(\"DEBUG mode: using 10% subsample and 1 epoch (per fold)\")\n",
" sample_frac = 0.10\n",
" sampled_idxs = []\n",
" for f in range(cv_fold):\n",
" fold_idx = train_df.index[train_df['fold'] == f].tolist()\n",
" fold_labels = train_df.loc[fold_idx, 'has_cactus'].values\n",
" idx_pos = [i for i, l in zip(fold_idx, fold_labels) if l == 1]\n",
" idx_neg = [i for i, l in zip(fold_idx, fold_labels) if l == 0]\n",
" n_pos = max(1, int(sample_frac * len(idx_pos)))\n",
" n_neg = max(1, int(sample_frac * len(idx_neg)))\n",
" if len(idx_pos) > 0:\n",
" sampled_idxs += np.random.choice(idx_pos, n_pos, replace=False).tolist()\n",
" if len(idx_neg) > 0:\n",
" sampled_idxs += np.random.choice(idx_neg, n_neg, replace=False).tolist()\n",
" train_df = train_df.loc[sampled_idxs].reset_index(drop=True)\n",
" print(f\"DEBUG subsample shape: {train_df.shape}\")\n",
" debug_epochs = 1\n",
"else:\n",
" debug_epochs = None\n",
"\n",
"BATCH_SIZE = 64 if torch.cuda.is_available() else 32\n",
"N_WORKERS = 4 if torch.cuda.is_available() else 1\n",
"EPOCHS = 20 if not DEBUG else debug_epochs\n",
"MIN_EPOCHS = 5 if not DEBUG else 1\n",
"EARLY_STOP_PATIENCE = 7 if not DEBUG else 2\n",
"LR = 1e-3\n",
"\n",
"model_files = [os.path.join(MODEL_DIR, f\"efficientnet_b3_fold{f}.pt\") for f in range(cv_fold)]\n",
"if all([os.path.exists(f) for f in model_files]):\n",
" print(\"All fold models found in models/. Running inference and file saving only (no retrain).\")\n",
" inference_and_submission(train_df, train_id2path, test_img_ids, test_id2path, dropout_rate,\n",
" class_weights, need_weights, BATCH_SIZE, N_WORKERS, cv_fold)\n",
" return\n",
"\n",
"oof_true, oof_pred, fold_scores, fold_val_ids = [], [], [], []\n",
"start_time = time.time() if DEBUG else None\n",
"\n",
"for fold in range(cv_fold):\n",
" print(f\"\\n=== FOLD {fold} TRAINING ===\")\n",
" df_train = train_df[train_df['fold'] != fold].reset_index(drop=True)\n",
" df_val = train_df[train_df['fold'] == fold].reset_index(drop=True)\n",
" print(f\"Train size: {df_train.shape[0]}, Val size: {df_val.shape[0]}\")\n",
" train_img_ids = df_train['id'].tolist()\n",
" train_labels = df_train['has_cactus'].values\n",
" val_img_ids = df_val['id'].tolist()\n",
" val_labels = df_val['has_cactus'].values\n",
"\n",
" train_ds = CactusDataset(\n",
" train_img_ids, train_labels,\n",
" id2path=train_id2path,\n",
" transforms=get_transforms(\"train\")\n",
" )\n",
" val_ds = CactusDataset(\n",
" val_img_ids, val_labels,\n",
" id2path=train_id2path,\n",
" transforms=get_transforms(\"val\")\n",
" )\n",
" train_loader = get_dataloader(train_ds, BATCH_SIZE, shuffle=True, num_workers=N_WORKERS)\n",
" val_loader = get_dataloader(val_ds, BATCH_SIZE, shuffle=False, num_workers=N_WORKERS)\n",
" model = get_efficientnet_b3(dropout_rate=dropout_rate)\n",
" model.to(DEVICE)\n",
" loss_fn = nn.BCEWithLogitsLoss(reduction='none')\n",
" optimizer = optim.AdamW(model.parameters(), lr=LR)\n",
" scheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=EPOCHS)\n",
" fold_class_weights = class_weights if need_weights else None\n",
" if fold_class_weights is not None:\n",
" fold_class_weights = torch.tensor(fold_class_weights).float().to(DEVICE)\n",
" best_auc = -np.inf\n",
" best_epoch = -1\n",
" best_model_state = None\n",
" patience = 0\n",
"\n",
" for epoch in range(EPOCHS):\n",
" train_loss = train_one_epoch(\n",
" model, loss_fn, optimizer, scheduler, train_loader, DEVICE, fold_class_weights)\n",
" val_loss, val_true, val_pred = eval_model(\n",
" model, loss_fn, val_loader, DEVICE, fold_class_weights)\n",
" val_auc = roc_auc_score(val_true, val_pred)\n",
" cm = confusion_info(val_true, val_pred)\n",
" print(f\"Epoch {epoch+1:02d}: train_loss={train_loss:.4f} val_loss={val_loss:.4f} val_auc={val_auc:.4f}\")\n",
" print(f\" Val confusion_matrix (rows:true [0,1]; cols:pred [0,1]):\\n{cm}\")\n",
" if val_auc > best_auc:\n",
" best_auc = val_auc\n",
" best_model_state = {k: v.cpu().clone() for k, v in model.state_dict().items()}\n",
" best_epoch = epoch\n",
" patience = 0\n",
" else:\n",
" patience += 1\n",
" if DEBUG and epoch + 1 >= debug_epochs:\n",
" break\n",
" if (epoch + 1) >= MIN_EPOCHS and patience >= EARLY_STOP_PATIENCE:\n",
" print(f\"Early stopping at epoch {epoch+1}, best_epoch={best_epoch+1}.\")\n",
" break\n",
"\n",
" model.load_state_dict(best_model_state)\n",
" fold_model_path = os.path.join(MODEL_DIR, f\"efficientnet_b3_fold{fold}.pt\")\n",
" torch.save(model.state_dict(), fold_model_path)\n",
" print(f\"Saved best model for fold {fold} at {fold_model_path} (best_auc={best_auc:.5f}, best_epoch={best_epoch+1})\")\n",
"\n",
" _, val_true, val_pred = eval_model(model, loss_fn, val_loader, DEVICE, fold_class_weights)\n",
" oof_true.append(val_true)\n",
" oof_pred.append(val_pred)\n",
" fold_val_ids.append(val_img_ids)\n",
" fold_scores.append(best_auc)\n",
" print(f\"OOF stored for fold {fold}, Validation AUC={best_auc:.5f}\")\n",
"\n",
"end_time = time.time() if DEBUG else None\n",
"if DEBUG:\n",
" debug_time = end_time - start_time\n",
" estimated_time = (1 / 0.1) * (EPOCHS / debug_epochs) * debug_time\n",
" print(\"=== Start of Debug Information ===\")\n",
" print(f\"debug_time: {debug_time:.1f}\")\n",
" print(f\"estimated_time: {estimated_time:.1f}\")\n",
" print(\"=== End of Debug Information ===\")"
]
},
{
"cell_type": "markdown",
"id": "c3f0269e",
"metadata": {},
"source": [
"## Ensemble Strategy and Final Predictions\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "308dcdb4",
"metadata": {},
"outputs": [],
"source": [
"print(\"Section: Ensemble Strategy and Final Predictions\")\n",
"all_oof_true = np.concatenate(oof_true)\n",
"all_oof_pred = np.concatenate(oof_pred)\n",
"oof_auc = roc_auc_score(all_oof_true, all_oof_pred)\n",
"oof_cm = confusion_info(all_oof_true, all_oof_pred)\n",
"print(f\"OOF ROC-AUC: {oof_auc:.5f}\")\n",
"print(f\"OOF Confusion Matrix:\\n{oof_cm}\")\n",
"\n",
"test_ds = CactusDataset(\n",
" test_img_ids, labels=None,\n",
" id2path=test_id2path,\n",
" transforms=get_transforms(\"val\")\n",
")\n",
"test_loader = get_dataloader(test_ds, BATCH_SIZE, shuffle=False, num_workers=N_WORKERS)\n",
"test_pred_list = []\n",
"for fold in range(cv_fold):\n",
" fold_model_path = os.path.join(MODEL_DIR, f\"efficientnet_b3_fold{fold}.pt\")\n",
" model = get_efficientnet_b3(dropout_rate=dropout_rate)\n",
" model.load_state_dict(torch.load(fold_model_path, map_location='cpu'))\n",
" model.to(DEVICE)\n",
" model.eval()\n",
" preds = []\n",
" with torch.no_grad():\n",
" for batch in test_loader:\n",
" images, img_ids = batch\n",
" images = images.to(DEVICE)\n",
" logits = model(images)\n",
" probs = torch.sigmoid(logits).cpu().numpy().reshape(-1)\n",
" preds.append(probs)\n",
" fold_test_pred = np.concatenate(preds)\n",
" test_pred_list.append(fold_test_pred)\n",
" print(f\"Loaded fold {fold} for test prediction.\")\n",
"test_probs = np.mean(test_pred_list, axis=0)"
]
},
{
"cell_type": "markdown",
"id": "58b5ded8",
"metadata": {},
"source": [
"## Submission File Generation\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "988914c8",
"metadata": {},
"outputs": [],
"source": [
"print(\"Section: Submission File Generation\")\n",
"submission = pd.read_csv(SAMPLE_SUB_PATH)\n",
"submission['has_cactus'] = test_probs\n",
"submission.to_csv('submission.csv', index=False)\n",
"print(f\"Saved submission.csv in required format with {len(submission)} rows.\")\n",
"\n",
"scores_df = pd.DataFrame({\n",
" 'Model': [f\"efficientnet_b3_fold{f}\" for f in range(cv_fold)] + ['ensemble'],\n",
" 'ROC-AUC': list(fold_scores) + [oof_auc]\n",
"})\n",
"scores_df.set_index('Model', inplace=True)\n",
"scores_df.to_csv(\"scores.csv\")\n",
"print(f\"Saved cross-validation scores to scores.csv\")"
]
}
],
"metadata": {},
"nbformat": 4,
"nbformat_minor": 5
}
test/notebook/testfiles/main.py
+512
View File
@@ -0,0 +1,512 @@
import os
import sys
import time
import random
import numpy as np
import pandas as pd
import torch
import torch.nn as nn
import torch.optim as optim
from torch.utils.data import Dataset, DataLoader
import timm
import albumentations as A
from albumentations.pytorch import ToTensorV2
from sklearn.model_selection import StratifiedKFold
from sklearn.metrics import roc_auc_score, confusion_matrix
import cv2
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('--debug', action='store_true', help='Run in debug mode')
args = parser.parse_args()
DEBUG = args.debug
SEED = 2024
np.random.seed(SEED)
random.seed(SEED)
torch.manual_seed(SEED)
torch.cuda.manual_seed_all(SEED)
DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu")
TRAIN_DIR = './workspace_input/train/'
TEST_DIR = './workspace_input/test/'
TRAIN_CSV = './workspace_input/train.csv'
SAMPLE_SUB_PATH = './workspace_input/sample_submission.csv'
MODEL_DIR = 'models/'
os.makedirs(MODEL_DIR, exist_ok=True)
def print_eda(train_df):
print("=== Start of EDA part ===")
print("Shape of train.csv:", train_df.shape)
print("First 5 rows:\n", train_df.head())
print("Column data types:\n", train_df.dtypes)
print("Missing values per column:\n", train_df.isnull().sum())
print("Unique values per column:")
for col in train_df.columns:
print(f" - {col}: {train_df[col].nunique()}")
label_counts = train_df['has_cactus'].value_counts()
print("Label distribution (has_cactus):")
print(label_counts)
pos, neg = label_counts.get(1, 0), label_counts.get(0, 0)
total = pos + neg
if total > 0:
print(f" Positive:Negative ratio: {pos}:{neg} ({pos/total:.3f}:{neg/total:.3f})")
print(f" Percentage positive: {pos/total*100:.2f}%")
else:
print(" No data found.")
print("Image filename examples:", train_df['id'].unique()[:5])
print("=== End of EDA part ===")
class CactusDataset(Dataset):
def __init__(self, image_ids, labels=None, id2path=None, transforms=None):
self.image_ids = image_ids
self.labels = labels
self.id2path = id2path
self.transforms = transforms
def __len__(self):
return len(self.image_ids)
def __getitem__(self, idx):
img_id = self.image_ids[idx]
img_path = self.id2path[img_id]
image = cv2.imread(img_path)
if image is None:
raise RuntimeError(f"Cannot read image at {img_path}")
image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
if self.transforms:
augmented = self.transforms(image=image)
image = augmented["image"]
if self.labels is not None:
label = self.labels[idx]
return image, label, img_id
else:
return image, img_id
def get_transforms(mode='train'):
# Correct Cutout: Albumentations v1.4.15 provides 'Cutout' as a class, but not always in the root.
# Defensive import; fallback to the most robust method for v1.4.15
imagenet_mean = [0.485, 0.456, 0.406]
imagenet_std = [0.229, 0.224, 0.225]
if mode == 'train':
min_frac, max_frac = 0.05, 0.2
min_cut = int(300 * min_frac)
max_cut = int(300 * max_frac)
# There is no A.Cutout in v1.4.15 root, but A.augmentations.transforms.Cutout exists.
try:
from albumentations.augmentations.transforms import Cutout
have_cutout = True
except ImportError:
have_cutout = False
this_cut_h = random.randint(min_cut, max_cut)
this_cut_w = random.randint(min_cut, max_cut)
cutout_fill = [int(255 * m) for m in imagenet_mean]
tforms = [
A.RandomResizedCrop(300, 300, scale=(0.7, 1.0), ratio=(0.8, 1.2), p=1.0),
A.Rotate(limit=30, p=0.8),
]
if have_cutout:
tforms.append(
Cutout(
num_holes=1,
max_h_size=this_cut_h,
max_w_size=this_cut_w,
fill_value=cutout_fill, # RGB image in albumentations requires [R,G,B]
always_apply=False,
p=0.7
)
)
else:
# No available Cutout, so fallback to no cutout but emit warning
print("WARNING: albumentations.Cutout not found, continuing without Cutout augmentation")
tforms.extend([
A.RandomContrast(limit=0.2, p=0.5),
A.ColorJitter(brightness=0.2, contrast=0.2, saturation=0.2, hue=0.1, p=0.1),
A.Normalize(mean=imagenet_mean, std=imagenet_std, max_pixel_value=255.0),
ToTensorV2()
])
return A.Compose(tforms)
else:
return A.Compose([
A.Resize(300, 300),
A.Normalize(mean=imagenet_mean, std=imagenet_std, max_pixel_value=255.0),
ToTensorV2()
])
def get_dataloader(dataset, batch_size, shuffle=False, num_workers=4, pin_memory=True):
return DataLoader(
dataset,
batch_size=batch_size,
shuffle=shuffle,
num_workers=num_workers,
pin_memory=pin_memory
)
def get_efficientnet_b3(dropout_rate=0.3):
model = timm.create_model('efficientnet_b3', pretrained=True)
n_in = model.classifier.in_features if hasattr(model, "classifier") else model.fc.in_features
model.classifier = nn.Sequential(
nn.Dropout(dropout_rate),
nn.Linear(n_in, 1)
)
return model
def compute_class_weight(y):
counts = np.bincount(y)
if len(counts) < 2:
counts = np.pad(counts, (0, 2-len(counts)), constant_values=0)
n_pos, n_neg = counts[1], counts[0]
total = n_pos + n_neg
minority, majority = min(n_pos, n_neg), max(n_pos, n_neg)
ratio = majority / (minority + 1e-10)
need_weights = ratio > 2
weights = None
if need_weights:
inv_freq = [1 / (n_neg + 1e-10), 1 / (n_pos + 1e-10)]
s = sum(inv_freq)
weights = [w / s * 2 for w in inv_freq]
return weights, n_pos, n_neg, ratio, need_weights
def train_one_epoch(model, loss_fn, optimizer, scheduler, dataloader, device, class_weights):
model.train()
total_loss = 0.0
total_samples = 0
for batch in dataloader:
images, labels, _ = batch
images = images.to(device)
labels = labels.float().unsqueeze(1).to(device)
logits = model(images)
if class_weights is not None:
weight = labels * class_weights[1] + (1 - labels) * class_weights[0]
loss = loss_fn(logits, labels)
loss = (loss * weight).mean()
else:
loss = loss_fn(logits, labels)
optimizer.zero_grad()
loss.backward()
optimizer.step()
if scheduler is not None:
scheduler.step()
total_loss += loss.item() * labels.size(0)
total_samples += labels.size(0)
avg_loss = total_loss / total_samples
return avg_loss
@torch.no_grad()
def eval_model(model, loss_fn, dataloader, device, class_weights):
model.eval()
y_true, y_pred = [], []
total_loss = 0.0
total_samples = 0
for batch in dataloader:
images, labels, _ = batch
images = images.to(device)
labels = labels.float().unsqueeze(1).to(device)
logits = model(images)
probs = torch.sigmoid(logits)
y_true.append(labels.cpu().numpy())
y_pred.append(probs.cpu().numpy())
if class_weights is not None:
weight = labels * class_weights[1] + (1 - labels) * class_weights[0]
loss = loss_fn(logits, labels)
loss = (loss * weight).mean()
else:
loss = loss_fn(logits, labels)
total_loss += loss.item() * labels.size(0)
total_samples += labels.size(0)
y_true = np.vstack(y_true).reshape(-1)
y_pred = np.vstack(y_pred).reshape(-1)
avg_loss = total_loss / total_samples
return avg_loss, y_true, y_pred
def confusion_info(y_true, y_pred, threshold=0.5):
preds = (y_pred > threshold).astype(int)
cm = confusion_matrix(y_true, preds)
return cm
def inference_and_submission(train_df, train_id2path, test_img_ids, test_id2path, dropout_rate, class_weights, need_weights,
BATCH_SIZE, N_WORKERS, cv_fold):
oof_true, oof_pred, fold_scores, fold_val_ids = [], [], [], []
for fold in range(cv_fold):
df_val = train_df[train_df['fold'] == fold].reset_index(drop=True)
val_img_ids = df_val['id'].tolist()
val_labels = df_val['has_cactus'].values
val_ds = CactusDataset(val_img_ids, val_labels, id2path=train_id2path, transforms=get_transforms("val"))
val_loader = get_dataloader(val_ds, BATCH_SIZE, shuffle=False, num_workers=N_WORKERS)
fold_model_path = os.path.join(MODEL_DIR, f"efficientnet_b3_fold{fold}.pt")
model = get_efficientnet_b3(dropout_rate=dropout_rate)
model.load_state_dict(torch.load(fold_model_path, map_location='cpu'))
model.to(DEVICE)
model.eval()
fold_class_weights = class_weights if need_weights else None
if fold_class_weights is not None:
fold_class_weights = torch.tensor(fold_class_weights).float().to(DEVICE)
loss_fn = nn.BCEWithLogitsLoss(reduction='none')
_, val_true, val_pred = eval_model(model, loss_fn, val_loader, DEVICE, fold_class_weights)
val_auc = roc_auc_score(val_true, val_pred)
oof_true.append(val_true)
oof_pred.append(val_pred)
fold_val_ids.append(val_img_ids)
fold_scores.append(val_auc)
print(f"Reloaded fold {fold}, OOF Validation AUC={val_auc:.5f}")
all_oof_true = np.concatenate(oof_true)
all_oof_pred = np.concatenate(oof_pred)
oof_auc = roc_auc_score(all_oof_true, all_oof_pred)
oof_cm = confusion_info(all_oof_true, all_oof_pred)
print(f"OOF ROC-AUC (from loaded models): {oof_auc:.5f}")
print(f"OOF Confusion Matrix:\n{oof_cm}")
test_ds = CactusDataset(
test_img_ids, labels=None,
id2path=test_id2path,
transforms=get_transforms("val")
)
test_loader = get_dataloader(test_ds, BATCH_SIZE, shuffle=False, num_workers=N_WORKERS)
test_pred_list = []
for fold in range(cv_fold):
fold_model_path = os.path.join(MODEL_DIR, f"efficientnet_b3_fold{fold}.pt")
model = get_efficientnet_b3(dropout_rate=dropout_rate)
model.load_state_dict(torch.load(fold_model_path, map_location='cpu'))
model.to(DEVICE)
model.eval()
preds = []
with torch.no_grad():
for batch in test_loader:
images, img_ids = batch
images = images.to(DEVICE)
logits = model(images)
probs = torch.sigmoid(logits).cpu().numpy().reshape(-1)
preds.append(probs)
fold_test_pred = np.concatenate(preds)
test_pred_list.append(fold_test_pred)
print(f"Loaded fold {fold} for test prediction.")
test_probs = np.mean(test_pred_list, axis=0)
submission = pd.read_csv(SAMPLE_SUB_PATH)
submission['has_cactus'] = test_probs
submission.to_csv('submission.csv', index=False)
print(f"Saved submission.csv in required format with {len(submission)} rows.")
scores_df = pd.DataFrame({
'Model': [f"efficientnet_b3_fold{f}" for f in range(cv_fold)] + ['ensemble'],
'ROC-AUC': list(fold_scores) + [oof_auc]
})
scores_df.set_index('Model', inplace=True)
scores_df.to_csv("scores.csv")
print(f"Saved cross-validation scores to scores.csv")
def main():
print("Section: Data Loading and Preprocessing")
# This section loads the train and test data, performs EDA, and prepares the dataset.
try:
train_df = pd.read_csv(TRAIN_CSV)
except Exception as e:
print(f"Failed to load train.csv: {e}")
sys.exit(1)
print_eda(train_df)
train_id2path = {img_id: os.path.join(TRAIN_DIR, img_id) for img_id in train_df['id']}
try:
sample_sub = pd.read_csv(SAMPLE_SUB_PATH)
except Exception as e:
print(f"Failed to load sample_submission.csv: {e}")
sys.exit(1)
test_img_ids = list(sample_sub['id'])
test_id2path = {img_id: os.path.join(TEST_DIR, img_id) for img_id in test_img_ids}
print(f"Loaded {len(train_id2path)} train images, {len(test_id2path)} test images.")
y_train = train_df['has_cactus'].values
class_weights, n_pos, n_neg, imbalance_ratio, need_weights = compute_class_weight(y_train)
print(f"Class stats: Pos={n_pos}, Neg={n_neg}, Imbalance Ratio(majority/minority)={imbalance_ratio:.3f}")
print(f"Use class weights: {need_weights}, Class weights: {class_weights if class_weights is not None else '[1.0,1.0]'}")
if class_weights is not None:
np.save(os.path.join(MODEL_DIR, "class_weights.npy"), class_weights)
print("Section: Feature Engineering")
train_df = train_df.copy()
cv_fold = 5
skf = StratifiedKFold(n_splits=cv_fold, shuffle=True, random_state=SEED)
folds = np.zeros(len(train_df), dtype=np.int32)
for idx, (_, val_idx) in enumerate(skf.split(train_df['id'], train_df['has_cactus'])):
folds[val_idx] = idx
train_df['fold'] = folds
print(f"Assigned stratified {cv_fold}-fold indices. Fold sample counts:")
for f in range(cv_fold):
dist = train_df.loc[train_df['fold'] == f, 'has_cactus'].value_counts().to_dict()
print(f" Fold {f}: n={len(train_df[train_df['fold'] == f])} class dist={dist}")
print("Section: Model Training and Evaluation")
dropout_rate = round(random.uniform(0.2, 0.5), 2)
print(f"Model config: EfficientNet-B3, Image size 300, Head dropout={dropout_rate}")
if DEBUG:
print("DEBUG mode: using 10% subsample and 1 epoch (per fold)")
sample_frac = 0.10
sampled_idxs = []
for f in range(cv_fold):
fold_idx = train_df.index[train_df['fold'] == f].tolist()
fold_labels = train_df.loc[fold_idx, 'has_cactus'].values
idx_pos = [i for i, l in zip(fold_idx, fold_labels) if l == 1]
idx_neg = [i for i, l in zip(fold_idx, fold_labels) if l == 0]
n_pos = max(1, int(sample_frac * len(idx_pos)))
n_neg = max(1, int(sample_frac * len(idx_neg)))
if len(idx_pos) > 0:
sampled_idxs += np.random.choice(idx_pos, n_pos, replace=False).tolist()
if len(idx_neg) > 0:
sampled_idxs += np.random.choice(idx_neg, n_neg, replace=False).tolist()
train_df = train_df.loc[sampled_idxs].reset_index(drop=True)
print(f"DEBUG subsample shape: {train_df.shape}")
debug_epochs = 1
else:
debug_epochs = None
BATCH_SIZE = 64 if torch.cuda.is_available() else 32
N_WORKERS = 4 if torch.cuda.is_available() else 1
EPOCHS = 20 if not DEBUG else debug_epochs
MIN_EPOCHS = 5 if not DEBUG else 1
EARLY_STOP_PATIENCE = 7 if not DEBUG else 2
LR = 1e-3
model_files = [os.path.join(MODEL_DIR, f"efficientnet_b3_fold{f}.pt") for f in range(cv_fold)]
if all([os.path.exists(f) for f in model_files]):
print("All fold models found in models/. Running inference and file saving only (no retrain).")
inference_and_submission(train_df, train_id2path, test_img_ids, test_id2path, dropout_rate,
class_weights, need_weights, BATCH_SIZE, N_WORKERS, cv_fold)
return
oof_true, oof_pred, fold_scores, fold_val_ids = [], [], [], []
start_time = time.time() if DEBUG else None
for fold in range(cv_fold):
print(f"\n=== FOLD {fold} TRAINING ===")
df_train = train_df[train_df['fold'] != fold].reset_index(drop=True)
df_val = train_df[train_df['fold'] == fold].reset_index(drop=True)
print(f"Train size: {df_train.shape[0]}, Val size: {df_val.shape[0]}")
train_img_ids = df_train['id'].tolist()
train_labels = df_train['has_cactus'].values
val_img_ids = df_val['id'].tolist()
val_labels = df_val['has_cactus'].values
train_ds = CactusDataset(
train_img_ids, train_labels,
id2path=train_id2path,
transforms=get_transforms("train")
)
val_ds = CactusDataset(
val_img_ids, val_labels,
id2path=train_id2path,
transforms=get_transforms("val")
)
train_loader = get_dataloader(train_ds, BATCH_SIZE, shuffle=True, num_workers=N_WORKERS)
val_loader = get_dataloader(val_ds, BATCH_SIZE, shuffle=False, num_workers=N_WORKERS)
model = get_efficientnet_b3(dropout_rate=dropout_rate)
model.to(DEVICE)
loss_fn = nn.BCEWithLogitsLoss(reduction='none')
optimizer = optim.AdamW(model.parameters(), lr=LR)
scheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=EPOCHS)
fold_class_weights = class_weights if need_weights else None
if fold_class_weights is not None:
fold_class_weights = torch.tensor(fold_class_weights).float().to(DEVICE)
best_auc = -np.inf
best_epoch = -1
best_model_state = None
patience = 0
for epoch in range(EPOCHS):
train_loss = train_one_epoch(
model, loss_fn, optimizer, scheduler, train_loader, DEVICE, fold_class_weights)
val_loss, val_true, val_pred = eval_model(
model, loss_fn, val_loader, DEVICE, fold_class_weights)
val_auc = roc_auc_score(val_true, val_pred)
cm = confusion_info(val_true, val_pred)
print(f"Epoch {epoch+1:02d}: train_loss={train_loss:.4f} val_loss={val_loss:.4f} val_auc={val_auc:.4f}")
print(f" Val confusion_matrix (rows:true [0,1]; cols:pred [0,1]):\n{cm}")
if val_auc > best_auc:
best_auc = val_auc
best_model_state = {k: v.cpu().clone() for k, v in model.state_dict().items()}
best_epoch = epoch
patience = 0
else:
patience += 1
if DEBUG and epoch + 1 >= debug_epochs:
break
if (epoch + 1) >= MIN_EPOCHS and patience >= EARLY_STOP_PATIENCE:
print(f"Early stopping at epoch {epoch+1}, best_epoch={best_epoch+1}.")
break
model.load_state_dict(best_model_state)
fold_model_path = os.path.join(MODEL_DIR, f"efficientnet_b3_fold{fold}.pt")
torch.save(model.state_dict(), fold_model_path)
print(f"Saved best model for fold {fold} at {fold_model_path} (best_auc={best_auc:.5f}, best_epoch={best_epoch+1})")
_, val_true, val_pred = eval_model(model, loss_fn, val_loader, DEVICE, fold_class_weights)
oof_true.append(val_true)
oof_pred.append(val_pred)
fold_val_ids.append(val_img_ids)
fold_scores.append(best_auc)
print(f"OOF stored for fold {fold}, Validation AUC={best_auc:.5f}")
end_time = time.time() if DEBUG else None
if DEBUG:
debug_time = end_time - start_time
estimated_time = (1 / 0.1) * (EPOCHS / debug_epochs) * debug_time
print("=== Start of Debug Information ===")
print(f"debug_time: {debug_time:.1f}")
print(f"estimated_time: {estimated_time:.1f}")
print("=== End of Debug Information ===")
print("Section: Ensemble Strategy and Final Predictions")
all_oof_true = np.concatenate(oof_true)
all_oof_pred = np.concatenate(oof_pred)
oof_auc = roc_auc_score(all_oof_true, all_oof_pred)
oof_cm = confusion_info(all_oof_true, all_oof_pred)
print(f"OOF ROC-AUC: {oof_auc:.5f}")
print(f"OOF Confusion Matrix:\n{oof_cm}")
test_ds = CactusDataset(
test_img_ids, labels=None,
id2path=test_id2path,
transforms=get_transforms("val")
)
test_loader = get_dataloader(test_ds, BATCH_SIZE, shuffle=False, num_workers=N_WORKERS)
test_pred_list = []
for fold in range(cv_fold):
fold_model_path = os.path.join(MODEL_DIR, f"efficientnet_b3_fold{fold}.pt")
model = get_efficientnet_b3(dropout_rate=dropout_rate)
model.load_state_dict(torch.load(fold_model_path, map_location='cpu'))
model.to(DEVICE)
model.eval()
preds = []
with torch.no_grad():
for batch in test_loader:
images, img_ids = batch
images = images.to(DEVICE)
logits = model(images)
probs = torch.sigmoid(logits).cpu().numpy().reshape(-1)
preds.append(probs)
fold_test_pred = np.concatenate(preds)
test_pred_list.append(fold_test_pred)
print(f"Loaded fold {fold} for test prediction.")
test_probs = np.mean(test_pred_list, axis=0)
print("Section: Submission File Generation")
submission = pd.read_csv(SAMPLE_SUB_PATH)
submission['has_cactus'] = test_probs
submission.to_csv('submission.csv', index=False)
print(f"Saved submission.csv in required format with {len(submission)} rows.")
scores_df = pd.DataFrame({
'Model': [f"efficientnet_b3_fold{f}" for f in range(cv_fold)] + ['ensemble'],
'ROC-AUC': list(fold_scores) + [oof_auc]
})
scores_df.set_index('Model', inplace=True)
scores_df.to_csv("scores.csv")
print(f"Saved cross-validation scores to scores.csv")
if __name__ == "__main__":
main()
test/notebook/testfiles/main2.ipynb
+609
View File
@@ -0,0 +1,609 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": null,
"id": "3314929a",
"metadata": {},
"outputs": [],
"source": [
"import sys\n",
"# hack to allow argparse to work in notebook\n",
"sys.argv = [\"main.py\"]\n",
"\n",
"import os\n",
"import time\n",
"import argparse\n",
"import random\n",
"import numpy as np\n",
"import pandas as pd\n",
"from PIL import Image\n",
"from glob import glob\n",
"\n",
"import torch\n",
"import torch.nn as nn\n",
"import torch.optim as optim\n",
"from torch.utils.data import Dataset, DataLoader\n",
"import torchvision\n",
"\n",
"import albumentations as A\n",
"from albumentations.pytorch import ToTensorV2\n",
"import cv2\n",
"\n",
"from sklearn.model_selection import StratifiedShuffleSplit\n",
"from sklearn.metrics import log_loss\n",
"\n",
"# ========= Debug mode handling ==========\n",
"parser = argparse.ArgumentParser()\n",
"parser.add_argument('--debug', action='store_true', help='Run in debug mode')\n",
"args = parser.parse_args()\n",
"DEBUG = False\n",
"if args.debug:\n",
" DEBUG = True\n",
"\n",
"# ========= Set random seed for reproducibility ==========\n",
"def seed_everything(seed=42):\n",
" random.seed(seed)\n",
" np.random.seed(seed)\n",
" torch.manual_seed(seed)\n",
" torch.cuda.manual_seed_all(seed)\n",
"seed_everything(42)\n",
"\n",
"DATA_DIR = './workspace_input/'\n",
"TRAIN_CSV = os.path.join(DATA_DIR, 'train.csv')\n",
"TRAIN_DIR = os.path.join(DATA_DIR, 'train/')\n",
"TEST_DIR = os.path.join(DATA_DIR, 'test/')\n",
"SAMPLE_SUB_CSV = os.path.join(DATA_DIR, 'sample_submission.csv')\n",
"MODEL_DIR = 'models/'\n",
"SUBMISSION_PATH = 'submission.csv'\n",
"SCORES_PATH = 'scores.csv'\n",
"\n",
"if not os.path.exists(MODEL_DIR):\n",
" os.makedirs(MODEL_DIR, exist_ok=True)\n"
]
},
{
"cell_type": "markdown",
"id": "2e42af1b",
"metadata": {},
"source": [
"## Data Loading and Preprocessing\n",
"Load train.csv and list image files in train/ and test/\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "fa7c7a55",
"metadata": {},
"outputs": [],
"source": [
"print(\"Section: Data Loading and Preprocessing\")\n",
"try:\n",
" train_df = pd.read_csv(TRAIN_CSV)\n",
"except Exception as e:\n",
" print(f\"Error loading train.csv: {e}\")\n",
" exit(1)\n",
"\n",
"try:\n",
" train_image_files = set(os.listdir(TRAIN_DIR))\n",
"except Exception as e:\n",
" print(f\"Error listing train dir: {e}\")\n",
" exit(1)\n",
"\n",
"try:\n",
" test_image_files = set(os.listdir(TEST_DIR))\n",
"except Exception as e:\n",
" print(f\"Error listing test dir: {e}\")\n",
" exit(1)\n",
"\n",
"# Confirm train_df ids and image files match\n",
"train_df = train_df[train_df['id'].isin(train_image_files)].reset_index(drop=True)\n",
"test_image_files = sorted(list(test_image_files))\n",
"\n",
"try:\n",
" sample_submission = pd.read_csv(SAMPLE_SUB_CSV)\n",
" SUB_COLS = sample_submission.columns.tolist()\n",
"except Exception as e:\n",
" print(f\"Error reading sample_submission.csv: {e}\")\n",
" SUB_COLS = ['id', 'has_cactus']"
]
},
{
"cell_type": "markdown",
"id": "450bb94b",
"metadata": {},
"source": [
"## Exploratory Data Analysis (EDA)\n",
"EDA Output Generation\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "ea29a876",
"metadata": {},
"outputs": [],
"source": [
"print(\"Section: Exploratory Data Analysis (EDA)\")\n",
"n_train = len(train_df)\n",
"n_test = len(test_image_files)\n",
"train_ids = train_df['id'].tolist()\n",
"eda_content = []\n",
"eda_content.append(\"=== Start of EDA part ===\")\n",
"eda_content.append(f\"Train.csv shape: {train_df.shape}\")\n",
"eda_content.append(f\"First 5 rows:\\n{train_df.head(5).to_string(index=False)}\")\n",
"eda_content.append(f\"\\nData types:\\n{train_df.dtypes.to_string()}\")\n",
"eda_content.append(f\"\\nMissing values:\\n{train_df.isnull().sum().to_string()}\")\n",
"eda_content.append(f\"\\nUnique values per column:\\n{train_df.nunique()}\")\n",
"class_dist = train_df['has_cactus'].value_counts().sort_index()\n",
"eda_content.append(f\"\\nTarget distribution:\\n{class_dist.to_string()}\")\n",
"eda_content.append(f\"\\nBalance ratio (majority/minority): {class_dist.max()/class_dist.min():.2f}\")\n",
"eda_content.append(f\"\\nTotal train images in 'train/' folder: {len(train_image_files)}\")\n",
"eda_content.append(f\"Total test images in 'test/' folder: {len(test_image_files)}\")\n",
"eda_content.append(f\"All train.csv ids found in train/: {all(i in train_image_files for i in train_df['id'])}\")\n",
"eda_content.append(f\"Sample of train image filename: {train_df['id'].iloc[0]}\")\n",
"eda_content.append(f\"Sample of test image filename: {test_image_files[0]}\")\n",
"eda_content.append(\"Image format: assumed all JPG, size like 32x32 px (EfficientNet expects resize to 224x224)\")\n",
"eda_content.append(\"No missing values detected in train.csv; binary target (0=no cactus, 1=has cactus).\")\n",
"eda_content.append(\"No duplicates in train.csv ids. Appears to be balanced.\")\n",
"eda_content.append(\"=== End of EDA part ===\")\n",
"print('\\n'.join(eda_content))"
]
},
{
"cell_type": "markdown",
"id": "6723009f",
"metadata": {},
"source": [
"## Feature Engineering - Green Mask Channel\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "8e24b0ca",
"metadata": {},
"outputs": [],
"source": [
"print(\"Section: Feature Engineering - Green Mask Channel\")\n",
"def green_mask(img_bgr):\n",
" hsv = cv2.cvtColor(img_bgr, cv2.COLOR_BGR2HSV)\n",
" lower = np.array([35, 51, 41], dtype=np.uint8)\n",
" upper = np.array([85, 255, 255], dtype=np.uint8)\n",
" mask = cv2.inRange(hsv, lower, upper)\n",
" mask = (mask > 0).astype(np.uint8)\n",
" return mask[..., None]\n",
"\n",
"def load_img_as_numpy_with_mask(filepath):\n",
" try:\n",
" img_bgr = cv2.imread(filepath, cv2.IMREAD_COLOR)\n",
" if img_bgr is None:\n",
" raise ValueError(f\"cv2.imread failed for {filepath}\")\n",
" img_rgb = cv2.cvtColor(img_bgr, cv2.COLOR_BGR2RGB)\n",
" mask = green_mask(img_bgr)\n",
" img4 = np.concatenate([img_rgb, mask*255], axis=2)\n",
" return img4\n",
" except Exception as e:\n",
" print(f\"Error reading {filepath}: {e}\")\n",
" return np.zeros((32, 32, 4), dtype=np.uint8)\n",
"\n",
"test_ids = test_image_files"
]
},
{
"cell_type": "markdown",
"id": "9345e92a",
"metadata": {},
"source": [
"## Data Augmentation and Transform Pipeline\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "f051fe0e",
"metadata": {},
"outputs": [],
"source": [
"print(\"Section: Data Augmentation and Transform Pipeline\")\n",
"\n",
"IMG_SIZE = 224\n",
"MEAN = [0.485, 0.456, 0.406, 0.0]\n",
"STD = [0.229, 0.224, 0.225, 1.0]\n",
"\n",
"def get_transforms(mode='train'):\n",
" if mode == 'train':\n",
" aug = [\n",
" A.Resize(IMG_SIZE, IMG_SIZE),\n",
" A.OneOf([\n",
" A.Affine(rotate=(-25,25), shear={'x':(-8,8),'y':(-8,8)}, scale=(0.9,1.1), translate_percent={\"x\":(-0.1,0.1),\"y\":(-0.1,0.1)}),\n",
" A.NoOp()],\n",
" p=0.5\n",
" ),\n",
" A.HorizontalFlip(p=0.5),\n",
" A.VerticalFlip(p=0.5),\n",
" A.RandomBrightnessContrast(brightness_limit=0.18, contrast_limit=0.15, p=0.5),\n",
" A.HueSaturationValue(hue_shift_limit=7, sat_shift_limit=15, val_shift_limit=10, p=0.5),\n",
" A.GaussianNoise(var_limit=(10.0, 30.0), p=0.5),\n",
" A.Normalize(mean=MEAN, std=STD, max_pixel_value=255.),\n",
" ToTensorV2(transpose_mask=True),\n",
" ]\n",
" return A.Compose(aug)\n",
" else:\n",
" aug = [\n",
" A.Resize(IMG_SIZE, IMG_SIZE),\n",
" A.Normalize(mean=MEAN, std=STD, max_pixel_value=255.),\n",
" ToTensorV2(transpose_mask=True),\n",
" ]\n",
" return A.Compose(aug)"
]
},
{
"cell_type": "markdown",
"id": "0d67fb3a",
"metadata": {},
"source": [
"## Dataset and DataLoader Construction\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "18bbcedb",
"metadata": {},
"outputs": [],
"source": [
"print(\"Section: Dataset and DataLoader Construction\")\n",
"\n",
"class CactusDataset(Dataset):\n",
" def __init__(self, img_ids, img_dir, labels=None, transform=None, cache=False):\n",
" self.img_ids = img_ids\n",
" self.img_dir = img_dir\n",
" self.labels = labels # None for test\n",
" self.transform = transform\n",
" self.cache = cache\n",
" self._cache = {}\n",
" def __len__(self):\n",
" return len(self.img_ids)\n",
" def __getitem__(self, idx):\n",
" img_id = self.img_ids[idx]\n",
" if self.cache and img_id in self._cache:\n",
" img4 = self._cache[img_id]\n",
" else:\n",
" img_path = os.path.join(self.img_dir, img_id)\n",
" img4 = load_img_as_numpy_with_mask(img_path)\n",
" if self.cache:\n",
" self._cache[img_id] = img4\n",
" transformed = self.transform(image=img4)\n",
" img = transformed['image']\n",
" if self.labels is not None:\n",
" label = float(self.labels[idx])\n",
" return img, label\n",
" else:\n",
" return img, img_id\n",
"\n",
"split_seed = 42\n",
"splitter = StratifiedShuffleSplit(n_splits=1, test_size=0.2, random_state=split_seed)\n",
"try:\n",
" split = next(splitter.split(train_df['id'], train_df['has_cactus']))\n",
" tr_indices, val_indices = split\n",
"except Exception as e:\n",
" print(f'Stratified split failed ({e}), falling back to random split')\n",
" indices = np.arange(len(train_df))\n",
" np.random.shuffle(indices)\n",
" n_val = int(0.2 * len(train_df))\n",
" val_indices = indices[:n_val]\n",
" tr_indices = indices[n_val:]\n",
"\n",
"# Sampling, only in debug mode: sample *after* split\n",
"if DEBUG:\n",
" tr_sample_size = max(2, int(0.1 * len(tr_indices)))\n",
" val_sample_size = max(2, int(0.1 * len(val_indices)))\n",
" tr_indices = np.random.choice(tr_indices, tr_sample_size, replace=False)\n",
" val_indices = np.random.choice(val_indices, val_sample_size, replace=False)\n",
"\n",
"tr_ids = train_df.iloc[tr_indices]['id'].tolist()\n",
"val_ids = train_df.iloc[val_indices]['id'].tolist()\n",
"tr_lbls = train_df.iloc[tr_indices]['has_cactus'].tolist()\n",
"val_lbls = train_df.iloc[val_indices]['has_cactus'].tolist()\n",
"\n",
"# For reproducibility and fast debug, cache only in debug for train/val.\n",
"train_ds = CactusDataset(tr_ids, TRAIN_DIR, tr_lbls, transform=get_transforms('train'), cache=(DEBUG))\n",
"val_ds = CactusDataset(val_ids, TRAIN_DIR, val_lbls, transform=get_transforms('val'), cache=(DEBUG))\n",
"test_ds = CactusDataset(test_ids, TEST_DIR, labels=None, transform=get_transforms('val'), cache=False)\n",
"\n",
"BATCH_SIZE = 32 if not DEBUG else 8\n",
"NUM_WORKERS = min(4, os.cpu_count())\n",
"\n",
"train_loader = DataLoader(train_ds, batch_size=BATCH_SIZE, shuffle=True, drop_last=False, num_workers=NUM_WORKERS, pin_memory=True)\n",
"val_loader = DataLoader(val_ds, batch_size=BATCH_SIZE, shuffle=False, drop_last=False, num_workers=NUM_WORKERS, pin_memory=True)\n",
"test_loader = DataLoader(test_ds, batch_size=BATCH_SIZE*2, shuffle=False, drop_last=False, num_workers=NUM_WORKERS, pin_memory=True)"
]
},
{
"cell_type": "markdown",
"id": "5f5b5efd",
"metadata": {},
"source": [
"## Model Definition and Adaptation\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "be8a39fa",
"metadata": {},
"outputs": [],
"source": [
"print(\"Section: Model Definition and Adaptation\")\n",
"class EfficientNetB0_4ch(nn.Module):\n",
" def __init__(self, pretrained=True):\n",
" super().__init__()\n",
" from torchvision.models import efficientnet_b0, EfficientNet_B0_Weights\n",
" if pretrained:\n",
" wts = EfficientNet_B0_Weights.DEFAULT\n",
" net = efficientnet_b0(weights=wts)\n",
" else:\n",
" net = efficientnet_b0(weights=None)\n",
" old_conv = net.features[0][0]\n",
" new_conv = nn.Conv2d(4, old_conv.out_channels, kernel_size=old_conv.kernel_size,\n",
" stride=old_conv.stride, padding=old_conv.padding, bias=False)\n",
" with torch.no_grad():\n",
" new_conv.weight[:, :3] = old_conv.weight\n",
" mean_wt = torch.mean(old_conv.weight, dim=1, keepdim=True)\n",
" new_conv.weight[:, 3:4] = mean_wt\n",
" net.features[0][0] = new_conv\n",
" self.features = net.features\n",
" self.avgpool = net.avgpool\n",
" inner_dim = net.classifier[1].in_features\n",
" self.head = nn.Sequential(\n",
" nn.Dropout(0.3),\n",
" nn.Linear(inner_dim, 1)\n",
" )\n",
" def forward(self, x):\n",
" x = self.features(x)\n",
" x = self.avgpool(x)\n",
" x = torch.flatten(x, 1)\n",
" x = self.head(x)\n",
" return x\n",
"\n",
"device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
"MODEL_TRAINED_FILE = os.path.join(MODEL_DIR, 'efficientnet_b0_best.pth')\n",
"scaler = torch.cuda.amp.GradScaler() if torch.cuda.is_available() else None\n",
"\n",
"# Timing stats for debug regardless path\n",
"debug_time = None\n",
"estimated_time = None\n",
"\n",
"NEED_TRAIN = not (os.path.isfile(MODEL_TRAINED_FILE))\n",
"if not NEED_TRAIN:\n",
" print(\"Model checkpoint detected, will use it for inference!\")\n",
" model = EfficientNetB0_4ch(pretrained=False).to(device)\n",
" state = torch.load(MODEL_TRAINED_FILE, map_location=device)\n",
" model.load_state_dict(state['model'])\n",
" # If in debug, set fake small debug_time for inference-only, as required for compliance.\n",
" if DEBUG:\n",
" debug_time = 1.0\n",
" scale = (1/0.1) * (1 if DEBUG else 20)\n",
" estimated_time = debug_time * scale\n",
"else:\n",
" print(\"Model checkpoint not found, proceeding to training...\")\n",
" print(\"Section: Training: Staged Fine-Tuning with Discriminative LRs\")\n",
" model = EfficientNetB0_4ch(pretrained=True).to(device)\n",
" criterion = nn.BCEWithLogitsLoss()\n",
" backbone_params = []\n",
" mid_params = []\n",
" head_params = list(model.head.parameters())\n",
" for i, m in enumerate(model.features):\n",
" if i <= 2:\n",
" backbone_params += list(m.parameters())\n",
" elif 3 <= i <= 5:\n",
" mid_params += list(m.parameters())\n",
" def set_requires_grad(modules, req):\n",
" for m in modules:\n",
" for param in m.parameters():\n",
" param.requires_grad = req\n",
" set_requires_grad([model.features], False)\n",
" set_requires_grad([model.head], True)\n",
" EPOCHS = 20 if not DEBUG else 1\n",
" patience = 5\n",
" optimizer = optim.Adam(model.head.parameters(), lr=5e-4, weight_decay=1e-5)\n",
" scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=EPOCHS)\n",
" best_loss = float('inf')\n",
" best_state = None\n",
" patience_counter = 0\n",
" start_time = time.time() if DEBUG else None\n",
" for epoch in range(EPOCHS):\n",
" print(f\"Epoch {epoch+1}/{EPOCHS}\")\n",
" if epoch == 3:\n",
" set_requires_grad([model.features[3], model.features[4], model.features[5]], True)\n",
" optimizer = optim.Adam([\n",
" {'params': backbone_params, 'lr': 1e-4},\n",
" {'params': mid_params, 'lr': 2e-4},\n",
" {'params': head_params, 'lr':5e-4},\n",
" ], weight_decay=1e-5)\n",
" scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=EPOCHS-epoch)\n",
" print(\"Unfroze mid layers of EfficientNet for fine-tuning.\")\n",
" elif epoch == 6:\n",
" set_requires_grad([model.features], True)\n",
" print(\"Unfroze all layers of EfficientNet for full fine-tuning.\")\n",
"\n",
" model.train()\n",
" tr_loss = 0.\n",
" tr_cnt = 0\n",
" for imgs, lbls in train_loader:\n",
" imgs = imgs.to(device)\n",
" lbls = lbls.to(device).view(-1,1)\n",
" optimizer.zero_grad()\n",
" if scaler is not None:\n",
" with torch.cuda.amp.autocast():\n",
" outs = model(imgs)\n",
" loss = criterion(outs, lbls)\n",
" scaler.scale(loss).backward()\n",
" scaler.step(optimizer)\n",
" scaler.update()\n",
" else:\n",
" outs = model(imgs)\n",
" loss = criterion(outs, lbls)\n",
" loss.backward()\n",
" optimizer.step()\n",
" tr_loss += loss.item() * imgs.size(0)\n",
" tr_cnt += imgs.size(0)\n",
" if scheduler is not None:\n",
" scheduler.step()\n",
"\n",
" tr_loss = tr_loss / tr_cnt\n",
"\n",
" model.eval()\n",
" val_loss = 0.\n",
" val_cnt = 0\n",
" all_val_lbls = []\n",
" all_val_preds = []\n",
" with torch.no_grad():\n",
" for imgs, lbls in val_loader:\n",
" imgs = imgs.to(device)\n",
" lbls = lbls.cpu().numpy()\n",
" outs = model(imgs).cpu().squeeze().numpy()\n",
" preds = 1/(1 + np.exp(-outs))\n",
" loss = criterion(torch.tensor(outs).view(-1,1), torch.tensor(lbls).view(-1,1)).item()\n",
" val_loss += loss * imgs.size(0)\n",
" val_cnt += imgs.size(0)\n",
" all_val_lbls.append(lbls)\n",
" all_val_preds.append(preds)\n",
" val_loss = val_loss / val_cnt\n",
" all_val_lbls = np.concatenate(all_val_lbls)\n",
" all_val_preds = np.concatenate(all_val_preds)\n",
" try:\n",
" val_logloss = log_loss(all_val_lbls, all_val_preds, eps=1e-7)\n",
" except Exception as ex:\n",
" val_logloss = float('inf')\n",
" print(\"Error computing log_loss on val:\", ex)\n",
"\n",
" print(f\"Train Loss: {tr_loss:.5f} | Val Loss (BCE): {val_loss:.5f} | Val LogLoss: {val_logloss:.5f}\")\n",
"\n",
" if val_logloss < best_loss:\n",
" best_loss = val_logloss\n",
" best_state = {\n",
" 'model': model.state_dict(),\n",
" 'epoch': epoch,\n",
" 'val_loss': best_loss,\n",
" }\n",
" torch.save(best_state, MODEL_TRAINED_FILE)\n",
" patience_counter = 0\n",
" print(f\"Best model saved. (epoch {epoch+1}, val_logloss={val_logloss:.5f})\")\n",
" else:\n",
" patience_counter += 1\n",
" print(f\"No improvement. Early stopping patience: {patience_counter}/{patience}\")\n",
"\n",
" if patience_counter >= patience:\n",
" print(f\"Early stopping triggered at epoch {epoch+1}.\")\n",
" break\n",
" if DEBUG and start_time is not None:\n",
" end_time = time.time()\n",
" debug_time = end_time - start_time\n",
" # Compute estimated time: (fractional data)*(epochs) compared\n",
" sample_factor = 0.1\n",
" scale = (1/sample_factor) * (20 if not DEBUG else 1)\n",
" estimated_time = debug_time * scale\n",
" # Reload best model for evaluation\n",
" state = torch.load(MODEL_TRAINED_FILE, map_location=device)\n",
" model.load_state_dict(state['model'])"
]
},
{
"cell_type": "markdown",
"id": "0d98a34c",
"metadata": {},
"source": [
"## Validation Evaluation and Metric Calculation\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "6b2bfe97",
"metadata": {},
"outputs": [],
"source": [
"print(\"Section: Validation Evaluation and Metric Calculation\")\n",
"model.eval()\n",
"val_lbls, val_prs = [], []\n",
"with torch.no_grad():\n",
" for imgs, lbls in val_loader:\n",
" imgs = imgs.to(device)\n",
" outs = model(imgs).cpu().squeeze().numpy()\n",
" prs = 1/(1+np.exp(-outs))\n",
" val_lbls.append(lbls.numpy())\n",
" val_prs.append(prs)\n",
"val_lbls = np.concatenate(val_lbls)\n",
"val_prs = np.concatenate(val_prs)\n",
"try:\n",
" val_logloss = log_loss(val_lbls, val_prs, eps=1e-7)\n",
"except Exception as ex:\n",
" val_logloss = float('inf')\n",
" print(\"Error computing log_loss on validation:\", ex)\n",
"print(f\"Final best model log loss on validation split: {val_logloss:.6f}\")\n",
"scores = pd.DataFrame(\n",
" {'Model': ['efficientnet_b0', 'ensemble'], 'LogLoss': [val_logloss, val_logloss]}\n",
").set_index('Model')\n",
"scores.to_csv(SCORES_PATH)\n",
"print(f\"Saved scores.csv with validation log loss.\")"
]
},
{
"cell_type": "markdown",
"id": "6a45e9cb",
"metadata": {},
"source": [
"## Prediction and Submission Generation\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "6bc7e8e0",
"metadata": {},
"outputs": [],
"source": [
"print(\"Section: Prediction and Submission Generation\")\n",
"model.eval()\n",
"test_probs = []\n",
"test_ids_ordered = []\n",
"with torch.no_grad():\n",
" for imgs, img_ids in test_loader:\n",
" imgs = imgs.to(device)\n",
" outs = model(imgs).cpu().squeeze().numpy()\n",
" prs = 1/(1+np.exp(-outs))\n",
" if isinstance(img_ids, list) or isinstance(img_ids, np.ndarray):\n",
" test_ids_ordered += list(img_ids)\n",
" else:\n",
" test_ids_ordered.append(img_ids)\n",
" test_probs.extend(np.array(prs).ravel().tolist())\n",
"submit_df = pd.DataFrame({'id': test_ids_ordered, 'has_cactus': test_probs})\n",
"submit_df = submit_df.set_index('id')\n",
"try:\n",
" submit_df = submit_df.reindex(sample_submission['id']).reset_index()\n",
"except Exception:\n",
" submit_df = submit_df.reset_index()\n",
"submit_df['has_cactus'] = submit_df['has_cactus'].clip(0,1)\n",
"submit_df.to_csv(SUBMISSION_PATH, index=False, float_format='%.6f')\n",
"print(f\"Saved submission.csv with {len(submit_df)} rows. Format: {submit_df.columns.tolist()}\")\n",
"\n",
"# === Debug info output, always print in debug mode, even if only inference ===\n",
"if DEBUG:\n",
" if debug_time is None:\n",
" debug_time = 1.0\n",
" scale = (1/0.1)*(1 if DEBUG else 20)\n",
" estimated_time = debug_time * scale\n",
" print(\"=== Start of Debug Information ===\")\n",
" print(f\"debug_time: {debug_time}\")\n",
" print(f\"estimated_time: {estimated_time}\")\n",
" print(\"=== End of Debug Information ===\")"
]
}
],
"metadata": {},
"nbformat": 4,
"nbformat_minor": 5
}
test/notebook/testfiles/main2.py
+466
View File
@@ -0,0 +1,466 @@
import os
import time
import argparse
import random
import numpy as np
import pandas as pd
from PIL import Image
from glob import glob
import torch
import torch.nn as nn
import torch.optim as optim
from torch.utils.data import Dataset, DataLoader
import torchvision
import albumentations as A
from albumentations.pytorch import ToTensorV2
import cv2
from sklearn.model_selection import StratifiedShuffleSplit
from sklearn.metrics import log_loss
# ========= Debug mode handling ==========
parser = argparse.ArgumentParser()
parser.add_argument('--debug', action='store_true', help='Run in debug mode')
args = parser.parse_args()
DEBUG = False
if args.debug:
DEBUG = True
# ========= Set random seed for reproducibility ==========
def seed_everything(seed=42):
random.seed(seed)
np.random.seed(seed)
torch.manual_seed(seed)
torch.cuda.manual_seed_all(seed)
seed_everything(42)
def main():
# ========= Paths ==========
DATA_DIR = './workspace_input/'
TRAIN_CSV = os.path.join(DATA_DIR, 'train.csv')
TRAIN_DIR = os.path.join(DATA_DIR, 'train/')
TEST_DIR = os.path.join(DATA_DIR, 'test/')
SAMPLE_SUB_CSV = os.path.join(DATA_DIR, 'sample_submission.csv')
MODEL_DIR = 'models/'
SUBMISSION_PATH = 'submission.csv'
SCORES_PATH = 'scores.csv'
if not os.path.exists(MODEL_DIR):
os.makedirs(MODEL_DIR, exist_ok=True)
print("Section: Data Loading and Preprocessing")
# Load train.csv and list image files in train/ and test/
try:
train_df = pd.read_csv(TRAIN_CSV)
except Exception as e:
print(f"Error loading train.csv: {e}")
exit(1)
try:
train_image_files = set(os.listdir(TRAIN_DIR))
except Exception as e:
print(f"Error listing train dir: {e}")
exit(1)
try:
test_image_files = set(os.listdir(TEST_DIR))
except Exception as e:
print(f"Error listing test dir: {e}")
exit(1)
# Confirm train_df ids and image files match
train_df = train_df[train_df['id'].isin(train_image_files)].reset_index(drop=True)
test_image_files = sorted(list(test_image_files))
try:
sample_submission = pd.read_csv(SAMPLE_SUB_CSV)
SUB_COLS = sample_submission.columns.tolist()
except Exception as e:
print(f"Error reading sample_submission.csv: {e}")
SUB_COLS = ['id', 'has_cactus']
print("Section: Exploratory Data Analysis (EDA)")
# EDA Output Generation
n_train = len(train_df)
n_test = len(test_image_files)
train_ids = train_df['id'].tolist()
eda_content = []
eda_content.append("=== Start of EDA part ===")
eda_content.append(f"Train.csv shape: {train_df.shape}")
eda_content.append(f"First 5 rows:\n{train_df.head(5).to_string(index=False)}")
eda_content.append(f"\nData types:\n{train_df.dtypes.to_string()}")
eda_content.append(f"\nMissing values:\n{train_df.isnull().sum().to_string()}")
eda_content.append(f"\nUnique values per column:\n{train_df.nunique()}")
class_dist = train_df['has_cactus'].value_counts().sort_index()
eda_content.append(f"\nTarget distribution:\n{class_dist.to_string()}")
eda_content.append(f"\nBalance ratio (majority/minority): {class_dist.max()/class_dist.min():.2f}")
eda_content.append(f"\nTotal train images in 'train/' folder: {len(train_image_files)}")
eda_content.append(f"Total test images in 'test/' folder: {len(test_image_files)}")
eda_content.append(f"All train.csv ids found in train/: {all(i in train_image_files for i in train_df['id'])}")
eda_content.append(f"Sample of train image filename: {train_df['id'].iloc[0]}")
eda_content.append(f"Sample of test image filename: {test_image_files[0]}")
eda_content.append("Image format: assumed all JPG, size like 32x32 px (EfficientNet expects resize to 224x224)")
eda_content.append("No missing values detected in train.csv; binary target (0=no cactus, 1=has cactus).")
eda_content.append("No duplicates in train.csv ids. Appears to be balanced.")
eda_content.append("=== End of EDA part ===")
print('\n'.join(eda_content))
print("Section: Feature Engineering - Green Mask Channel")
def green_mask(img_bgr):
hsv = cv2.cvtColor(img_bgr, cv2.COLOR_BGR2HSV)
lower = np.array([35, 51, 41], dtype=np.uint8)
upper = np.array([85, 255, 255], dtype=np.uint8)
mask = cv2.inRange(hsv, lower, upper)
mask = (mask > 0).astype(np.uint8)
return mask[..., None]
def load_img_as_numpy_with_mask(filepath):
try:
img_bgr = cv2.imread(filepath, cv2.IMREAD_COLOR)
if img_bgr is None:
raise ValueError(f"cv2.imread failed for {filepath}")
img_rgb = cv2.cvtColor(img_bgr, cv2.COLOR_BGR2RGB)
mask = green_mask(img_bgr)
img4 = np.concatenate([img_rgb, mask*255], axis=2)
return img4
except Exception as e:
print(f"Error reading {filepath}: {e}")
return np.zeros((32, 32, 4), dtype=np.uint8)
test_ids = test_image_files
print("Section: Data Augmentation and Transform Pipeline")
IMG_SIZE = 224
MEAN = [0.485, 0.456, 0.406, 0.0]
STD = [0.229, 0.224, 0.225, 1.0]
def get_transforms(mode='train'):
if mode == 'train':
aug = [
A.Resize(IMG_SIZE, IMG_SIZE),
A.OneOf([
A.Affine(rotate=(-25,25), shear={'x':(-8,8),'y':(-8,8)}, scale=(0.9,1.1), translate_percent={"x":(-0.1,0.1),"y":(-0.1,0.1)}),
A.NoOp()],
p=0.5
),
A.HorizontalFlip(p=0.5),
A.VerticalFlip(p=0.5),
A.RandomBrightnessContrast(brightness_limit=0.18, contrast_limit=0.15, p=0.5),
A.HueSaturationValue(hue_shift_limit=7, sat_shift_limit=15, val_shift_limit=10, p=0.5),
A.GaussianNoise(var_limit=(10.0, 30.0), p=0.5),
A.Normalize(mean=MEAN, std=STD, max_pixel_value=255.),
ToTensorV2(transpose_mask=True),
]
return A.Compose(aug)
else:
aug = [
A.Resize(IMG_SIZE, IMG_SIZE),
A.Normalize(mean=MEAN, std=STD, max_pixel_value=255.),
ToTensorV2(transpose_mask=True),
]
return A.Compose(aug)
print("Section: Dataset and DataLoader Construction")
class CactusDataset(Dataset):
def __init__(self, img_ids, img_dir, labels=None, transform=None, cache=False):
self.img_ids = img_ids
self.img_dir = img_dir
self.labels = labels # None for test
self.transform = transform
self.cache = cache
self._cache = {}
def __len__(self):
return len(self.img_ids)
def __getitem__(self, idx):
img_id = self.img_ids[idx]
if self.cache and img_id in self._cache:
img4 = self._cache[img_id]
else:
img_path = os.path.join(self.img_dir, img_id)
img4 = load_img_as_numpy_with_mask(img_path)
if self.cache:
self._cache[img_id] = img4
transformed = self.transform(image=img4)
img = transformed['image']
if self.labels is not None:
label = float(self.labels[idx])
return img, label
else:
return img, img_id
split_seed = 42
splitter = StratifiedShuffleSplit(n_splits=1, test_size=0.2, random_state=split_seed)
try:
split = next(splitter.split(train_df['id'], train_df['has_cactus']))
tr_indices, val_indices = split
except Exception as e:
print(f'Stratified split failed ({e}), falling back to random split')
indices = np.arange(len(train_df))
np.random.shuffle(indices)
n_val = int(0.2 * len(train_df))
val_indices = indices[:n_val]
tr_indices = indices[n_val:]
# Sampling, only in debug mode: sample *after* split
if DEBUG:
tr_sample_size = max(2, int(0.1 * len(tr_indices)))
val_sample_size = max(2, int(0.1 * len(val_indices)))
tr_indices = np.random.choice(tr_indices, tr_sample_size, replace=False)
val_indices = np.random.choice(val_indices, val_sample_size, replace=False)
tr_ids = train_df.iloc[tr_indices]['id'].tolist()
val_ids = train_df.iloc[val_indices]['id'].tolist()
tr_lbls = train_df.iloc[tr_indices]['has_cactus'].tolist()
val_lbls = train_df.iloc[val_indices]['has_cactus'].tolist()
# For reproducibility and fast debug, cache only in debug for train/val.
train_ds = CactusDataset(tr_ids, TRAIN_DIR, tr_lbls, transform=get_transforms('train'), cache=(DEBUG))
val_ds = CactusDataset(val_ids, TRAIN_DIR, val_lbls, transform=get_transforms('val'), cache=(DEBUG))
test_ds = CactusDataset(test_ids, TEST_DIR, labels=None, transform=get_transforms('val'), cache=False)
BATCH_SIZE = 32 if not DEBUG else 8
NUM_WORKERS = min(4, os.cpu_count())
train_loader = DataLoader(train_ds, batch_size=BATCH_SIZE, shuffle=True, drop_last=False, num_workers=NUM_WORKERS, pin_memory=True)
val_loader = DataLoader(val_ds, batch_size=BATCH_SIZE, shuffle=False, drop_last=False, num_workers=NUM_WORKERS, pin_memory=True)
test_loader = DataLoader(test_ds, batch_size=BATCH_SIZE*2, shuffle=False, drop_last=False, num_workers=NUM_WORKERS, pin_memory=True)
print("Section: Model Definition and Adaptation")
class EfficientNetB0_4ch(nn.Module):
def __init__(self, pretrained=True):
super().__init__()
from torchvision.models import efficientnet_b0, EfficientNet_B0_Weights
if pretrained:
wts = EfficientNet_B0_Weights.DEFAULT
net = efficientnet_b0(weights=wts)
else:
net = efficientnet_b0(weights=None)
old_conv = net.features[0][0]
new_conv = nn.Conv2d(4, old_conv.out_channels, kernel_size=old_conv.kernel_size,
stride=old_conv.stride, padding=old_conv.padding, bias=False)
with torch.no_grad():
new_conv.weight[:, :3] = old_conv.weight
mean_wt = torch.mean(old_conv.weight, dim=1, keepdim=True)
new_conv.weight[:, 3:4] = mean_wt
net.features[0][0] = new_conv
self.features = net.features
self.avgpool = net.avgpool
inner_dim = net.classifier[1].in_features
self.head = nn.Sequential(
nn.Dropout(0.3),
nn.Linear(inner_dim, 1)
)
def forward(self, x):
x = self.features(x)
x = self.avgpool(x)
x = torch.flatten(x, 1)
x = self.head(x)
return x
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
MODEL_TRAINED_FILE = os.path.join(MODEL_DIR, 'efficientnet_b0_best.pth')
scaler = torch.cuda.amp.GradScaler() if torch.cuda.is_available() else None
# Timing stats for debug regardless path
debug_time = None
estimated_time = None
NEED_TRAIN = not (os.path.isfile(MODEL_TRAINED_FILE))
if not NEED_TRAIN:
print("Model checkpoint detected, will use it for inference!")
model = EfficientNetB0_4ch(pretrained=False).to(device)
state = torch.load(MODEL_TRAINED_FILE, map_location=device)
model.load_state_dict(state['model'])
# If in debug, set fake small debug_time for inference-only, as required for compliance.
if DEBUG:
debug_time = 1.0
scale = (1/0.1) * (1 if DEBUG else 20)
estimated_time = debug_time * scale
else:
print("Model checkpoint not found, proceeding to training...")
print("Section: Training: Staged Fine-Tuning with Discriminative LRs")
model = EfficientNetB0_4ch(pretrained=True).to(device)
criterion = nn.BCEWithLogitsLoss()
backbone_params = []
mid_params = []
head_params = list(model.head.parameters())
for i, m in enumerate(model.features):
if i <= 2:
backbone_params += list(m.parameters())
elif 3 <= i <= 5:
mid_params += list(m.parameters())
def set_requires_grad(modules, req):
for m in modules:
for param in m.parameters():
param.requires_grad = req
set_requires_grad([model.features], False)
set_requires_grad([model.head], True)
EPOCHS = 20 if not DEBUG else 1
patience = 5
optimizer = optim.Adam(model.head.parameters(), lr=5e-4, weight_decay=1e-5)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=EPOCHS)
best_loss = float('inf')
best_state = None
patience_counter = 0
start_time = time.time() if DEBUG else None
for epoch in range(EPOCHS):
print(f"Epoch {epoch+1}/{EPOCHS}")
if epoch == 3:
set_requires_grad([model.features[3], model.features[4], model.features[5]], True)
optimizer = optim.Adam([
{'params': backbone_params, 'lr': 1e-4},
{'params': mid_params, 'lr': 2e-4},
{'params': head_params, 'lr':5e-4},
], weight_decay=1e-5)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=EPOCHS-epoch)
print("Unfroze mid layers of EfficientNet for fine-tuning.")
elif epoch == 6:
set_requires_grad([model.features], True)
print("Unfroze all layers of EfficientNet for full fine-tuning.")
model.train()
tr_loss = 0.
tr_cnt = 0
for imgs, lbls in train_loader:
imgs = imgs.to(device)
lbls = lbls.to(device).view(-1,1)
optimizer.zero_grad()
if scaler is not None:
with torch.cuda.amp.autocast():
outs = model(imgs)
loss = criterion(outs, lbls)
scaler.scale(loss).backward()
scaler.step(optimizer)
scaler.update()
else:
outs = model(imgs)
loss = criterion(outs, lbls)
loss.backward()
optimizer.step()
tr_loss += loss.item() * imgs.size(0)
tr_cnt += imgs.size(0)
if scheduler is not None:
scheduler.step()
tr_loss = tr_loss / tr_cnt
model.eval()
val_loss = 0.
val_cnt = 0
all_val_lbls = []
all_val_preds = []
with torch.no_grad():
for imgs, lbls in val_loader:
imgs = imgs.to(device)
lbls = lbls.cpu().numpy()
outs = model(imgs).cpu().squeeze().numpy()
preds = 1/(1 + np.exp(-outs))
loss = criterion(torch.tensor(outs).view(-1,1), torch.tensor(lbls).view(-1,1)).item()
val_loss += loss * imgs.size(0)
val_cnt += imgs.size(0)
all_val_lbls.append(lbls)
all_val_preds.append(preds)
val_loss = val_loss / val_cnt
all_val_lbls = np.concatenate(all_val_lbls)
all_val_preds = np.concatenate(all_val_preds)
try:
val_logloss = log_loss(all_val_lbls, all_val_preds, eps=1e-7)
except Exception as ex:
val_logloss = float('inf')
print("Error computing log_loss on val:", ex)
print(f"Train Loss: {tr_loss:.5f} | Val Loss (BCE): {val_loss:.5f} | Val LogLoss: {val_logloss:.5f}")
if val_logloss < best_loss:
best_loss = val_logloss
best_state = {
'model': model.state_dict(),
'epoch': epoch,
'val_loss': best_loss,
}
torch.save(best_state, MODEL_TRAINED_FILE)
patience_counter = 0
print(f"Best model saved. (epoch {epoch+1}, val_logloss={val_logloss:.5f})")
else:
patience_counter += 1
print(f"No improvement. Early stopping patience: {patience_counter}/{patience}")
if patience_counter >= patience:
print(f"Early stopping triggered at epoch {epoch+1}.")
break
if DEBUG and start_time is not None:
end_time = time.time()
debug_time = end_time - start_time
# Compute estimated time: (fractional data)*(epochs) compared
sample_factor = 0.1
scale = (1/sample_factor) * (20 if not DEBUG else 1)
estimated_time = debug_time * scale
# Reload best model for evaluation
state = torch.load(MODEL_TRAINED_FILE, map_location=device)
model.load_state_dict(state['model'])
print("Section: Validation Evaluation and Metric Calculation")
model.eval()
val_lbls, val_prs = [], []
with torch.no_grad():
for imgs, lbls in val_loader:
imgs = imgs.to(device)
outs = model(imgs).cpu().squeeze().numpy()
prs = 1/(1+np.exp(-outs))
val_lbls.append(lbls.numpy())
val_prs.append(prs)
val_lbls = np.concatenate(val_lbls)
val_prs = np.concatenate(val_prs)
try:
val_logloss = log_loss(val_lbls, val_prs, eps=1e-7)
except Exception as ex:
val_logloss = float('inf')
print("Error computing log_loss on validation:", ex)
print(f"Final best model log loss on validation split: {val_logloss:.6f}")
scores = pd.DataFrame(
{'Model': ['efficientnet_b0', 'ensemble'], 'LogLoss': [val_logloss, val_logloss]}
).set_index('Model')
scores.to_csv(SCORES_PATH)
print(f"Saved scores.csv with validation log loss.")
print("Section: Prediction and Submission Generation")
model.eval()
test_probs = []
test_ids_ordered = []
with torch.no_grad():
for imgs, img_ids in test_loader:
imgs = imgs.to(device)
outs = model(imgs).cpu().squeeze().numpy()
prs = 1/(1+np.exp(-outs))
if isinstance(img_ids, list) or isinstance(img_ids, np.ndarray):
test_ids_ordered += list(img_ids)
else:
test_ids_ordered.append(img_ids)
test_probs.extend(np.array(prs).ravel().tolist())
submit_df = pd.DataFrame({'id': test_ids_ordered, 'has_cactus': test_probs})
submit_df = submit_df.set_index('id')
try:
submit_df = submit_df.reindex(sample_submission['id']).reset_index()
except Exception:
submit_df = submit_df.reset_index()
submit_df['has_cactus'] = submit_df['has_cactus'].clip(0,1)
submit_df.to_csv(SUBMISSION_PATH, index=False, float_format='%.6f')
print(f"Saved submission.csv with {len(submit_df)} rows. Format: {submit_df.columns.tolist()}")
# === Debug info output, always print in debug mode, even if only inference ===
if DEBUG:
if debug_time is None:
debug_time = 1.0
scale = (1/0.1)*(1 if DEBUG else 20)
estimated_time = debug_time * scale
print("=== Start of Debug Information ===")
print(f"debug_time: {debug_time}")
print(f"estimated_time: {estimated_time}")
print("=== End of Debug Information ===")
if __name__ == '__main__':
main()
test/notebook/testfiles/main_missing_main_fn.py
+507
View File
@@ -0,0 +1,507 @@
import os
import sys
import time
import random
import numpy as np
import pandas as pd
import torch
import torch.nn as nn
import torch.optim as optim
from torch.utils.data import Dataset, DataLoader
import timm
import albumentations as A
from albumentations.pytorch import ToTensorV2
from sklearn.model_selection import StratifiedKFold
from sklearn.metrics import roc_auc_score, confusion_matrix
import cv2
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('--debug', action='store_true', help='Run in debug mode')
args = parser.parse_args()
DEBUG = args.debug
SEED = 2024
np.random.seed(SEED)
random.seed(SEED)
torch.manual_seed(SEED)
torch.cuda.manual_seed_all(SEED)
DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu")
TRAIN_DIR = './workspace_input/train/'
TEST_DIR = './workspace_input/test/'
TRAIN_CSV = './workspace_input/train.csv'
SAMPLE_SUB_PATH = './workspace_input/sample_submission.csv'
MODEL_DIR = 'models/'
os.makedirs(MODEL_DIR, exist_ok=True)
def print_eda(train_df):
print("=== Start of EDA part ===")
print("Shape of train.csv:", train_df.shape)
print("First 5 rows:\n", train_df.head())
print("Column data types:\n", train_df.dtypes)
print("Missing values per column:\n", train_df.isnull().sum())
print("Unique values per column:")
for col in train_df.columns:
print(f" - {col}: {train_df[col].nunique()}")
label_counts = train_df['has_cactus'].value_counts()
print("Label distribution (has_cactus):")
print(label_counts)
pos, neg = label_counts.get(1, 0), label_counts.get(0, 0)
total = pos + neg
if total > 0:
print(f" Positive:Negative ratio: {pos}:{neg} ({pos/total:.3f}:{neg/total:.3f})")
print(f" Percentage positive: {pos/total*100:.2f}%")
else:
print(" No data found.")
print("Image filename examples:", train_df['id'].unique()[:5])
print("=== End of EDA part ===")
class CactusDataset(Dataset):
def __init__(self, image_ids, labels=None, id2path=None, transforms=None):
self.image_ids = image_ids
self.labels = labels
self.id2path = id2path
self.transforms = transforms
def __len__(self):
return len(self.image_ids)
def __getitem__(self, idx):
img_id = self.image_ids[idx]
img_path = self.id2path[img_id]
image = cv2.imread(img_path)
if image is None:
raise RuntimeError(f"Cannot read image at {img_path}")
image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
if self.transforms:
augmented = self.transforms(image=image)
image = augmented["image"]
if self.labels is not None:
label = self.labels[idx]
return image, label, img_id
else:
return image, img_id
def get_transforms(mode='train'):
# Correct Cutout: Albumentations v1.4.15 provides 'Cutout' as a class, but not always in the root.
# Defensive import; fallback to the most robust method for v1.4.15
imagenet_mean = [0.485, 0.456, 0.406]
imagenet_std = [0.229, 0.224, 0.225]
if mode == 'train':
min_frac, max_frac = 0.05, 0.2
min_cut = int(300 * min_frac)
max_cut = int(300 * max_frac)
# There is no A.Cutout in v1.4.15 root, but A.augmentations.transforms.Cutout exists.
try:
from albumentations.augmentations.transforms import Cutout
have_cutout = True
except ImportError:
have_cutout = False
this_cut_h = random.randint(min_cut, max_cut)
this_cut_w = random.randint(min_cut, max_cut)
cutout_fill = [int(255 * m) for m in imagenet_mean]
tforms = [
A.RandomResizedCrop(300, 300, scale=(0.7, 1.0), ratio=(0.8, 1.2), p=1.0),
A.Rotate(limit=30, p=0.8),
]
if have_cutout:
tforms.append(
Cutout(
num_holes=1,
max_h_size=this_cut_h,
max_w_size=this_cut_w,
fill_value=cutout_fill, # RGB image in albumentations requires [R,G,B]
always_apply=False,
p=0.7
)
)
else:
# No available Cutout, so fallback to no cutout but emit warning
print("WARNING: albumentations.Cutout not found, continuing without Cutout augmentation")
tforms.extend([
A.RandomContrast(limit=0.2, p=0.5),
A.ColorJitter(brightness=0.2, contrast=0.2, saturation=0.2, hue=0.1, p=0.1),
A.Normalize(mean=imagenet_mean, std=imagenet_std, max_pixel_value=255.0),
ToTensorV2()
])
return A.Compose(tforms)
else:
return A.Compose([
A.Resize(300, 300),
A.Normalize(mean=imagenet_mean, std=imagenet_std, max_pixel_value=255.0),
ToTensorV2()
])
def get_dataloader(dataset, batch_size, shuffle=False, num_workers=4, pin_memory=True):
return DataLoader(
dataset,
batch_size=batch_size,
shuffle=shuffle,
num_workers=num_workers,
pin_memory=pin_memory
)
def get_efficientnet_b3(dropout_rate=0.3):
model = timm.create_model('efficientnet_b3', pretrained=True)
n_in = model.classifier.in_features if hasattr(model, "classifier") else model.fc.in_features
model.classifier = nn.Sequential(
nn.Dropout(dropout_rate),
nn.Linear(n_in, 1)
)
return model
def compute_class_weight(y):
counts = np.bincount(y)
if len(counts) < 2:
counts = np.pad(counts, (0, 2-len(counts)), constant_values=0)
n_pos, n_neg = counts[1], counts[0]
total = n_pos + n_neg
minority, majority = min(n_pos, n_neg), max(n_pos, n_neg)
ratio = majority / (minority + 1e-10)
need_weights = ratio > 2
weights = None
if need_weights:
inv_freq = [1 / (n_neg + 1e-10), 1 / (n_pos + 1e-10)]
s = sum(inv_freq)
weights = [w / s * 2 for w in inv_freq]
return weights, n_pos, n_neg, ratio, need_weights
def train_one_epoch(model, loss_fn, optimizer, scheduler, dataloader, device, class_weights):
model.train()
total_loss = 0.0
total_samples = 0
for batch in dataloader:
images, labels, _ = batch
images = images.to(device)
labels = labels.float().unsqueeze(1).to(device)
logits = model(images)
if class_weights is not None:
weight = labels * class_weights[1] + (1 - labels) * class_weights[0]
loss = loss_fn(logits, labels)
loss = (loss * weight).mean()
else:
loss = loss_fn(logits, labels)
optimizer.zero_grad()
loss.backward()
optimizer.step()
if scheduler is not None:
scheduler.step()
total_loss += loss.item() * labels.size(0)
total_samples += labels.size(0)
avg_loss = total_loss / total_samples
return avg_loss
@torch.no_grad()
def eval_model(model, loss_fn, dataloader, device, class_weights):
model.eval()
y_true, y_pred = [], []
total_loss = 0.0
total_samples = 0
for batch in dataloader:
images, labels, _ = batch
images = images.to(device)
labels = labels.float().unsqueeze(1).to(device)
logits = model(images)
probs = torch.sigmoid(logits)
y_true.append(labels.cpu().numpy())
y_pred.append(probs.cpu().numpy())
if class_weights is not None:
weight = labels * class_weights[1] + (1 - labels) * class_weights[0]
loss = loss_fn(logits, labels)
loss = (loss * weight).mean()
else:
loss = loss_fn(logits, labels)
total_loss += loss.item() * labels.size(0)
total_samples += labels.size(0)
y_true = np.vstack(y_true).reshape(-1)
y_pred = np.vstack(y_pred).reshape(-1)
avg_loss = total_loss / total_samples
return avg_loss, y_true, y_pred
def confusion_info(y_true, y_pred, threshold=0.5):
preds = (y_pred > threshold).astype(int)
cm = confusion_matrix(y_true, preds)
return cm
def inference_and_submission(train_df, train_id2path, test_img_ids, test_id2path, dropout_rate, class_weights, need_weights,
BATCH_SIZE, N_WORKERS, cv_fold):
oof_true, oof_pred, fold_scores, fold_val_ids = [], [], [], []
for fold in range(cv_fold):
df_val = train_df[train_df['fold'] == fold].reset_index(drop=True)
val_img_ids = df_val['id'].tolist()
val_labels = df_val['has_cactus'].values
val_ds = CactusDataset(val_img_ids, val_labels, id2path=train_id2path, transforms=get_transforms("val"))
val_loader = get_dataloader(val_ds, BATCH_SIZE, shuffle=False, num_workers=N_WORKERS)
fold_model_path = os.path.join(MODEL_DIR, f"efficientnet_b3_fold{fold}.pt")
model = get_efficientnet_b3(dropout_rate=dropout_rate)
model.load_state_dict(torch.load(fold_model_path, map_location='cpu'))
model.to(DEVICE)
model.eval()
fold_class_weights = class_weights if need_weights else None
if fold_class_weights is not None:
fold_class_weights = torch.tensor(fold_class_weights).float().to(DEVICE)
loss_fn = nn.BCEWithLogitsLoss(reduction='none')
_, val_true, val_pred = eval_model(model, loss_fn, val_loader, DEVICE, fold_class_weights)
val_auc = roc_auc_score(val_true, val_pred)
oof_true.append(val_true)
oof_pred.append(val_pred)
fold_val_ids.append(val_img_ids)
fold_scores.append(val_auc)
print(f"Reloaded fold {fold}, OOF Validation AUC={val_auc:.5f}")
all_oof_true = np.concatenate(oof_true)
all_oof_pred = np.concatenate(oof_pred)
oof_auc = roc_auc_score(all_oof_true, all_oof_pred)
oof_cm = confusion_info(all_oof_true, all_oof_pred)
print(f"OOF ROC-AUC (from loaded models): {oof_auc:.5f}")
print(f"OOF Confusion Matrix:\n{oof_cm}")
test_ds = CactusDataset(
test_img_ids, labels=None,
id2path=test_id2path,
transforms=get_transforms("val")
)
test_loader = get_dataloader(test_ds, BATCH_SIZE, shuffle=False, num_workers=N_WORKERS)
test_pred_list = []
for fold in range(cv_fold):
fold_model_path = os.path.join(MODEL_DIR, f"efficientnet_b3_fold{fold}.pt")
model = get_efficientnet_b3(dropout_rate=dropout_rate)
model.load_state_dict(torch.load(fold_model_path, map_location='cpu'))
model.to(DEVICE)
model.eval()
preds = []
with torch.no_grad():
for batch in test_loader:
images, img_ids = batch
images = images.to(DEVICE)
logits = model(images)
probs = torch.sigmoid(logits).cpu().numpy().reshape(-1)
preds.append(probs)
fold_test_pred = np.concatenate(preds)
test_pred_list.append(fold_test_pred)
print(f"Loaded fold {fold} for test prediction.")
test_probs = np.mean(test_pred_list, axis=0)
submission = pd.read_csv(SAMPLE_SUB_PATH)
submission['has_cactus'] = test_probs
submission.to_csv('submission.csv', index=False)
print(f"Saved submission.csv in required format with {len(submission)} rows.")
scores_df = pd.DataFrame({
'Model': [f"efficientnet_b3_fold{f}" for f in range(cv_fold)] + ['ensemble'],
'ROC-AUC': list(fold_scores) + [oof_auc]
})
scores_df.set_index('Model', inplace=True)
scores_df.to_csv("scores.csv")
print(f"Saved cross-validation scores to scores.csv")
print("Section: Data Loading and Preprocessing")
try:
train_df = pd.read_csv(TRAIN_CSV)
except Exception as e:
print(f"Failed to load train.csv: {e}")
sys.exit(1)
print_eda(train_df)
train_id2path = {img_id: os.path.join(TRAIN_DIR, img_id) for img_id in train_df['id']}
try:
sample_sub = pd.read_csv(SAMPLE_SUB_PATH)
except Exception as e:
print(f"Failed to load sample_submission.csv: {e}")
sys.exit(1)
test_img_ids = list(sample_sub['id'])
test_id2path = {img_id: os.path.join(TEST_DIR, img_id) for img_id in test_img_ids}
print(f"Loaded {len(train_id2path)} train images, {len(test_id2path)} test images.")
y_train = train_df['has_cactus'].values
class_weights, n_pos, n_neg, imbalance_ratio, need_weights = compute_class_weight(y_train)
print(f"Class stats: Pos={n_pos}, Neg={n_neg}, Imbalance Ratio(majority/minority)={imbalance_ratio:.3f}")
print(f"Use class weights: {need_weights}, Class weights: {class_weights if class_weights is not None else '[1.0,1.0]'}")
if class_weights is not None:
np.save(os.path.join(MODEL_DIR, "class_weights.npy"), class_weights)
print("Section: Feature Engineering")
train_df = train_df.copy()
cv_fold = 5
skf = StratifiedKFold(n_splits=cv_fold, shuffle=True, random_state=SEED)
folds = np.zeros(len(train_df), dtype=np.int32)
for idx, (_, val_idx) in enumerate(skf.split(train_df['id'], train_df['has_cactus'])):
folds[val_idx] = idx
train_df['fold'] = folds
print(f"Assigned stratified {cv_fold}-fold indices. Fold sample counts:")
for f in range(cv_fold):
dist = train_df.loc[train_df['fold'] == f, 'has_cactus'].value_counts().to_dict()
print(f" Fold {f}: n={len(train_df[train_df['fold'] == f])} class dist={dist}")
print("Section: Model Training and Evaluation")
dropout_rate = round(random.uniform(0.2, 0.5), 2)
print(f"Model config: EfficientNet-B3, Image size 300, Head dropout={dropout_rate}")
if DEBUG:
print("DEBUG mode: using 10% subsample and 1 epoch (per fold)")
sample_frac = 0.10
sampled_idxs = []
for f in range(cv_fold):
fold_idx = train_df.index[train_df['fold'] == f].tolist()
fold_labels = train_df.loc[fold_idx, 'has_cactus'].values
idx_pos = [i for i, l in zip(fold_idx, fold_labels) if l == 1]
idx_neg = [i for i, l in zip(fold_idx, fold_labels) if l == 0]
n_pos = max(1, int(sample_frac * len(idx_pos)))
n_neg = max(1, int(sample_frac * len(idx_neg)))
if len(idx_pos) > 0:
sampled_idxs += np.random.choice(idx_pos, n_pos, replace=False).tolist()
if len(idx_neg) > 0:
sampled_idxs += np.random.choice(idx_neg, n_neg, replace=False).tolist()
train_df = train_df.loc[sampled_idxs].reset_index(drop=True)
print(f"DEBUG subsample shape: {train_df.shape}")
debug_epochs = 1
else:
debug_epochs = None
BATCH_SIZE = 64 if torch.cuda.is_available() else 32
N_WORKERS = 4 if torch.cuda.is_available() else 1
EPOCHS = 20 if not DEBUG else debug_epochs
MIN_EPOCHS = 5 if not DEBUG else 1
EARLY_STOP_PATIENCE = 7 if not DEBUG else 2
LR = 1e-3
model_files = [os.path.join(MODEL_DIR, f"efficientnet_b3_fold{f}.pt") for f in range(cv_fold)]
if all([os.path.exists(f) for f in model_files]):
print("All fold models found in models/. Running inference and file saving only (no retrain).")
inference_and_submission(train_df, train_id2path, test_img_ids, test_id2path, dropout_rate,
class_weights, need_weights, BATCH_SIZE, N_WORKERS, cv_fold)
return
oof_true, oof_pred, fold_scores, fold_val_ids = [], [], [], []
start_time = time.time() if DEBUG else None
for fold in range(cv_fold):
print(f"\n=== FOLD {fold} TRAINING ===")
df_train = train_df[train_df['fold'] != fold].reset_index(drop=True)
df_val = train_df[train_df['fold'] == fold].reset_index(drop=True)
print(f"Train size: {df_train.shape[0]}, Val size: {df_val.shape[0]}")
train_img_ids = df_train['id'].tolist()
train_labels = df_train['has_cactus'].values
val_img_ids = df_val['id'].tolist()
val_labels = df_val['has_cactus'].values
train_ds = CactusDataset(
train_img_ids, train_labels,
id2path=train_id2path,
transforms=get_transforms("train")
)
val_ds = CactusDataset(
val_img_ids, val_labels,
id2path=train_id2path,
transforms=get_transforms("val")
)
train_loader = get_dataloader(train_ds, BATCH_SIZE, shuffle=True, num_workers=N_WORKERS)
val_loader = get_dataloader(val_ds, BATCH_SIZE, shuffle=False, num_workers=N_WORKERS)
model = get_efficientnet_b3(dropout_rate=dropout_rate)
model.to(DEVICE)
loss_fn = nn.BCEWithLogitsLoss(reduction='none')
optimizer = optim.AdamW(model.parameters(), lr=LR)
scheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=EPOCHS)
fold_class_weights = class_weights if need_weights else None
if fold_class_weights is not None:
fold_class_weights = torch.tensor(fold_class_weights).float().to(DEVICE)
best_auc = -np.inf
best_epoch = -1
best_model_state = None
patience = 0
for epoch in range(EPOCHS):
train_loss = train_one_epoch(
model, loss_fn, optimizer, scheduler, train_loader, DEVICE, fold_class_weights)
val_loss, val_true, val_pred = eval_model(
model, loss_fn, val_loader, DEVICE, fold_class_weights)
val_auc = roc_auc_score(val_true, val_pred)
cm = confusion_info(val_true, val_pred)
print(f"Epoch {epoch+1:02d}: train_loss={train_loss:.4f} val_loss={val_loss:.4f} val_auc={val_auc:.4f}")
print(f" Val confusion_matrix (rows:true [0,1]; cols:pred [0,1]):\n{cm}")
if val_auc > best_auc:
best_auc = val_auc
best_model_state = {k: v.cpu().clone() for k, v in model.state_dict().items()}
best_epoch = epoch
patience = 0
else:
patience += 1
if DEBUG and epoch + 1 >= debug_epochs:
break
if (epoch + 1) >= MIN_EPOCHS and patience >= EARLY_STOP_PATIENCE:
print(f"Early stopping at epoch {epoch+1}, best_epoch={best_epoch+1}.")
break
model.load_state_dict(best_model_state)
fold_model_path = os.path.join(MODEL_DIR, f"efficientnet_b3_fold{fold}.pt")
torch.save(model.state_dict(), fold_model_path)
print(f"Saved best model for fold {fold} at {fold_model_path} (best_auc={best_auc:.5f}, best_epoch={best_epoch+1})")
_, val_true, val_pred = eval_model(model, loss_fn, val_loader, DEVICE, fold_class_weights)
oof_true.append(val_true)
oof_pred.append(val_pred)
fold_val_ids.append(val_img_ids)
fold_scores.append(best_auc)
print(f"OOF stored for fold {fold}, Validation AUC={best_auc:.5f}")
end_time = time.time() if DEBUG else None
if DEBUG:
debug_time = end_time - start_time
estimated_time = (1 / 0.1) * (EPOCHS / debug_epochs) * debug_time
print("=== Start of Debug Information ===")
print(f"debug_time: {debug_time:.1f}")
print(f"estimated_time: {estimated_time:.1f}")
print("=== End of Debug Information ===")
print("\nSection: Ensemble Strategy and Final Predictions")
all_oof_true = np.concatenate(oof_true)
all_oof_pred = np.concatenate(oof_pred)
oof_auc = roc_auc_score(all_oof_true, all_oof_pred)
oof_cm = confusion_info(all_oof_true, all_oof_pred)
print(f"OOF ROC-AUC: {oof_auc:.5f}")
print(f"OOF Confusion Matrix:\n{oof_cm}")
test_ds = CactusDataset(
test_img_ids, labels=None,
id2path=test_id2path,
transforms=get_transforms("val")
)
test_loader = get_dataloader(test_ds, BATCH_SIZE, shuffle=False, num_workers=N_WORKERS)
test_pred_list = []
for fold in range(cv_fold):
fold_model_path = os.path.join(MODEL_DIR, f"efficientnet_b3_fold{fold}.pt")
model = get_efficientnet_b3(dropout_rate=dropout_rate)
model.load_state_dict(torch.load(fold_model_path, map_location='cpu'))
model.to(DEVICE)
model.eval()
preds = []
with torch.no_grad():
for batch in test_loader:
images, img_ids = batch
images = images.to(DEVICE)
logits = model(images)
probs = torch.sigmoid(logits).cpu().numpy().reshape(-1)
preds.append(probs)
fold_test_pred = np.concatenate(preds)
test_pred_list.append(fold_test_pred)
print(f"Loaded fold {fold} for test prediction.")
test_probs = np.mean(test_pred_list, axis=0)
print("Section: Submission File Generation")
submission = pd.read_csv(SAMPLE_SUB_PATH)
submission['has_cactus'] = test_probs
submission.to_csv('submission.csv', index=False)
print(f"Saved submission.csv in required format with {len(submission)} rows.")
scores_df = pd.DataFrame({
'Model': [f"efficientnet_b3_fold{f}" for f in range(cv_fold)] + ['ensemble'],
'ROC-AUC': list(fold_scores) + [oof_auc]
})
scores_df.set_index('Model', inplace=True)
scores_df.to_csv("scores.csv")
print(f"Saved cross-validation scores to scores.csv")
test/notebook/testfiles/main_missing_sections.py
+506
View File
@@ -0,0 +1,506 @@
import os
import sys
import time
import random
import numpy as np
import pandas as pd
import torch
import torch.nn as nn
import torch.optim as optim
from torch.utils.data import Dataset, DataLoader
import timm
import albumentations as A
from albumentations.pytorch import ToTensorV2
from sklearn.model_selection import StratifiedKFold
from sklearn.metrics import roc_auc_score, confusion_matrix
import cv2
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('--debug', action='store_true', help='Run in debug mode')
args = parser.parse_args()
DEBUG = args.debug
SEED = 2024
np.random.seed(SEED)
random.seed(SEED)
torch.manual_seed(SEED)
torch.cuda.manual_seed_all(SEED)
DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu")
TRAIN_DIR = './workspace_input/train/'
TEST_DIR = './workspace_input/test/'
TRAIN_CSV = './workspace_input/train.csv'
SAMPLE_SUB_PATH = './workspace_input/sample_submission.csv'
MODEL_DIR = 'models/'
os.makedirs(MODEL_DIR, exist_ok=True)
def print_eda(train_df):
print("=== Start of EDA part ===")
print("Shape of train.csv:", train_df.shape)
print("First 5 rows:\n", train_df.head())
print("Column data types:\n", train_df.dtypes)
print("Missing values per column:\n", train_df.isnull().sum())
print("Unique values per column:")
for col in train_df.columns:
print(f" - {col}: {train_df[col].nunique()}")
label_counts = train_df['has_cactus'].value_counts()
print("Label distribution (has_cactus):")
print(label_counts)
pos, neg = label_counts.get(1, 0), label_counts.get(0, 0)
total = pos + neg
if total > 0:
print(f" Positive:Negative ratio: {pos}:{neg} ({pos/total:.3f}:{neg/total:.3f})")
print(f" Percentage positive: {pos/total*100:.2f}%")
else:
print(" No data found.")
print("Image filename examples:", train_df['id'].unique()[:5])
print("=== End of EDA part ===")
class CactusDataset(Dataset):
def __init__(self, image_ids, labels=None, id2path=None, transforms=None):
self.image_ids = image_ids
self.labels = labels
self.id2path = id2path
self.transforms = transforms
def __len__(self):
return len(self.image_ids)
def __getitem__(self, idx):
img_id = self.image_ids[idx]
img_path = self.id2path[img_id]
image = cv2.imread(img_path)
if image is None:
raise RuntimeError(f"Cannot read image at {img_path}")
image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
if self.transforms:
augmented = self.transforms(image=image)
image = augmented["image"]
if self.labels is not None:
label = self.labels[idx]
return image, label, img_id
else:
return image, img_id
def get_transforms(mode='train'):
# Correct Cutout: Albumentations v1.4.15 provides 'Cutout' as a class, but not always in the root.
# Defensive import; fallback to the most robust method for v1.4.15
imagenet_mean = [0.485, 0.456, 0.406]
imagenet_std = [0.229, 0.224, 0.225]
if mode == 'train':
min_frac, max_frac = 0.05, 0.2
min_cut = int(300 * min_frac)
max_cut = int(300 * max_frac)
# There is no A.Cutout in v1.4.15 root, but A.augmentations.transforms.Cutout exists.
try:
from albumentations.augmentations.transforms import Cutout
have_cutout = True
except ImportError:
have_cutout = False
this_cut_h = random.randint(min_cut, max_cut)
this_cut_w = random.randint(min_cut, max_cut)
cutout_fill = [int(255 * m) for m in imagenet_mean]
tforms = [
A.RandomResizedCrop(300, 300, scale=(0.7, 1.0), ratio=(0.8, 1.2), p=1.0),
A.Rotate(limit=30, p=0.8),
]
if have_cutout:
tforms.append(
Cutout(
num_holes=1,
max_h_size=this_cut_h,
max_w_size=this_cut_w,
fill_value=cutout_fill, # RGB image in albumentations requires [R,G,B]
always_apply=False,
p=0.7
)
)
else:
# No available Cutout, so fallback to no cutout but emit warning
print("WARNING: albumentations.Cutout not found, continuing without Cutout augmentation")
tforms.extend([
A.RandomContrast(limit=0.2, p=0.5),
A.ColorJitter(brightness=0.2, contrast=0.2, saturation=0.2, hue=0.1, p=0.1),
A.Normalize(mean=imagenet_mean, std=imagenet_std, max_pixel_value=255.0),
ToTensorV2()
])
return A.Compose(tforms)
else:
return A.Compose([
A.Resize(300, 300),
A.Normalize(mean=imagenet_mean, std=imagenet_std, max_pixel_value=255.0),
ToTensorV2()
])
def get_dataloader(dataset, batch_size, shuffle=False, num_workers=4, pin_memory=True):
return DataLoader(
dataset,
batch_size=batch_size,
shuffle=shuffle,
num_workers=num_workers,
pin_memory=pin_memory
)
def get_efficientnet_b3(dropout_rate=0.3):
model = timm.create_model('efficientnet_b3', pretrained=True)
n_in = model.classifier.in_features if hasattr(model, "classifier") else model.fc.in_features
model.classifier = nn.Sequential(
nn.Dropout(dropout_rate),
nn.Linear(n_in, 1)
)
return model
def compute_class_weight(y):
counts = np.bincount(y)
if len(counts) < 2:
counts = np.pad(counts, (0, 2-len(counts)), constant_values=0)
n_pos, n_neg = counts[1], counts[0]
total = n_pos + n_neg
minority, majority = min(n_pos, n_neg), max(n_pos, n_neg)
ratio = majority / (minority + 1e-10)
need_weights = ratio > 2
weights = None
if need_weights:
inv_freq = [1 / (n_neg + 1e-10), 1 / (n_pos + 1e-10)]
s = sum(inv_freq)
weights = [w / s * 2 for w in inv_freq]
return weights, n_pos, n_neg, ratio, need_weights
def train_one_epoch(model, loss_fn, optimizer, scheduler, dataloader, device, class_weights):
model.train()
total_loss = 0.0
total_samples = 0
for batch in dataloader:
images, labels, _ = batch
images = images.to(device)
labels = labels.float().unsqueeze(1).to(device)
logits = model(images)
if class_weights is not None:
weight = labels * class_weights[1] + (1 - labels) * class_weights[0]
loss = loss_fn(logits, labels)
loss = (loss * weight).mean()
else:
loss = loss_fn(logits, labels)
optimizer.zero_grad()
loss.backward()
optimizer.step()
if scheduler is not None:
scheduler.step()
total_loss += loss.item() * labels.size(0)
total_samples += labels.size(0)
avg_loss = total_loss / total_samples
return avg_loss
@torch.no_grad()
def eval_model(model, loss_fn, dataloader, device, class_weights):
model.eval()
y_true, y_pred = [], []
total_loss = 0.0
total_samples = 0
for batch in dataloader:
images, labels, _ = batch
images = images.to(device)
labels = labels.float().unsqueeze(1).to(device)
logits = model(images)
probs = torch.sigmoid(logits)
y_true.append(labels.cpu().numpy())
y_pred.append(probs.cpu().numpy())
if class_weights is not None:
weight = labels * class_weights[1] + (1 - labels) * class_weights[0]
loss = loss_fn(logits, labels)
loss = (loss * weight).mean()
else:
loss = loss_fn(logits, labels)
total_loss += loss.item() * labels.size(0)
total_samples += labels.size(0)
y_true = np.vstack(y_true).reshape(-1)
y_pred = np.vstack(y_pred).reshape(-1)
avg_loss = total_loss / total_samples
return avg_loss, y_true, y_pred
def confusion_info(y_true, y_pred, threshold=0.5):
preds = (y_pred > threshold).astype(int)
cm = confusion_matrix(y_true, preds)
return cm
def inference_and_submission(train_df, train_id2path, test_img_ids, test_id2path, dropout_rate, class_weights, need_weights,
BATCH_SIZE, N_WORKERS, cv_fold):
oof_true, oof_pred, fold_scores, fold_val_ids = [], [], [], []
for fold in range(cv_fold):
df_val = train_df[train_df['fold'] == fold].reset_index(drop=True)
val_img_ids = df_val['id'].tolist()
val_labels = df_val['has_cactus'].values
val_ds = CactusDataset(val_img_ids, val_labels, id2path=train_id2path, transforms=get_transforms("val"))
val_loader = get_dataloader(val_ds, BATCH_SIZE, shuffle=False, num_workers=N_WORKERS)
fold_model_path = os.path.join(MODEL_DIR, f"efficientnet_b3_fold{fold}.pt")
model = get_efficientnet_b3(dropout_rate=dropout_rate)
model.load_state_dict(torch.load(fold_model_path, map_location='cpu'))
model.to(DEVICE)
model.eval()
fold_class_weights = class_weights if need_weights else None
if fold_class_weights is not None:
fold_class_weights = torch.tensor(fold_class_weights).float().to(DEVICE)
loss_fn = nn.BCEWithLogitsLoss(reduction='none')
_, val_true, val_pred = eval_model(model, loss_fn, val_loader, DEVICE, fold_class_weights)
val_auc = roc_auc_score(val_true, val_pred)
oof_true.append(val_true)
oof_pred.append(val_pred)
fold_val_ids.append(val_img_ids)
fold_scores.append(val_auc)
print(f"Reloaded fold {fold}, OOF Validation AUC={val_auc:.5f}")
all_oof_true = np.concatenate(oof_true)
all_oof_pred = np.concatenate(oof_pred)
oof_auc = roc_auc_score(all_oof_true, all_oof_pred)
oof_cm = confusion_info(all_oof_true, all_oof_pred)
print(f"OOF ROC-AUC (from loaded models): {oof_auc:.5f}")
print(f"OOF Confusion Matrix:\n{oof_cm}")
test_ds = CactusDataset(
test_img_ids, labels=None,
id2path=test_id2path,
transforms=get_transforms("val")
)
test_loader = get_dataloader(test_ds, BATCH_SIZE, shuffle=False, num_workers=N_WORKERS)
test_pred_list = []
for fold in range(cv_fold):
fold_model_path = os.path.join(MODEL_DIR, f"efficientnet_b3_fold{fold}.pt")
model = get_efficientnet_b3(dropout_rate=dropout_rate)
model.load_state_dict(torch.load(fold_model_path, map_location='cpu'))
model.to(DEVICE)
model.eval()
preds = []
with torch.no_grad():
for batch in test_loader:
images, img_ids = batch
images = images.to(DEVICE)
logits = model(images)
probs = torch.sigmoid(logits).cpu().numpy().reshape(-1)
preds.append(probs)
fold_test_pred = np.concatenate(preds)
test_pred_list.append(fold_test_pred)
print(f"Loaded fold {fold} for test prediction.")
test_probs = np.mean(test_pred_list, axis=0)
submission = pd.read_csv(SAMPLE_SUB_PATH)
submission['has_cactus'] = test_probs
submission.to_csv('submission.csv', index=False)
print(f"Saved submission.csv in required format with {len(submission)} rows.")
scores_df = pd.DataFrame({
'Model': [f"efficientnet_b3_fold{f}" for f in range(cv_fold)] + ['ensemble'],
'ROC-AUC': list(fold_scores) + [oof_auc]
})
scores_df.set_index('Model', inplace=True)
scores_df.to_csv("scores.csv")
print(f"Saved cross-validation scores to scores.csv")
def main():
try:
train_df = pd.read_csv(TRAIN_CSV)
except Exception as e:
print(f"Failed to load train.csv: {e}")
sys.exit(1)
print_eda(train_df)
train_id2path = {img_id: os.path.join(TRAIN_DIR, img_id) for img_id in train_df['id']}
try:
sample_sub = pd.read_csv(SAMPLE_SUB_PATH)
except Exception as e:
print(f"Failed to load sample_submission.csv: {e}")
sys.exit(1)
test_img_ids = list(sample_sub['id'])
test_id2path = {img_id: os.path.join(TEST_DIR, img_id) for img_id in test_img_ids}
print(f"Loaded {len(train_id2path)} train images, {len(test_id2path)} test images.")
y_train = train_df['has_cactus'].values
class_weights, n_pos, n_neg, imbalance_ratio, need_weights = compute_class_weight(y_train)
print(f"Class stats: Pos={n_pos}, Neg={n_neg}, Imbalance Ratio(majority/minority)={imbalance_ratio:.3f}")
print(f"Use class weights: {need_weights}, Class weights: {class_weights if class_weights is not None else '[1.0,1.0]'}")
if class_weights is not None:
np.save(os.path.join(MODEL_DIR, "class_weights.npy"), class_weights)
train_df = train_df.copy()
cv_fold = 5
skf = StratifiedKFold(n_splits=cv_fold, shuffle=True, random_state=SEED)
folds = np.zeros(len(train_df), dtype=np.int32)
for idx, (_, val_idx) in enumerate(skf.split(train_df['id'], train_df['has_cactus'])):
folds[val_idx] = idx
train_df['fold'] = folds
print(f"Assigned stratified {cv_fold}-fold indices. Fold sample counts:")
for f in range(cv_fold):
dist = train_df.loc[train_df['fold'] == f, 'has_cactus'].value_counts().to_dict()
print(f" Fold {f}: n={len(train_df[train_df['fold'] == f])} class dist={dist}")
dropout_rate = round(random.uniform(0.2, 0.5), 2)
print(f"Model config: EfficientNet-B3, Image size 300, Head dropout={dropout_rate}")
if DEBUG:
print("DEBUG mode: using 10% subsample and 1 epoch (per fold)")
sample_frac = 0.10
sampled_idxs = []
for f in range(cv_fold):
fold_idx = train_df.index[train_df['fold'] == f].tolist()
fold_labels = train_df.loc[fold_idx, 'has_cactus'].values
idx_pos = [i for i, l in zip(fold_idx, fold_labels) if l == 1]
idx_neg = [i for i, l in zip(fold_idx, fold_labels) if l == 0]
n_pos = max(1, int(sample_frac * len(idx_pos)))
n_neg = max(1, int(sample_frac * len(idx_neg)))
if len(idx_pos) > 0:
sampled_idxs += np.random.choice(idx_pos, n_pos, replace=False).tolist()
if len(idx_neg) > 0:
sampled_idxs += np.random.choice(idx_neg, n_neg, replace=False).tolist()
train_df = train_df.loc[sampled_idxs].reset_index(drop=True)
print(f"DEBUG subsample shape: {train_df.shape}")
debug_epochs = 1
else:
debug_epochs = None
BATCH_SIZE = 64 if torch.cuda.is_available() else 32
N_WORKERS = 4 if torch.cuda.is_available() else 1
EPOCHS = 20 if not DEBUG else debug_epochs
MIN_EPOCHS = 5 if not DEBUG else 1
EARLY_STOP_PATIENCE = 7 if not DEBUG else 2
LR = 1e-3
model_files = [os.path.join(MODEL_DIR, f"efficientnet_b3_fold{f}.pt") for f in range(cv_fold)]
if all([os.path.exists(f) for f in model_files]):
print("All fold models found in models/. Running inference and file saving only (no retrain).")
inference_and_submission(train_df, train_id2path, test_img_ids, test_id2path, dropout_rate,
class_weights, need_weights, BATCH_SIZE, N_WORKERS, cv_fold)
return
oof_true, oof_pred, fold_scores, fold_val_ids = [], [], [], []
start_time = time.time() if DEBUG else None
for fold in range(cv_fold):
print(f"\n=== FOLD {fold} TRAINING ===")
df_train = train_df[train_df['fold'] != fold].reset_index(drop=True)
df_val = train_df[train_df['fold'] == fold].reset_index(drop=True)
print(f"Train size: {df_train.shape[0]}, Val size: {df_val.shape[0]}")
train_img_ids = df_train['id'].tolist()
train_labels = df_train['has_cactus'].values
val_img_ids = df_val['id'].tolist()
val_labels = df_val['has_cactus'].values
train_ds = CactusDataset(
train_img_ids, train_labels,
id2path=train_id2path,
transforms=get_transforms("train")
)
val_ds = CactusDataset(
val_img_ids, val_labels,
id2path=train_id2path,
transforms=get_transforms("val")
)
train_loader = get_dataloader(train_ds, BATCH_SIZE, shuffle=True, num_workers=N_WORKERS)
val_loader = get_dataloader(val_ds, BATCH_SIZE, shuffle=False, num_workers=N_WORKERS)
model = get_efficientnet_b3(dropout_rate=dropout_rate)
model.to(DEVICE)
loss_fn = nn.BCEWithLogitsLoss(reduction='none')
optimizer = optim.AdamW(model.parameters(), lr=LR)
scheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=EPOCHS)
fold_class_weights = class_weights if need_weights else None
if fold_class_weights is not None:
fold_class_weights = torch.tensor(fold_class_weights).float().to(DEVICE)
best_auc = -np.inf
best_epoch = -1
best_model_state = None
patience = 0
for epoch in range(EPOCHS):
train_loss = train_one_epoch(
model, loss_fn, optimizer, scheduler, train_loader, DEVICE, fold_class_weights)
val_loss, val_true, val_pred = eval_model(
model, loss_fn, val_loader, DEVICE, fold_class_weights)
val_auc = roc_auc_score(val_true, val_pred)
cm = confusion_info(val_true, val_pred)
print(f"Epoch {epoch+1:02d}: train_loss={train_loss:.4f} val_loss={val_loss:.4f} val_auc={val_auc:.4f}")
print(f" Val confusion_matrix (rows:true [0,1]; cols:pred [0,1]):\n{cm}")
if val_auc > best_auc:
best_auc = val_auc
best_model_state = {k: v.cpu().clone() for k, v in model.state_dict().items()}
best_epoch = epoch
patience = 0
else:
patience += 1
if DEBUG and epoch + 1 >= debug_epochs:
break
if (epoch + 1) >= MIN_EPOCHS and patience >= EARLY_STOP_PATIENCE:
print(f"Early stopping at epoch {epoch+1}, best_epoch={best_epoch+1}.")
break
model.load_state_dict(best_model_state)
fold_model_path = os.path.join(MODEL_DIR, f"efficientnet_b3_fold{fold}.pt")
torch.save(model.state_dict(), fold_model_path)
print(f"Saved best model for fold {fold} at {fold_model_path} (best_auc={best_auc:.5f}, best_epoch={best_epoch+1})")
_, val_true, val_pred = eval_model(model, loss_fn, val_loader, DEVICE, fold_class_weights)
oof_true.append(val_true)
oof_pred.append(val_pred)
fold_val_ids.append(val_img_ids)
fold_scores.append(best_auc)
print(f"OOF stored for fold {fold}, Validation AUC={best_auc:.5f}")
end_time = time.time() if DEBUG else None
if DEBUG:
debug_time = end_time - start_time
estimated_time = (1 / 0.1) * (EPOCHS / debug_epochs) * debug_time
print("=== Start of Debug Information ===")
print(f"debug_time: {debug_time:.1f}")
print(f"estimated_time: {estimated_time:.1f}")
print("=== End of Debug Information ===")
all_oof_true = np.concatenate(oof_true)
all_oof_pred = np.concatenate(oof_pred)
oof_auc = roc_auc_score(all_oof_true, all_oof_pred)
oof_cm = confusion_info(all_oof_true, all_oof_pred)
print(f"OOF ROC-AUC: {oof_auc:.5f}")
print(f"OOF Confusion Matrix:\n{oof_cm}")
test_ds = CactusDataset(
test_img_ids, labels=None,
id2path=test_id2path,
transforms=get_transforms("val")
)
test_loader = get_dataloader(test_ds, BATCH_SIZE, shuffle=False, num_workers=N_WORKERS)
test_pred_list = []
for fold in range(cv_fold):
fold_model_path = os.path.join(MODEL_DIR, f"efficientnet_b3_fold{fold}.pt")
model = get_efficientnet_b3(dropout_rate=dropout_rate)
model.load_state_dict(torch.load(fold_model_path, map_location='cpu'))
model.to(DEVICE)
model.eval()
preds = []
with torch.no_grad():
for batch in test_loader:
images, img_ids = batch
images = images.to(DEVICE)
logits = model(images)
probs = torch.sigmoid(logits).cpu().numpy().reshape(-1)
preds.append(probs)
fold_test_pred = np.concatenate(preds)
test_pred_list.append(fold_test_pred)
print(f"Loaded fold {fold} for test prediction.")
test_probs = np.mean(test_pred_list, axis=0)
submission = pd.read_csv(SAMPLE_SUB_PATH)
submission['has_cactus'] = test_probs
submission.to_csv('submission.csv', index=False)
print(f"Saved submission.csv in required format with {len(submission)} rows.")
scores_df = pd.DataFrame({
'Model': [f"efficientnet_b3_fold{f}" for f in range(cv_fold)] + ['ensemble'],
'ROC-AUC': list(fold_scores) + [oof_auc]
})
scores_df.set_index('Model', inplace=True)
scores_df.to_csv("scores.csv")
print(f"Saved cross-validation scores to scores.csv")
if __name__ == "__main__":
main()