From 573353421bd10313347ce6445422e83305ef6933 Mon Sep 17 00:00:00 2001
From: "2569718930@qq.com" <2569718930@qq.com>
Date: Mon, 1 Jun 2026 14:21:43 +0800
Subject: [PATCH] Retry deploy compose recreate races

---
 deploy.sh                               | 33 ++++++++++++++++++++++---
 tests/test_deployment_runtime_config.py |  9 +++++++
 2 files changed, 38 insertions(+), 4 deletions(-)

diff --git a/deploy.sh b/deploy.sh
index 22643c76..81239c02 100644
--- a/deploy.sh
+++ b/deploy.sh
@@ -30,13 +30,38 @@ rollback_to_previous() {
         echo "Rolling back to $PREVIOUS_TAG..."
         export IMAGE_TAG="$PREVIOUS_TAG"
         docker compose pull
-        docker compose up -d
+        compose_up_retry "rollback" -d
         echo "✅ Rolled back to $PREVIOUS_TAG"
     else
         echo "⚠️  No previous tag to rollback to"
     fi
 }
 
+compose_up_retry() {
+    local name="$1"
+    shift
+    local output=""
+
+    for attempt in $(seq 1 6); do
+        if output=$(docker compose up "$@" 2>&1); then
+            echo "$output"
+            return 0
+        fi
+
+        echo "$output"
+        if echo "$output" | grep -qi "removal of container .* is already in progress"; then
+            echo "Container removal is still in progress during ${name}; retry ${attempt}/6..."
+            sleep 5
+            continue
+        fi
+
+        return 1
+    done
+
+    echo "❌ docker compose up failed for ${name} after retries"
+    return 1
+}
+
 export IMAGE_TAG="$NEW_TAG"
 pull_ok=0
 for pull_attempt in $(seq 1 6); do
@@ -117,10 +142,10 @@ warm_public_route() {
 }
 
 echo "Updating Redis dependency..."
-docker compose up -d polyweather_redis
+compose_up_retry "redis" -d polyweather_redis
 
 echo "Updating backend services..."
-docker compose up -d --no-deps polyweather_web polyweather
+compose_up_retry "backend services" -d --no-deps polyweather_web polyweather
 
 echo "Waiting for backend..."
 wait_for_local_service "backend healthz" "http://127.0.0.1:8000/healthz" 5 30 5 || FAILED_BACKEND=1
@@ -132,7 +157,7 @@ if [ "$FAILED_BACKEND" = "1" ]; then
 fi
 
 echo "Updating frontend..."
-docker compose up -d --no-deps polyweather_frontend
+compose_up_retry "frontend" -d --no-deps polyweather_frontend
 
 echo "Waiting for frontend..."
 wait_for_local_service "frontend root" "http://127.0.0.1:3001/" 5 40 2 || FAILED_FRONTEND=1
diff --git a/tests/test_deployment_runtime_config.py b/tests/test_deployment_runtime_config.py
index d42cecc1..f26cf649 100644
--- a/tests/test_deployment_runtime_config.py
+++ b/tests/test_deployment_runtime_config.py
@@ -114,6 +114,15 @@ def test_deploy_script_retries_startup_smoke_checks():
     assert 'smoke_check "frontend" "https://www.polyweather.top/" 15 3 5' in script
 
 
+def test_deploy_script_retries_compose_recreate_races():
+    script = (ROOT / "deploy.sh").read_text(encoding="utf-8")
+
+    assert "compose_up_retry()" in script
+    assert "removal of container .* is already in progress" in script
+    assert 'compose_up_retry "backend services" -d --no-deps polyweather_web polyweather' in script
+    assert 'compose_up_retry "frontend" -d --no-deps polyweather_frontend' in script
+
+
 def test_docker_compose_keeps_polyweather_ports_on_loopback():
     compose = (ROOT / "docker-compose.yml").read_text(encoding="utf-8")