diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 543deb36..155640b5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -122,6 +122,9 @@ jobs: needs: [build-and-push] if: github.event_name == 'push' && github.ref == 'refs/heads/main' runs-on: ubuntu-latest + concurrency: + group: polyweather-production-deploy + cancel-in-progress: false steps: - name: Checkout uses: actions/checkout@v4 diff --git a/deploy.sh b/deploy.sh index 6a64f69f..7cdf49d5 100644 --- a/deploy.sh +++ b/deploy.sh @@ -5,6 +5,14 @@ GHCR_PAT="$1" NEW_TAG="${2:-latest}" TAG_FILE="/var/lib/polyweather/.current_tag" COMPOSE_DIR="/root/PolyWeather" +LOCK_FILE="${POLYWEATHER_DEPLOY_LOCK_FILE:-/var/lock/polyweather-deploy.lock}" + +mkdir -p "$(dirname "$LOCK_FILE")" +exec 9>"$LOCK_FILE" +if ! flock -n 9; then + echo "❌ Another PolyWeather deploy is already running" + exit 1 +fi echo "$GHCR_PAT" | docker login ghcr.io -u yangyuan-zhen --password-stdin @@ -17,6 +25,18 @@ if [ -f "$TAG_FILE" ]; then echo "Previous tag: $PREVIOUS_TAG" fi +rollback_to_previous() { + if [ -n "$PREVIOUS_TAG" ]; then + echo "Rolling back to $PREVIOUS_TAG..." + export IMAGE_TAG="$PREVIOUS_TAG" + docker compose pull + docker compose up -d + echo "✅ Rolled back to $PREVIOUS_TAG" + else + echo "⚠️ No previous tag to rollback to" + fi +} + export IMAGE_TAG="$NEW_TAG" pull_ok=0 for pull_attempt in $(seq 1 6); do @@ -28,7 +48,6 @@ if [ "$pull_ok" != "1" ]; then echo "❌ Image pull failed after retries" exit 1 fi -docker compose up -d smoke_check() { local name="$1" @@ -52,16 +71,81 @@ smoke_check() { return 1 } -# Wait for backend to be ready (retry up to 150s) +wait_for_local_service() { + local name="$1" + local url="$2" + local timeout="${3:-5}" + local attempts="${4:-30}" + local delay="${5:-2}" + + for i in $(seq 1 "$attempts"); do + if curl -fsSo /dev/null --max-time "$timeout" "$url"; then + echo "✅ $name ready after attempt $i/$attempts" + return 0 + fi + if [ "$i" != "$attempts" ]; then + echo " $name warming $i/$attempts..." + sleep "$delay" + fi + done + + echo "❌ $name did not become ready" + return 1 +} + +warm_public_route() { + local name="$1" + local url="$2" + local timeout="${3:-15}" + local attempts="${4:-3}" + local delay="${5:-2}" + + for i in $(seq 1 "$attempts"); do + if curl -fsSo /dev/null --max-time "$timeout" "$url"; then + echo "✅ warmed $name" + return 0 + fi + if [ "$i" != "$attempts" ]; then + echo " warm $name retry $i/$attempts..." + sleep "$delay" + fi + done + + echo "⚠️ warm $name failed" + return 0 +} + +echo "Updating Redis dependency..." +docker compose up -d polyweather_redis + +echo "Updating backend services..." +docker compose up -d --no-deps polyweather_web polyweather + echo "Waiting for backend..." -for i in $(seq 1 30); do - sleep 5 - if curl -fsSo /dev/null --max-time 5 "https://api.polyweather.top/healthz"; then - echo "✅ healthz ready after ${i}x5s" - break - fi - echo " retry $i/30..." -done +wait_for_local_service "backend healthz" "http://127.0.0.1:8000/healthz" 5 30 5 || FAILED_BACKEND=1 +FAILED_BACKEND="${FAILED_BACKEND:-0}" +if [ "$FAILED_BACKEND" = "1" ]; then + echo "❌ Backend did not become healthy" + rollback_to_previous + exit 1 +fi + +echo "Updating frontend..." +docker compose up -d --no-deps polyweather_frontend + +echo "Waiting for frontend..." +wait_for_local_service "frontend root" "http://127.0.0.1:3001/" 5 40 2 || FAILED_FRONTEND=1 +wait_for_local_service "frontend terminal" "http://127.0.0.1:3001/terminal" 10 20 2 || FAILED_FRONTEND=1 +FAILED_FRONTEND="${FAILED_FRONTEND:-0}" +if [ "$FAILED_FRONTEND" = "1" ]; then + echo "❌ Frontend did not become healthy" + rollback_to_previous + exit 1 +fi + +warm_public_route "terminal" "https://polyweather.top/terminal" 20 4 3 +warm_public_route "auth snapshot" "https://polyweather.top/api/auth/me?prefer_snapshot=1" 10 3 2 +warm_public_route "cities" "https://polyweather.top/api/cities" 20 3 2 FAILED=0 smoke_check "healthz" "https://api.polyweather.top/healthz" 15 3 5 || FAILED=1 @@ -70,14 +154,7 @@ smoke_check "frontend" "https://www.polyweather.top/" 15 3 5 || FAILED=1 if [ "$FAILED" = "1" ]; then echo "❌ Smoke tests failed. Rolling back..." - if [ -n "$PREVIOUS_TAG" ]; then - export IMAGE_TAG="$PREVIOUS_TAG" - docker compose pull - docker compose up -d - echo "✅ Rolled back to $PREVIOUS_TAG" - else - echo "⚠️ No previous tag to rollback to" - fi + rollback_to_previous exit 1 fi diff --git a/frontend/components/dashboard/scan-terminal/__tests__/deployStability.test.ts b/frontend/components/dashboard/scan-terminal/__tests__/deployStability.test.ts new file mode 100644 index 00000000..c060719c --- /dev/null +++ b/frontend/components/dashboard/scan-terminal/__tests__/deployStability.test.ts @@ -0,0 +1,37 @@ +import fs from "node:fs"; +import path from "node:path"; + +function assert(condition: unknown, message: string) { + if (!condition) throw new Error(message); +} + +function readRepoFile(...parts: string[]) { + return fs.readFileSync(path.resolve(process.cwd(), "..", ...parts), "utf8"); +} + +export function runTests() { + const workflow = readRepoFile(".github", "workflows", "ci.yml"); + const deployScript = readRepoFile("deploy.sh"); + + assert( + workflow.includes("group: polyweather-production-deploy") && + workflow.includes("cancel-in-progress: false"), + "production deploy job must be serialized so overlapping pushes cannot restart the VPS concurrently", + ); + assert( + deployScript.includes("POLYWEATHER_DEPLOY_LOCK_FILE") && + deployScript.includes("flock -n"), + "VPS deploy script must take a host-level deploy lock for manual or retried deploys", + ); + assert( + deployScript.includes("wait_for_local_service") && + deployScript.includes("http://127.0.0.1:3001/terminal"), + "deploy script must wait for the local frontend before relying on Cloudflare/public smoke checks", + ); + assert( + deployScript.includes("warm_public_route") && + deployScript.includes("https://polyweather.top/terminal") && + deployScript.includes("https://polyweather.top/api/auth/me?prefer_snapshot=1"), + "deploy script must warm terminal and auth snapshot routes after container replacement", + ); +}