name: Rollback # Emergency rollback for production (Hetzner k3s) and preprod (Portainer). # All images are on Scaleway registry. # # Production (k3s): # Option A — "previous": kubectl rollout undo (instant, reverts to last ReplicaSet) # Option B — "specific-version": kubectl set image to a Scaleway prod-SHA tag # # Preprod (Portainer): # Re-tags a Scaleway preprod-SHA image back to :preprod, triggers Portainer webhook. # # Run from: GitHub Actions → Workflows → Rollback → Run workflow on: workflow_dispatch: inputs: environment: description: 'Target environment' required: true type: choice options: - production - preprod strategy: description: 'Rollback strategy ("previous" = kubectl rollout undo, prod only)' required: true type: choice options: - previous - specific-version version_tag: description: 'Tag for specific-version (e.g. prod-a1b2c3d or preprod-a1b2c3d)' required: false type: string reason: description: 'Reason for rollback (audit trail)' required: true type: string env: REGISTRY: rg.fr-par.scw.cloud/weworkstudio IMAGE_BACKEND: rg.fr-par.scw.cloud/weworkstudio/xpeditis-backend IMAGE_FRONTEND: rg.fr-par.scw.cloud/weworkstudio/xpeditis-frontend K8S_NAMESPACE: xpeditis-prod jobs: # ────────────────────────────────────────── # Validate inputs # ────────────────────────────────────────── validate: name: Validate Inputs runs-on: ubuntu-latest steps: - name: Validate run: | ENV="${{ github.event.inputs.environment }}" STRATEGY="${{ github.event.inputs.strategy }}" TAG="${{ github.event.inputs.version_tag }}" if [ "$STRATEGY" = "specific-version" ] && [ -z "$TAG" ]; then echo "ERROR: version_tag is required for specific-version strategy." exit 1 fi if [ "$STRATEGY" = "specific-version" ] && [ "$ENV" = "production" ]; then if [[ ! "$TAG" =~ ^prod- ]]; then echo "ERROR: Production tag must start with 'prod-' (got: $TAG)" exit 1 fi fi if [ "$STRATEGY" = "specific-version" ] && [ "$ENV" = "preprod" ]; then if [[ ! "$TAG" =~ ^preprod- ]]; then echo "ERROR: Preprod tag must start with 'preprod-' (got: $TAG)" exit 1 fi fi echo "Validation passed." echo " Environment : $ENV" echo " Strategy : $STRATEGY" echo " Version : ${TAG:-N/A (previous)}" echo " Reason : ${{ github.event.inputs.reason }}" # ────────────────────────────────────────── # PRODUCTION ROLLBACK — k3s via kubectl # ────────────────────────────────────────── rollback-production: name: Rollback Production (k3s) runs-on: ubuntu-latest needs: validate if: github.event.inputs.environment == 'production' environment: name: production url: https://app.xpeditis.com steps: - name: Configure kubectl run: | mkdir -p ~/.kube echo "${{ secrets.HETZNER_KUBECONFIG }}" | base64 -d > ~/.kube/config chmod 600 ~/.kube/config kubectl cluster-info # ── Strategy A: kubectl rollout undo (fastest) - name: Rollback to previous version if: github.event.inputs.strategy == 'previous' run: | echo "Rolling back backend..." kubectl rollout undo deployment/xpeditis-backend -n ${{ env.K8S_NAMESPACE }} kubectl rollout status deployment/xpeditis-backend -n ${{ env.K8S_NAMESPACE }} --timeout=180s echo "Rolling back frontend..." kubectl rollout undo deployment/xpeditis-frontend -n ${{ env.K8S_NAMESPACE }} kubectl rollout status deployment/xpeditis-frontend -n ${{ env.K8S_NAMESPACE }} --timeout=180s kubectl get pods -n ${{ env.K8S_NAMESPACE }} # ── Strategy B: kubectl set image to specific Scaleway tag - name: Set up Docker Buildx (for image inspect) if: github.event.inputs.strategy == 'specific-version' uses: docker/setup-buildx-action@v3 - name: Login to Scaleway Registry if: github.event.inputs.strategy == 'specific-version' uses: docker/login-action@v3 with: registry: ${{ env.REGISTRY }} username: nologin password: ${{ secrets.REGISTRY_TOKEN }} - name: Rollback to specific version if: github.event.inputs.strategy == 'specific-version' run: | TAG="${{ github.event.inputs.version_tag }}" BACKEND_IMAGE="${{ env.IMAGE_BACKEND }}:${TAG}" FRONTEND_IMAGE="${{ env.IMAGE_FRONTEND }}:${TAG}" echo "Verifying images exist in Scaleway..." docker buildx imagetools inspect "$BACKEND_IMAGE" || \ { echo "ERROR: Backend image not found: $BACKEND_IMAGE"; exit 1; } docker buildx imagetools inspect "$FRONTEND_IMAGE" || \ { echo "ERROR: Frontend image not found: $FRONTEND_IMAGE"; exit 1; } echo "Deploying backend: $BACKEND_IMAGE" kubectl set image deployment/xpeditis-backend backend="$BACKEND_IMAGE" -n ${{ env.K8S_NAMESPACE }} kubectl rollout status deployment/xpeditis-backend -n ${{ env.K8S_NAMESPACE }} --timeout=180s echo "Deploying frontend: $FRONTEND_IMAGE" kubectl set image deployment/xpeditis-frontend frontend="$FRONTEND_IMAGE" -n ${{ env.K8S_NAMESPACE }} kubectl rollout status deployment/xpeditis-frontend -n ${{ env.K8S_NAMESPACE }} --timeout=180s kubectl get pods -n ${{ env.K8S_NAMESPACE }} - name: Show rollout history if: always() run: | echo "=== Backend rollout history ===" kubectl rollout history deployment/xpeditis-backend -n ${{ env.K8S_NAMESPACE }} || true echo "=== Frontend rollout history ===" kubectl rollout history deployment/xpeditis-frontend -n ${{ env.K8S_NAMESPACE }} || true # ────────────────────────────────────────── # PREPROD ROLLBACK — Scaleway re-tag + Portainer # ────────────────────────────────────────── rollback-preprod: name: Rollback Preprod (Portainer) runs-on: ubuntu-latest needs: validate if: github.event.inputs.environment == 'preprod' steps: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - name: Login to Scaleway Registry uses: docker/login-action@v3 with: registry: ${{ env.REGISTRY }} username: nologin password: ${{ secrets.REGISTRY_TOKEN }} - name: Verify target images exist run: | TAG="${{ github.event.inputs.version_tag }}" docker buildx imagetools inspect "${{ env.IMAGE_BACKEND }}:${TAG}" || \ { echo "ERROR: Backend image not found: $TAG"; exit 1; } docker buildx imagetools inspect "${{ env.IMAGE_FRONTEND }}:${TAG}" || \ { echo "ERROR: Frontend image not found: $TAG"; exit 1; } - name: Re-tag target version as preprod run: | TAG="${{ github.event.inputs.version_tag }}" echo "Re-tagging $TAG → preprod..." docker buildx imagetools create \ --tag ${{ env.IMAGE_BACKEND }}:preprod \ ${{ env.IMAGE_BACKEND }}:${TAG} docker buildx imagetools create \ --tag ${{ env.IMAGE_FRONTEND }}:preprod \ ${{ env.IMAGE_FRONTEND }}:${TAG} echo "Re-tag complete." - name: Trigger Backend deployment (Portainer) run: | curl -sf -X POST -H "Content-Type: application/json" \ "${{ secrets.PORTAINER_WEBHOOK_BACKEND }}" echo "Backend webhook triggered." - name: Wait for backend run: sleep 20 - name: Trigger Frontend deployment (Portainer) run: | curl -sf -X POST -H "Content-Type: application/json" \ "${{ secrets.PORTAINER_WEBHOOK_FRONTEND }}" echo "Frontend webhook triggered." # ────────────────────────────────────────── # Smoke Tests # ────────────────────────────────────────── smoke-tests: name: Smoke Tests runs-on: ubuntu-latest needs: [rollback-production, rollback-preprod] if: always() && (needs.rollback-production.result == 'success' || needs.rollback-preprod.result == 'success') steps: - name: Set health check URLs id: urls run: | if [ "${{ github.event.inputs.environment }}" = "production" ]; then echo "backend=${{ secrets.PROD_BACKEND_URL }}/api/v1/health" >> $GITHUB_OUTPUT echo "frontend=${{ secrets.PROD_FRONTEND_URL }}" >> $GITHUB_OUTPUT echo "wait=30" >> $GITHUB_OUTPUT else echo "backend=${{ secrets.PREPROD_BACKEND_URL }}/api/v1/health" >> $GITHUB_OUTPUT echo "frontend=${{ secrets.PREPROD_FRONTEND_URL }}" >> $GITHUB_OUTPUT echo "wait=60" >> $GITHUB_OUTPUT fi - name: Wait for services run: sleep ${{ steps.urls.outputs.wait }} - name: Health check — Backend run: | for i in {1..12}; do STATUS=$(curl -s -o /dev/null -w "%{http_code}" --max-time 10 \ "${{ steps.urls.outputs.backend }}" 2>/dev/null || echo "000") echo " Attempt $i: HTTP $STATUS" if [ "$STATUS" = "200" ]; then echo "Backend healthy."; exit 0; fi sleep 15 done echo "CRITICAL: Backend unhealthy after rollback." exit 1 - name: Health check — Frontend run: | for i in {1..12}; do STATUS=$(curl -s -o /dev/null -w "%{http_code}" --max-time 10 \ "${{ steps.urls.outputs.frontend }}" 2>/dev/null || echo "000") echo " Attempt $i: HTTP $STATUS" if [ "$STATUS" = "200" ]; then echo "Frontend healthy."; exit 0; fi sleep 15 done echo "CRITICAL: Frontend unhealthy after rollback." exit 1 # ────────────────────────────────────────── # Discord notification # ────────────────────────────────────────── notify: name: Notify Rollback Result runs-on: ubuntu-latest needs: [rollback-production, rollback-preprod, smoke-tests] if: always() steps: - name: Notify success if: needs.smoke-tests.result == 'success' run: | curl -s -H "Content-Type: application/json" -d '{ "embeds": [{ "title": "↩️ Rollback Successful", "color": 16776960, "fields": [ {"name": "Environment", "value": "`${{ github.event.inputs.environment }}`", "inline": true}, {"name": "Strategy", "value": "`${{ github.event.inputs.strategy }}`", "inline": true}, {"name": "Version", "value": "`${{ github.event.inputs.version_tag || 'previous' }}`", "inline": true}, {"name": "Triggered by", "value": "${{ github.actor }}", "inline": true}, {"name": "Reason", "value": "${{ github.event.inputs.reason }}", "inline": false}, {"name": "Workflow", "value": "[${{ github.workflow }}](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})", "inline": false} ], "footer": {"text": "Xpeditis CI/CD • Rollback"} }] }' ${{ secrets.DISCORD_WEBHOOK_URL }} - name: Notify failure if: needs.smoke-tests.result != 'success' run: | curl -s -H "Content-Type: application/json" -d '{ "content": "@here ROLLBACK FAILED — MANUAL INTERVENTION REQUIRED", "embeds": [{ "title": "🔴 Rollback Failed", "description": "Service may be degraded. Escalate immediately.", "color": 15158332, "fields": [ {"name": "Environment", "value": "`${{ github.event.inputs.environment }}`", "inline": true}, {"name": "Attempted version", "value": "`${{ github.event.inputs.version_tag || 'previous' }}`", "inline": true}, {"name": "Triggered by", "value": "${{ github.actor }}", "inline": true}, {"name": "Reason", "value": "${{ github.event.inputs.reason }}", "inline": false}, {"name": "Workflow", "value": "[${{ github.workflow }}](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})", "inline": false} ], "footer": {"text": "Xpeditis CI/CD • Rollback"} }] }' ${{ secrets.DISCORD_WEBHOOK_URL }}