name: Rollback # Emergency rollback — production (Hetzner k3s) and preprod (Portainer). # # Production strategies: # previous — kubectl rollout undo (fastest, reverts to previous ReplicaSet) # specific-version — kubectl set image to a specific prod-SHA tag # # Preprod strategy: # Re-tags a preprod-SHA image back to :preprod, triggers Portainer webhook. # # Secrets required: # REGISTRY_TOKEN — Scaleway registry # HETZNER_KUBECONFIG — base64 kubeconfig (production only) # PORTAINER_WEBHOOK_BACKEND — Portainer webhook preprod backend # PORTAINER_WEBHOOK_FRONTEND — Portainer webhook preprod frontend # PROD_BACKEND_URL — https://api.xpeditis.com # PROD_FRONTEND_URL — https://app.xpeditis.com # PREPROD_BACKEND_URL — https://api.preprod.xpeditis.com # PREPROD_FRONTEND_URL — https://preprod.xpeditis.com # DISCORD_WEBHOOK_URL on: workflow_dispatch: inputs: environment: description: 'Target environment' required: true type: choice options: [production, preprod] strategy: description: 'Strategy (production only — "previous" = instant kubectl undo)' required: true type: choice options: [previous, specific-version] version_tag: description: 'Tag for specific-version (e.g. prod-a1b2c3d or preprod-a1b2c3d)' required: false type: string reason: description: 'Reason (audit trail)' required: true type: string env: REGISTRY: rg.fr-par.scw.cloud/weworkstudio K8S_NAMESPACE: xpeditis-prod jobs: validate: name: Validate Inputs runs-on: ubuntu-latest steps: - name: Check inputs run: | ENV="${{ github.event.inputs.environment }}" STRATEGY="${{ github.event.inputs.strategy }}" TAG="${{ github.event.inputs.version_tag }}" if [ "$STRATEGY" = "specific-version" ] && [ -z "$TAG" ]; then echo "ERROR: version_tag is required for specific-version strategy." exit 1 fi if [ "$ENV" = "production" ] && [ "$STRATEGY" = "specific-version" ]; then if [[ ! "$TAG" =~ ^prod- ]]; then echo "ERROR: Production tag must start with 'prod-' (got: $TAG)" exit 1 fi fi if [ "$ENV" = "preprod" ]; then if [[ ! "$TAG" =~ ^preprod- ]]; then echo "ERROR: Preprod tag must start with 'preprod-' (got: $TAG)" exit 1 fi fi echo "Validated — env: $ENV | strategy: $STRATEGY | tag: ${TAG:-N/A} | reason: ${{ github.event.inputs.reason }}" # ── Production rollback via kubectl ────────────────────────────────── rollback-production: name: Rollback Production runs-on: ubuntu-latest needs: validate if: github.event.inputs.environment == 'production' environment: name: production url: https://app.xpeditis.com steps: - name: Configure kubectl run: | mkdir -p ~/.kube echo "${{ secrets.HETZNER_KUBECONFIG }}" | base64 -d > ~/.kube/config chmod 600 ~/.kube/config kubectl cluster-info - name: Rollback — previous version if: github.event.inputs.strategy == 'previous' run: | kubectl rollout undo deployment/xpeditis-backend -n ${{ env.K8S_NAMESPACE }} kubectl rollout status deployment/xpeditis-backend -n ${{ env.K8S_NAMESPACE }} --timeout=180s kubectl rollout undo deployment/xpeditis-frontend -n ${{ env.K8S_NAMESPACE }} kubectl rollout status deployment/xpeditis-frontend -n ${{ env.K8S_NAMESPACE }} --timeout=180s kubectl get pods -n ${{ env.K8S_NAMESPACE }} - name: Login to Scaleway (for image verification) if: github.event.inputs.strategy == 'specific-version' uses: docker/login-action@v3 with: registry: ${{ env.REGISTRY }} username: nologin password: ${{ secrets.REGISTRY_TOKEN }} - uses: docker/setup-buildx-action@v3 if: github.event.inputs.strategy == 'specific-version' - name: Rollback — specific version if: github.event.inputs.strategy == 'specific-version' run: | TAG="${{ github.event.inputs.version_tag }}" BACKEND="${{ env.REGISTRY }}/xpeditis-backend:${TAG}" FRONTEND="${{ env.REGISTRY }}/xpeditis-frontend:${TAG}" echo "Verifying images exist..." docker buildx imagetools inspect "$BACKEND" || { echo "ERROR: $BACKEND not found"; exit 1; } docker buildx imagetools inspect "$FRONTEND" || { echo "ERROR: $FRONTEND not found"; exit 1; } kubectl set image deployment/xpeditis-backend backend="$BACKEND" -n ${{ env.K8S_NAMESPACE }} kubectl rollout status deployment/xpeditis-backend -n ${{ env.K8S_NAMESPACE }} --timeout=180s kubectl set image deployment/xpeditis-frontend frontend="$FRONTEND" -n ${{ env.K8S_NAMESPACE }} kubectl rollout status deployment/xpeditis-frontend -n ${{ env.K8S_NAMESPACE }} --timeout=180s kubectl get pods -n ${{ env.K8S_NAMESPACE }} - name: Rollout history if: always() run: | kubectl rollout history deployment/xpeditis-backend -n ${{ env.K8S_NAMESPACE }} || true kubectl rollout history deployment/xpeditis-frontend -n ${{ env.K8S_NAMESPACE }} || true # ── Preprod rollback via Portainer ─────────────────────────────────── rollback-preprod: name: Rollback Preprod runs-on: ubuntu-latest needs: validate if: github.event.inputs.environment == 'preprod' steps: - uses: docker/setup-buildx-action@v3 - uses: docker/login-action@v3 with: registry: ${{ env.REGISTRY }} username: nologin password: ${{ secrets.REGISTRY_TOKEN }} - name: Verify target image exists run: | TAG="${{ github.event.inputs.version_tag }}" docker buildx imagetools inspect "${{ env.REGISTRY }}/xpeditis-backend:${TAG}" || \ { echo "ERROR: backend image not found: $TAG"; exit 1; } docker buildx imagetools inspect "${{ env.REGISTRY }}/xpeditis-frontend:${TAG}" || \ { echo "ERROR: frontend image not found: $TAG"; exit 1; } - name: Re-tag as preprod run: | TAG="${{ github.event.inputs.version_tag }}" docker buildx imagetools create \ --tag ${{ env.REGISTRY }}/xpeditis-backend:preprod \ ${{ env.REGISTRY }}/xpeditis-backend:${TAG} docker buildx imagetools create \ --tag ${{ env.REGISTRY }}/xpeditis-frontend:preprod \ ${{ env.REGISTRY }}/xpeditis-frontend:${TAG} - name: Deploy backend (Portainer) run: curl -sf -X POST "${{ secrets.PORTAINER_WEBHOOK_BACKEND }}" - run: sleep 20 - name: Deploy frontend (Portainer) run: curl -sf -X POST "${{ secrets.PORTAINER_WEBHOOK_FRONTEND }}" # ── Smoke Tests ─────────────────────────────────────────────────────── smoke-tests: name: Smoke Tests Post-Rollback runs-on: ubuntu-latest needs: [rollback-production, rollback-preprod] if: always() && (needs.rollback-production.result == 'success' || needs.rollback-preprod.result == 'success') steps: - name: Set URLs id: urls run: | if [ "${{ github.event.inputs.environment }}" = "production" ]; then echo "backend=${{ secrets.PROD_BACKEND_URL }}/api/v1/health" >> $GITHUB_OUTPUT echo "frontend=${{ secrets.PROD_FRONTEND_URL }}" >> $GITHUB_OUTPUT echo "wait=30" >> $GITHUB_OUTPUT else echo "backend=${{ secrets.PREPROD_BACKEND_URL }}/api/v1/health" >> $GITHUB_OUTPUT echo "frontend=${{ secrets.PREPROD_FRONTEND_URL }}" >> $GITHUB_OUTPUT echo "wait=60" >> $GITHUB_OUTPUT fi - run: sleep ${{ steps.urls.outputs.wait }} - name: Health — Backend run: | for i in {1..12}; do STATUS=$(curl -s -o /dev/null -w "%{http_code}" --max-time 10 \ "${{ steps.urls.outputs.backend }}" 2>/dev/null || echo "000") echo " Attempt $i: HTTP $STATUS" if [ "$STATUS" = "200" ]; then echo "Backend OK."; exit 0; fi sleep 15 done echo "Backend unhealthy after rollback." exit 1 - name: Health — Frontend run: | for i in {1..12}; do STATUS=$(curl -s -o /dev/null -w "%{http_code}" --max-time 10 \ "${{ steps.urls.outputs.frontend }}" 2>/dev/null || echo "000") echo " Attempt $i: HTTP $STATUS" if [ "$STATUS" = "200" ]; then echo "Frontend OK."; exit 0; fi sleep 15 done echo "Frontend unhealthy after rollback." exit 1 # ── Notifications ───────────────────────────────────────────────────── notify: name: Notify runs-on: ubuntu-latest needs: [rollback-production, rollback-preprod, smoke-tests] if: always() steps: - name: Success if: needs.smoke-tests.result == 'success' run: | curl -s -H "Content-Type: application/json" -d '{ "embeds": [{ "title": "↩️ Rollback Successful", "color": 16776960, "fields": [ {"name": "Environment", "value": "`${{ github.event.inputs.environment }}`", "inline": true}, {"name": "Strategy", "value": "`${{ github.event.inputs.strategy }}`", "inline": true}, {"name": "Version", "value": "`${{ github.event.inputs.version_tag || 'previous' }}`", "inline": true}, {"name": "By", "value": "${{ github.actor }}", "inline": true}, {"name": "Reason", "value": "${{ github.event.inputs.reason }}", "inline": false} ], "footer": {"text": "Xpeditis CI/CD • Rollback"} }] }' ${{ secrets.DISCORD_WEBHOOK_URL }} - name: Failure if: needs.smoke-tests.result != 'success' run: | curl -s -H "Content-Type: application/json" -d '{ "content": "@here ROLLBACK FAILED — MANUAL INTERVENTION REQUIRED", "embeds": [{ "title": "🔴 Rollback Failed", "color": 15158332, "fields": [ {"name": "Environment", "value": "`${{ github.event.inputs.environment }}`", "inline": true}, {"name": "Attempted", "value": "`${{ github.event.inputs.version_tag || 'previous' }}`", "inline": true}, {"name": "By", "value": "${{ github.actor }}", "inline": true}, {"name": "Reason", "value": "${{ github.event.inputs.reason }}", "inline": false} ], "footer": {"text": "Xpeditis CI/CD • Rollback"} }] }' ${{ secrets.DISCORD_WEBHOOK_URL }}