Some checks failed
CD Production (Hetzner k3s) / Deployment Summary (push) Blocked by required conditions
CD Production (Hetzner k3s) / Notify Success (push) Blocked by required conditions
CD Production (Hetzner k3s) / Notify Failure (push) Blocked by required conditions
CD Production (Hetzner k3s) / Deploy to k3s (xpeditis-prod) (push) Blocked by required conditions
CD Production (Hetzner k3s) / Smoke Tests (push) Blocked by required conditions
Security Audit / npm audit (push) Failing after 7s
Security Audit / Dependency Review (push) Has been skipped
CD Production (Hetzner k3s) / Promote Images (preprod → prod) (push) Has been cancelled
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
318 lines
13 KiB
YAML
318 lines
13 KiB
YAML
name: Rollback
|
|
|
|
# Emergency rollback for production (Hetzner k3s) and preprod (Portainer).
|
|
# All images are on Scaleway registry.
|
|
#
|
|
# Production (k3s):
|
|
# Option A — "previous": kubectl rollout undo (instant, reverts to last ReplicaSet)
|
|
# Option B — "specific-version": kubectl set image to a Scaleway prod-SHA tag
|
|
#
|
|
# Preprod (Portainer):
|
|
# Re-tags a Scaleway preprod-SHA image back to :preprod, triggers Portainer webhook.
|
|
#
|
|
# Run from: GitHub Actions → Workflows → Rollback → Run workflow
|
|
|
|
on:
|
|
workflow_dispatch:
|
|
inputs:
|
|
environment:
|
|
description: 'Target environment'
|
|
required: true
|
|
type: choice
|
|
options:
|
|
- production
|
|
- preprod
|
|
strategy:
|
|
description: 'Rollback strategy ("previous" = kubectl rollout undo, prod only)'
|
|
required: true
|
|
type: choice
|
|
options:
|
|
- previous
|
|
- specific-version
|
|
version_tag:
|
|
description: 'Tag for specific-version (e.g. prod-a1b2c3d or preprod-a1b2c3d)'
|
|
required: false
|
|
type: string
|
|
reason:
|
|
description: 'Reason for rollback (audit trail)'
|
|
required: true
|
|
type: string
|
|
|
|
env:
|
|
REGISTRY: rg.fr-par.scw.cloud/weworkstudio
|
|
IMAGE_BACKEND: rg.fr-par.scw.cloud/weworkstudio/xpeditis-backend
|
|
IMAGE_FRONTEND: rg.fr-par.scw.cloud/weworkstudio/xpeditis-frontend
|
|
K8S_NAMESPACE: xpeditis-prod
|
|
|
|
jobs:
|
|
# ──────────────────────────────────────────
|
|
# Validate inputs
|
|
# ──────────────────────────────────────────
|
|
validate:
|
|
name: Validate Inputs
|
|
runs-on: ubuntu-latest
|
|
|
|
steps:
|
|
- name: Validate
|
|
run: |
|
|
ENV="${{ github.event.inputs.environment }}"
|
|
STRATEGY="${{ github.event.inputs.strategy }}"
|
|
TAG="${{ github.event.inputs.version_tag }}"
|
|
|
|
if [ "$STRATEGY" = "specific-version" ] && [ -z "$TAG" ]; then
|
|
echo "ERROR: version_tag is required for specific-version strategy."
|
|
exit 1
|
|
fi
|
|
|
|
if [ "$STRATEGY" = "specific-version" ] && [ "$ENV" = "production" ]; then
|
|
if [[ ! "$TAG" =~ ^prod- ]]; then
|
|
echo "ERROR: Production tag must start with 'prod-' (got: $TAG)"
|
|
exit 1
|
|
fi
|
|
fi
|
|
|
|
if [ "$STRATEGY" = "specific-version" ] && [ "$ENV" = "preprod" ]; then
|
|
if [[ ! "$TAG" =~ ^preprod- ]]; then
|
|
echo "ERROR: Preprod tag must start with 'preprod-' (got: $TAG)"
|
|
exit 1
|
|
fi
|
|
fi
|
|
|
|
echo "Validation passed."
|
|
echo " Environment : $ENV"
|
|
echo " Strategy : $STRATEGY"
|
|
echo " Version : ${TAG:-N/A (previous)}"
|
|
echo " Reason : ${{ github.event.inputs.reason }}"
|
|
|
|
# ──────────────────────────────────────────
|
|
# PRODUCTION ROLLBACK — k3s via kubectl
|
|
# ──────────────────────────────────────────
|
|
rollback-production:
|
|
name: Rollback Production (k3s)
|
|
runs-on: ubuntu-latest
|
|
needs: validate
|
|
if: github.event.inputs.environment == 'production'
|
|
environment:
|
|
name: production
|
|
url: https://app.xpeditis.com
|
|
|
|
steps:
|
|
- name: Configure kubectl
|
|
run: |
|
|
mkdir -p ~/.kube
|
|
echo "${{ secrets.HETZNER_KUBECONFIG }}" | base64 -d > ~/.kube/config
|
|
chmod 600 ~/.kube/config
|
|
kubectl cluster-info
|
|
|
|
# ── Strategy A: kubectl rollout undo (fastest)
|
|
- name: Rollback to previous version
|
|
if: github.event.inputs.strategy == 'previous'
|
|
run: |
|
|
echo "Rolling back backend..."
|
|
kubectl rollout undo deployment/xpeditis-backend -n ${{ env.K8S_NAMESPACE }}
|
|
kubectl rollout status deployment/xpeditis-backend -n ${{ env.K8S_NAMESPACE }} --timeout=180s
|
|
|
|
echo "Rolling back frontend..."
|
|
kubectl rollout undo deployment/xpeditis-frontend -n ${{ env.K8S_NAMESPACE }}
|
|
kubectl rollout status deployment/xpeditis-frontend -n ${{ env.K8S_NAMESPACE }} --timeout=180s
|
|
|
|
kubectl get pods -n ${{ env.K8S_NAMESPACE }}
|
|
|
|
# ── Strategy B: kubectl set image to specific Scaleway tag
|
|
- name: Set up Docker Buildx (for image inspect)
|
|
if: github.event.inputs.strategy == 'specific-version'
|
|
uses: docker/setup-buildx-action@v3
|
|
|
|
- name: Login to Scaleway Registry
|
|
if: github.event.inputs.strategy == 'specific-version'
|
|
uses: docker/login-action@v3
|
|
with:
|
|
registry: ${{ env.REGISTRY }}
|
|
username: nologin
|
|
password: ${{ secrets.REGISTRY_TOKEN }}
|
|
|
|
- name: Rollback to specific version
|
|
if: github.event.inputs.strategy == 'specific-version'
|
|
run: |
|
|
TAG="${{ github.event.inputs.version_tag }}"
|
|
BACKEND_IMAGE="${{ env.IMAGE_BACKEND }}:${TAG}"
|
|
FRONTEND_IMAGE="${{ env.IMAGE_FRONTEND }}:${TAG}"
|
|
|
|
echo "Verifying images exist in Scaleway..."
|
|
docker buildx imagetools inspect "$BACKEND_IMAGE" || \
|
|
{ echo "ERROR: Backend image not found: $BACKEND_IMAGE"; exit 1; }
|
|
docker buildx imagetools inspect "$FRONTEND_IMAGE" || \
|
|
{ echo "ERROR: Frontend image not found: $FRONTEND_IMAGE"; exit 1; }
|
|
|
|
echo "Deploying backend: $BACKEND_IMAGE"
|
|
kubectl set image deployment/xpeditis-backend backend="$BACKEND_IMAGE" -n ${{ env.K8S_NAMESPACE }}
|
|
kubectl rollout status deployment/xpeditis-backend -n ${{ env.K8S_NAMESPACE }} --timeout=180s
|
|
|
|
echo "Deploying frontend: $FRONTEND_IMAGE"
|
|
kubectl set image deployment/xpeditis-frontend frontend="$FRONTEND_IMAGE" -n ${{ env.K8S_NAMESPACE }}
|
|
kubectl rollout status deployment/xpeditis-frontend -n ${{ env.K8S_NAMESPACE }} --timeout=180s
|
|
|
|
kubectl get pods -n ${{ env.K8S_NAMESPACE }}
|
|
|
|
- name: Show rollout history
|
|
if: always()
|
|
run: |
|
|
echo "=== Backend rollout history ==="
|
|
kubectl rollout history deployment/xpeditis-backend -n ${{ env.K8S_NAMESPACE }} || true
|
|
echo "=== Frontend rollout history ==="
|
|
kubectl rollout history deployment/xpeditis-frontend -n ${{ env.K8S_NAMESPACE }} || true
|
|
|
|
# ──────────────────────────────────────────
|
|
# PREPROD ROLLBACK — Scaleway re-tag + Portainer
|
|
# ──────────────────────────────────────────
|
|
rollback-preprod:
|
|
name: Rollback Preprod (Portainer)
|
|
runs-on: ubuntu-latest
|
|
needs: validate
|
|
if: github.event.inputs.environment == 'preprod'
|
|
|
|
steps:
|
|
- name: Set up Docker Buildx
|
|
uses: docker/setup-buildx-action@v3
|
|
|
|
- name: Login to Scaleway Registry
|
|
uses: docker/login-action@v3
|
|
with:
|
|
registry: ${{ env.REGISTRY }}
|
|
username: nologin
|
|
password: ${{ secrets.REGISTRY_TOKEN }}
|
|
|
|
- name: Verify target images exist
|
|
run: |
|
|
TAG="${{ github.event.inputs.version_tag }}"
|
|
docker buildx imagetools inspect "${{ env.IMAGE_BACKEND }}:${TAG}" || \
|
|
{ echo "ERROR: Backend image not found: $TAG"; exit 1; }
|
|
docker buildx imagetools inspect "${{ env.IMAGE_FRONTEND }}:${TAG}" || \
|
|
{ echo "ERROR: Frontend image not found: $TAG"; exit 1; }
|
|
|
|
- name: Re-tag target version as preprod
|
|
run: |
|
|
TAG="${{ github.event.inputs.version_tag }}"
|
|
echo "Re-tagging $TAG → preprod..."
|
|
docker buildx imagetools create \
|
|
--tag ${{ env.IMAGE_BACKEND }}:preprod \
|
|
${{ env.IMAGE_BACKEND }}:${TAG}
|
|
docker buildx imagetools create \
|
|
--tag ${{ env.IMAGE_FRONTEND }}:preprod \
|
|
${{ env.IMAGE_FRONTEND }}:${TAG}
|
|
echo "Re-tag complete."
|
|
|
|
- name: Trigger Backend deployment (Portainer)
|
|
run: |
|
|
curl -sf -X POST -H "Content-Type: application/json" \
|
|
"${{ secrets.PORTAINER_WEBHOOK_BACKEND }}"
|
|
echo "Backend webhook triggered."
|
|
|
|
- name: Wait for backend
|
|
run: sleep 20
|
|
|
|
- name: Trigger Frontend deployment (Portainer)
|
|
run: |
|
|
curl -sf -X POST -H "Content-Type: application/json" \
|
|
"${{ secrets.PORTAINER_WEBHOOK_FRONTEND }}"
|
|
echo "Frontend webhook triggered."
|
|
|
|
# ──────────────────────────────────────────
|
|
# Smoke Tests
|
|
# ──────────────────────────────────────────
|
|
smoke-tests:
|
|
name: Smoke Tests
|
|
runs-on: ubuntu-latest
|
|
needs: [rollback-production, rollback-preprod]
|
|
if: always() && (needs.rollback-production.result == 'success' || needs.rollback-preprod.result == 'success')
|
|
|
|
steps:
|
|
- name: Set health check URLs
|
|
id: urls
|
|
run: |
|
|
if [ "${{ github.event.inputs.environment }}" = "production" ]; then
|
|
echo "backend=${{ secrets.PROD_BACKEND_URL }}/api/v1/health" >> $GITHUB_OUTPUT
|
|
echo "frontend=${{ secrets.PROD_FRONTEND_URL }}" >> $GITHUB_OUTPUT
|
|
echo "wait=30" >> $GITHUB_OUTPUT
|
|
else
|
|
echo "backend=${{ secrets.PREPROD_BACKEND_URL }}/api/v1/health" >> $GITHUB_OUTPUT
|
|
echo "frontend=${{ secrets.PREPROD_FRONTEND_URL }}" >> $GITHUB_OUTPUT
|
|
echo "wait=60" >> $GITHUB_OUTPUT
|
|
fi
|
|
|
|
- name: Wait for services
|
|
run: sleep ${{ steps.urls.outputs.wait }}
|
|
|
|
- name: Health check — Backend
|
|
run: |
|
|
for i in {1..12}; do
|
|
STATUS=$(curl -s -o /dev/null -w "%{http_code}" --max-time 10 \
|
|
"${{ steps.urls.outputs.backend }}" 2>/dev/null || echo "000")
|
|
echo " Attempt $i: HTTP $STATUS"
|
|
if [ "$STATUS" = "200" ]; then echo "Backend healthy."; exit 0; fi
|
|
sleep 15
|
|
done
|
|
echo "CRITICAL: Backend unhealthy after rollback."
|
|
exit 1
|
|
|
|
- name: Health check — Frontend
|
|
run: |
|
|
for i in {1..12}; do
|
|
STATUS=$(curl -s -o /dev/null -w "%{http_code}" --max-time 10 \
|
|
"${{ steps.urls.outputs.frontend }}" 2>/dev/null || echo "000")
|
|
echo " Attempt $i: HTTP $STATUS"
|
|
if [ "$STATUS" = "200" ]; then echo "Frontend healthy."; exit 0; fi
|
|
sleep 15
|
|
done
|
|
echo "CRITICAL: Frontend unhealthy after rollback."
|
|
exit 1
|
|
|
|
# ──────────────────────────────────────────
|
|
# Discord notification
|
|
# ──────────────────────────────────────────
|
|
notify:
|
|
name: Notify Rollback Result
|
|
runs-on: ubuntu-latest
|
|
needs: [rollback-production, rollback-preprod, smoke-tests]
|
|
if: always()
|
|
|
|
steps:
|
|
- name: Notify success
|
|
if: needs.smoke-tests.result == 'success'
|
|
run: |
|
|
curl -s -H "Content-Type: application/json" -d '{
|
|
"embeds": [{
|
|
"title": "↩️ Rollback Successful",
|
|
"color": 16776960,
|
|
"fields": [
|
|
{"name": "Environment", "value": "`${{ github.event.inputs.environment }}`", "inline": true},
|
|
{"name": "Strategy", "value": "`${{ github.event.inputs.strategy }}`", "inline": true},
|
|
{"name": "Version", "value": "`${{ github.event.inputs.version_tag || 'previous' }}`", "inline": true},
|
|
{"name": "Triggered by", "value": "${{ github.actor }}", "inline": true},
|
|
{"name": "Reason", "value": "${{ github.event.inputs.reason }}", "inline": false},
|
|
{"name": "Workflow", "value": "[${{ github.workflow }}](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})", "inline": false}
|
|
],
|
|
"footer": {"text": "Xpeditis CI/CD • Rollback"}
|
|
}]
|
|
}' ${{ secrets.DISCORD_WEBHOOK_URL }}
|
|
|
|
- name: Notify failure
|
|
if: needs.smoke-tests.result != 'success'
|
|
run: |
|
|
curl -s -H "Content-Type: application/json" -d '{
|
|
"content": "@here ROLLBACK FAILED — MANUAL INTERVENTION REQUIRED",
|
|
"embeds": [{
|
|
"title": "🔴 Rollback Failed",
|
|
"description": "Service may be degraded. Escalate immediately.",
|
|
"color": 15158332,
|
|
"fields": [
|
|
{"name": "Environment", "value": "`${{ github.event.inputs.environment }}`", "inline": true},
|
|
{"name": "Attempted version", "value": "`${{ github.event.inputs.version_tag || 'previous' }}`", "inline": true},
|
|
{"name": "Triggered by", "value": "${{ github.actor }}", "inline": true},
|
|
{"name": "Reason", "value": "${{ github.event.inputs.reason }}", "inline": false},
|
|
{"name": "Workflow", "value": "[${{ github.workflow }}](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})", "inline": false}
|
|
],
|
|
"footer": {"text": "Xpeditis CI/CD • Rollback"}
|
|
}]
|
|
}' ${{ secrets.DISCORD_WEBHOOK_URL }}
|