711aca5f40
Some checks failed
Dev CI / Frontend — Lint & Type-check (push) Failing after 6m12s
Dev CI / Frontend — Unit Tests (push) Has been skipped
Dev CI / Backend — Lint (push) Successful in 10m24s
Dev CI / Backend — Unit Tests (push) Failing after 5m30s
Dev CI / Notify Failure (push) Has been skipped
270 lines
11 KiB
YAML
270 lines
11 KiB
YAML
name: Rollback
|
|
|
|
# Emergency rollback — production (Hetzner k3s) and preprod (Portainer).
|
|
#
|
|
# Production strategies:
|
|
# previous — kubectl rollout undo (fastest, reverts to previous ReplicaSet)
|
|
# specific-version — kubectl set image to a specific prod-SHA tag
|
|
#
|
|
# Preprod strategy:
|
|
# Re-tags a preprod-SHA image back to :preprod, triggers Portainer webhook.
|
|
#
|
|
# Secrets required:
|
|
# REGISTRY_TOKEN — Scaleway registry
|
|
# HETZNER_KUBECONFIG — base64 kubeconfig (production only)
|
|
# PORTAINER_WEBHOOK_BACKEND — Portainer webhook preprod backend
|
|
# PORTAINER_WEBHOOK_FRONTEND — Portainer webhook preprod frontend
|
|
# PROD_BACKEND_URL — https://api.xpeditis.com
|
|
# PROD_FRONTEND_URL — https://app.xpeditis.com
|
|
# PREPROD_BACKEND_URL — https://api.preprod.xpeditis.com
|
|
# PREPROD_FRONTEND_URL — https://preprod.xpeditis.com
|
|
# DISCORD_WEBHOOK_URL
|
|
|
|
on:
|
|
workflow_dispatch:
|
|
inputs:
|
|
environment:
|
|
description: 'Target environment'
|
|
required: true
|
|
type: choice
|
|
options: [production, preprod]
|
|
strategy:
|
|
description: 'Strategy (production only — "previous" = instant kubectl undo)'
|
|
required: true
|
|
type: choice
|
|
options: [previous, specific-version]
|
|
version_tag:
|
|
description: 'Tag for specific-version (e.g. prod-a1b2c3d or preprod-a1b2c3d)'
|
|
required: false
|
|
type: string
|
|
reason:
|
|
description: 'Reason (audit trail)'
|
|
required: true
|
|
type: string
|
|
|
|
env:
|
|
REGISTRY: rg.fr-par.scw.cloud/weworkstudio
|
|
K8S_NAMESPACE: xpeditis-prod
|
|
|
|
jobs:
|
|
validate:
|
|
name: Validate Inputs
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- name: Check inputs
|
|
run: |
|
|
ENV="${{ github.event.inputs.environment }}"
|
|
STRATEGY="${{ github.event.inputs.strategy }}"
|
|
TAG="${{ github.event.inputs.version_tag }}"
|
|
|
|
if [ "$STRATEGY" = "specific-version" ] && [ -z "$TAG" ]; then
|
|
echo "ERROR: version_tag is required for specific-version strategy."
|
|
exit 1
|
|
fi
|
|
|
|
if [ "$ENV" = "production" ] && [ "$STRATEGY" = "specific-version" ]; then
|
|
if [[ ! "$TAG" =~ ^prod- ]]; then
|
|
echo "ERROR: Production tag must start with 'prod-' (got: $TAG)"
|
|
exit 1
|
|
fi
|
|
fi
|
|
|
|
if [ "$ENV" = "preprod" ]; then
|
|
if [[ ! "$TAG" =~ ^preprod- ]]; then
|
|
echo "ERROR: Preprod tag must start with 'preprod-' (got: $TAG)"
|
|
exit 1
|
|
fi
|
|
fi
|
|
|
|
echo "Validated — env: $ENV | strategy: $STRATEGY | tag: ${TAG:-N/A} | reason: ${{ github.event.inputs.reason }}"
|
|
|
|
# ── Production rollback via kubectl ──────────────────────────────────
|
|
rollback-production:
|
|
name: Rollback Production
|
|
runs-on: ubuntu-latest
|
|
needs: validate
|
|
if: github.event.inputs.environment == 'production'
|
|
environment:
|
|
name: production
|
|
url: https://app.xpeditis.com
|
|
steps:
|
|
- name: Configure kubectl
|
|
run: |
|
|
mkdir -p ~/.kube
|
|
echo "${{ secrets.HETZNER_KUBECONFIG }}" | base64 -d > ~/.kube/config
|
|
chmod 600 ~/.kube/config
|
|
kubectl cluster-info
|
|
|
|
- name: Rollback — previous version
|
|
if: github.event.inputs.strategy == 'previous'
|
|
run: |
|
|
kubectl rollout undo deployment/xpeditis-backend -n ${{ env.K8S_NAMESPACE }}
|
|
kubectl rollout status deployment/xpeditis-backend -n ${{ env.K8S_NAMESPACE }} --timeout=180s
|
|
kubectl rollout undo deployment/xpeditis-frontend -n ${{ env.K8S_NAMESPACE }}
|
|
kubectl rollout status deployment/xpeditis-frontend -n ${{ env.K8S_NAMESPACE }} --timeout=180s
|
|
kubectl get pods -n ${{ env.K8S_NAMESPACE }}
|
|
|
|
- name: Login to Scaleway (for image verification)
|
|
if: github.event.inputs.strategy == 'specific-version'
|
|
uses: docker/login-action@v3
|
|
with:
|
|
registry: ${{ env.REGISTRY }}
|
|
username: nologin
|
|
password: ${{ secrets.REGISTRY_TOKEN }}
|
|
|
|
- uses: docker/setup-buildx-action@v3
|
|
if: github.event.inputs.strategy == 'specific-version'
|
|
|
|
- name: Rollback — specific version
|
|
if: github.event.inputs.strategy == 'specific-version'
|
|
run: |
|
|
TAG="${{ github.event.inputs.version_tag }}"
|
|
BACKEND="${{ env.REGISTRY }}/xpeditis-backend:${TAG}"
|
|
FRONTEND="${{ env.REGISTRY }}/xpeditis-frontend:${TAG}"
|
|
|
|
echo "Verifying images exist..."
|
|
docker buildx imagetools inspect "$BACKEND" || { echo "ERROR: $BACKEND not found"; exit 1; }
|
|
docker buildx imagetools inspect "$FRONTEND" || { echo "ERROR: $FRONTEND not found"; exit 1; }
|
|
|
|
kubectl set image deployment/xpeditis-backend backend="$BACKEND" -n ${{ env.K8S_NAMESPACE }}
|
|
kubectl rollout status deployment/xpeditis-backend -n ${{ env.K8S_NAMESPACE }} --timeout=180s
|
|
|
|
kubectl set image deployment/xpeditis-frontend frontend="$FRONTEND" -n ${{ env.K8S_NAMESPACE }}
|
|
kubectl rollout status deployment/xpeditis-frontend -n ${{ env.K8S_NAMESPACE }} --timeout=180s
|
|
|
|
kubectl get pods -n ${{ env.K8S_NAMESPACE }}
|
|
|
|
- name: Rollout history
|
|
if: always()
|
|
run: |
|
|
kubectl rollout history deployment/xpeditis-backend -n ${{ env.K8S_NAMESPACE }} || true
|
|
kubectl rollout history deployment/xpeditis-frontend -n ${{ env.K8S_NAMESPACE }} || true
|
|
|
|
# ── Preprod rollback via Portainer ───────────────────────────────────
|
|
rollback-preprod:
|
|
name: Rollback Preprod
|
|
runs-on: ubuntu-latest
|
|
needs: validate
|
|
if: github.event.inputs.environment == 'preprod'
|
|
steps:
|
|
- uses: docker/setup-buildx-action@v3
|
|
|
|
- uses: docker/login-action@v3
|
|
with:
|
|
registry: ${{ env.REGISTRY }}
|
|
username: nologin
|
|
password: ${{ secrets.REGISTRY_TOKEN }}
|
|
|
|
- name: Verify target image exists
|
|
run: |
|
|
TAG="${{ github.event.inputs.version_tag }}"
|
|
docker buildx imagetools inspect "${{ env.REGISTRY }}/xpeditis-backend:${TAG}" || \
|
|
{ echo "ERROR: backend image not found: $TAG"; exit 1; }
|
|
docker buildx imagetools inspect "${{ env.REGISTRY }}/xpeditis-frontend:${TAG}" || \
|
|
{ echo "ERROR: frontend image not found: $TAG"; exit 1; }
|
|
|
|
- name: Re-tag as preprod
|
|
run: |
|
|
TAG="${{ github.event.inputs.version_tag }}"
|
|
docker buildx imagetools create \
|
|
--tag ${{ env.REGISTRY }}/xpeditis-backend:preprod \
|
|
${{ env.REGISTRY }}/xpeditis-backend:${TAG}
|
|
docker buildx imagetools create \
|
|
--tag ${{ env.REGISTRY }}/xpeditis-frontend:preprod \
|
|
${{ env.REGISTRY }}/xpeditis-frontend:${TAG}
|
|
|
|
- name: Deploy backend (Portainer)
|
|
run: curl -sf -X POST "${{ secrets.PORTAINER_WEBHOOK_BACKEND }}"
|
|
- run: sleep 20
|
|
- name: Deploy frontend (Portainer)
|
|
run: curl -sf -X POST "${{ secrets.PORTAINER_WEBHOOK_FRONTEND }}"
|
|
|
|
# ── Smoke Tests ───────────────────────────────────────────────────────
|
|
smoke-tests:
|
|
name: Smoke Tests Post-Rollback
|
|
runs-on: ubuntu-latest
|
|
needs: [rollback-production, rollback-preprod]
|
|
if: always() && (needs.rollback-production.result == 'success' || needs.rollback-preprod.result == 'success')
|
|
steps:
|
|
- name: Set URLs
|
|
id: urls
|
|
run: |
|
|
if [ "${{ github.event.inputs.environment }}" = "production" ]; then
|
|
echo "backend=${{ secrets.PROD_BACKEND_URL }}/api/v1/health" >> $GITHUB_OUTPUT
|
|
echo "frontend=${{ secrets.PROD_FRONTEND_URL }}" >> $GITHUB_OUTPUT
|
|
echo "wait=30" >> $GITHUB_OUTPUT
|
|
else
|
|
echo "backend=${{ secrets.PREPROD_BACKEND_URL }}/api/v1/health" >> $GITHUB_OUTPUT
|
|
echo "frontend=${{ secrets.PREPROD_FRONTEND_URL }}" >> $GITHUB_OUTPUT
|
|
echo "wait=60" >> $GITHUB_OUTPUT
|
|
fi
|
|
|
|
- run: sleep ${{ steps.urls.outputs.wait }}
|
|
|
|
- name: Health — Backend
|
|
run: |
|
|
for i in {1..12}; do
|
|
STATUS=$(curl -s -o /dev/null -w "%{http_code}" --max-time 10 \
|
|
"${{ steps.urls.outputs.backend }}" 2>/dev/null || echo "000")
|
|
echo " Attempt $i: HTTP $STATUS"
|
|
if [ "$STATUS" = "200" ]; then echo "Backend OK."; exit 0; fi
|
|
sleep 15
|
|
done
|
|
echo "Backend unhealthy after rollback."
|
|
exit 1
|
|
|
|
- name: Health — Frontend
|
|
run: |
|
|
for i in {1..12}; do
|
|
STATUS=$(curl -s -o /dev/null -w "%{http_code}" --max-time 10 \
|
|
"${{ steps.urls.outputs.frontend }}" 2>/dev/null || echo "000")
|
|
echo " Attempt $i: HTTP $STATUS"
|
|
if [ "$STATUS" = "200" ]; then echo "Frontend OK."; exit 0; fi
|
|
sleep 15
|
|
done
|
|
echo "Frontend unhealthy after rollback."
|
|
exit 1
|
|
|
|
# ── Notifications ─────────────────────────────────────────────────────
|
|
notify:
|
|
name: Notify
|
|
runs-on: ubuntu-latest
|
|
needs: [rollback-production, rollback-preprod, smoke-tests]
|
|
if: always()
|
|
steps:
|
|
- name: Success
|
|
if: needs.smoke-tests.result == 'success'
|
|
run: |
|
|
curl -s -H "Content-Type: application/json" -d '{
|
|
"embeds": [{
|
|
"title": "↩️ Rollback Successful",
|
|
"color": 16776960,
|
|
"fields": [
|
|
{"name": "Environment", "value": "`${{ github.event.inputs.environment }}`", "inline": true},
|
|
{"name": "Strategy", "value": "`${{ github.event.inputs.strategy }}`", "inline": true},
|
|
{"name": "Version", "value": "`${{ github.event.inputs.version_tag || 'previous' }}`", "inline": true},
|
|
{"name": "By", "value": "${{ github.actor }}", "inline": true},
|
|
{"name": "Reason", "value": "${{ github.event.inputs.reason }}", "inline": false}
|
|
],
|
|
"footer": {"text": "Xpeditis CI/CD • Rollback"}
|
|
}]
|
|
}' ${{ secrets.DISCORD_WEBHOOK_URL }}
|
|
|
|
- name: Failure
|
|
if: needs.smoke-tests.result != 'success'
|
|
run: |
|
|
curl -s -H "Content-Type: application/json" -d '{
|
|
"content": "@here ROLLBACK FAILED — MANUAL INTERVENTION REQUIRED",
|
|
"embeds": [{
|
|
"title": "🔴 Rollback Failed",
|
|
"color": 15158332,
|
|
"fields": [
|
|
{"name": "Environment", "value": "`${{ github.event.inputs.environment }}`", "inline": true},
|
|
{"name": "Attempted", "value": "`${{ github.event.inputs.version_tag || 'previous' }}`", "inline": true},
|
|
{"name": "By", "value": "${{ github.actor }}", "inline": true},
|
|
{"name": "Reason", "value": "${{ github.event.inputs.reason }}", "inline": false}
|
|
],
|
|
"footer": {"text": "Xpeditis CI/CD • Rollback"}
|
|
}]
|
|
}' ${{ secrets.DISCORD_WEBHOOK_URL }}
|