xpeditis2.0/infra/prod/k8s/monitoring/03-prometheus.yaml

# =============================================================================
# Prometheus -- metriques et regles d'alerte
# =============================================================================
# Perimetre volontairement reduit : kubelet/cAdvisor, node-exporter, Traefik,
# cert-manager et postgres-exporter. Pas de kube-state-metrics ni d'operateur :
# a un noeud et une dizaine de pods, ils coutent plus de RAM qu'ils n'apportent.
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: prometheus
  namespace: monitoring
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: prometheus
rules:
  - apiGroups: [""]
    resources: [nodes, nodes/metrics, nodes/proxy, services, endpoints, pods]
    verbs: [get, list, watch]
  - nonResourceURLs: ["/metrics", "/metrics/cadvisor"]
    verbs: [get]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: prometheus
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: prometheus
subjects:
  - kind: ServiceAccount
    name: prometheus
    namespace: monitoring
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: prometheus-config
  namespace: monitoring
data:
  prometheus.yml: |
    global:
      scrape_interval: 30s
      evaluation_interval: 30s
      external_labels:
        cluster: xpeditis-prod

    rule_files:
      - /etc/prometheus/rules/*.yml

    alerting:
      alertmanagers:
        - static_configs:
            - targets: ['alertmanager:9093']

    scrape_configs:
      - job_name: prometheus
        static_configs:
          - targets: ['localhost:9090']

      - job_name: node-exporter
        kubernetes_sd_configs:
          - role: endpoints
        relabel_configs:
          - source_labels: [__meta_kubernetes_service_name]
            regex: node-exporter
            action: keep

      # cAdvisor : consommation CPU/memoire par conteneur.
      - job_name: kubelet-cadvisor
        scheme: https
        tls_config:
          ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
          insecure_skip_verify: true
        bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
        kubernetes_sd_configs:
          - role: node
        relabel_configs:
          - action: labelmap
            regex: __meta_kubernetes_node_label_(.+)
          - target_label: __address__
            replacement: kubernetes.default.svc:443
          - source_labels: [__meta_kubernetes_node_name]
            regex: (.+)
            target_label: __metrics_path__
            replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor

      - job_name: traefik
        kubernetes_sd_configs:
          - role: pod
            namespaces:
              names: [kube-system]
        relabel_configs:
          - source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_name]
            regex: traefik
            action: keep
          - source_labels: [__address__]
            regex: '(.+?)(?::\d+)?'
            target_label: __address__
            replacement: '${1}:9100'

      - job_name: cert-manager
        kubernetes_sd_configs:
          - role: pod
            namespaces:
              names: [cert-manager]
        relabel_configs:
          - source_labels: [__meta_kubernetes_pod_container_port_number]
            regex: "9402"
            action: keep

      # PostgreSQL : l'exportateur tourne sur db-01, hors du cluster.
      - job_name: postgres
        static_configs:
          - targets: ['10.10.1.20:9187']
            labels:
              instance: xpeditis-prod-db-01

      - job_name: loki
        static_configs:
          - targets: ['loki:3100']    

  rules.yml: |
    groups:
      - name: xpeditis-disponibilite
        rules:
          - alert: BackendIndisponible
            # Traefik ne voit plus aucune instance saine derriere le service :
            # l'API est hors ligne pour les utilisateurs, quoi que dise k8s.
            expr: |
              sum by (service) (traefik_service_server_up{service=~"xpeditis-prod-xpeditis-backend.*"}) == 0
            for: 2m
            labels:
              severity: critique
            annotations:
              summary: "Aucune instance backend saine derriere l'ingress"
              description: "L'API Xpeditis ne repond plus. Verifier : kubectl -n xpeditis-prod get pods"

          - alert: FrontendIndisponible
            expr: |
              sum by (service) (traefik_service_server_up{service=~"xpeditis-prod-xpeditis-frontend.*"}) == 0
            for: 2m
            labels:
              severity: critique
            annotations:
              summary: "Aucune instance frontend saine derriere l'ingress"

          - alert: TauxErreur5xxEleve
            expr: |
              sum(rate(traefik_service_requests_total{code=~"5.."}[5m]))
                / clamp_min(sum(rate(traefik_service_requests_total[5m])), 0.001) > 0.05
            for: 5m
            labels:
              severity: critique
            annotations:
              summary: "Plus de 5% de reponses 5xx"
              description: "Taux d'erreur serveur anormal sur les 5 dernieres minutes."

          - alert: LatenceElevee
            expr: |
              histogram_quantile(0.95,
                sum(rate(traefik_service_request_duration_seconds_bucket[5m])) by (le, service)) > 2
            for: 10m
            labels:
              severity: avertissement
            annotations:
              summary: "P95 au-dessus de 2 s sur {{ $labels.service }}"

      - name: xpeditis-ressources
        rules:
          - alert: DisqueBientotPlein
            # Un disque plein arrete PostgreSQL et k3s. Seuil a 15% restants
            # pour laisser le temps d'intervenir.
            expr: |
              node_filesystem_avail_bytes{mountpoint="/",fstype!="tmpfs"}
                / node_filesystem_size_bytes{mountpoint="/",fstype!="tmpfs"} < 0.15
            for: 10m
            labels:
              severity: critique
            annotations:
              summary: "Moins de 15% d'espace disque libre sur {{ $labels.instance }}"

          - alert: MemoireNoeudSaturee
            expr: |
              (1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) > 0.90
            for: 10m
            labels:
              severity: avertissement
            annotations:
              summary: "Memoire du noeud a plus de 90%"

          - alert: ConteneurRedemarreEnBoucle
            expr: |
              increase(container_start_time_seconds{namespace="xpeditis-prod"}[15m]) > 3
            for: 5m
            labels:
              severity: critique
            annotations:
              summary: "Le conteneur {{ $labels.container }} redemarre en boucle"

      - name: xpeditis-donnees
        rules:
          - alert: PostgresInjoignable
            expr: pg_up == 0
            for: 2m
            labels:
              severity: critique
            annotations:
              summary: "PostgreSQL ne repond plus sur db-01"
              description: "Verifier db-01 : docker compose ps, journalctl -u docker"

          - alert: ConnexionsPostgresSaturees
            expr: |
              sum(pg_stat_database_numbackends) / on() pg_settings_max_connections > 0.80
            for: 5m
            labels:
              severity: avertissement
            annotations:
              summary: "Plus de 80% des connexions PostgreSQL consommees"

          # NOTE -- la surveillance des sauvegardes ne passe PAS par Prometheus.
          # Une alerte "la sauvegarde n'a pas tourne" evaluee par un composant
          # heberge sur la meme machine ne se declenche pas quand la machine est
          # eteinte, c'est-a-dire precisement quand elle serait utile.
          # Elle repose donc sur un battement de coeur externe : pg-backup.sh
          # appelle BACKUP_HEARTBEAT_URL a chaque succes, et le service externe
          # (BetterStack / healthchecks.io) alerte en cas de silence.
          # Cf. docs/mise-en-prod/12-sauvegardes-restauration.md.

      - name: xpeditis-tls
        rules:
          - alert: CertificatBientotExpire
            expr: |
              (certmanager_certificate_expiration_timestamp_seconds - time()) / 86400 < 15
            for: 1h
            labels:
              severity: critique
            annotations:
              summary: "Certificat {{ $labels.name }} expire dans moins de 15 jours"
              description: "Le renouvellement automatique ne fonctionne plus. Verifier : kubectl describe certificate -A"    
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: prometheus-data
  namespace: monitoring
spec:
  accessModes: [ReadWriteOnce]
  storageClassName: local-path
  resources:
    requests:
      storage: 15Gi
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: prometheus
  namespace: monitoring
  labels:
    app.kubernetes.io/name: prometheus
spec:
  replicas: 1
  strategy:
    type: Recreate
  selector:
    matchLabels:
      app.kubernetes.io/name: prometheus
  template:
    metadata:
      labels:
        app.kubernetes.io/name: prometheus
      annotations:
        # Redemarre Prometheus quand les regles changent.
        checksum/config: "PLACEHOLDER"
    spec:
      serviceAccountName: prometheus
      securityContext:
        runAsNonRoot: true
        runAsUser: 65534
        runAsGroup: 65534
        fsGroup: 65534
        seccompProfile:
          type: RuntimeDefault
      containers:
        - name: prometheus
          image: prom/prometheus:v3.1.0
          args:
            - --config.file=/etc/prometheus/prometheus.yml
            - --storage.tsdb.path=/prometheus
            # 15 jours : suffisant pour l'analyse d'incident, tient dans 15 Go.
            - --storage.tsdb.retention.time=15d
            - --web.enable-lifecycle
            - --web.listen-address=:9090
          ports:
            - name: http
              containerPort: 9090
          readinessProbe:
            httpGet:
              path: /-/ready
              port: http
            initialDelaySeconds: 20
          livenessProbe:
            httpGet:
              path: /-/healthy
              port: http
            initialDelaySeconds: 60
            periodSeconds: 30
          resources:
            requests:
              cpu: 150m
              memory: 512Mi
            limits:
              cpu: 1000m
              memory: 1536Mi
          securityContext:
            allowPrivilegeEscalation: false
            readOnlyRootFilesystem: true
            capabilities:
              drop: ["ALL"]
          volumeMounts:
            - name: tmp
              mountPath: /tmp
            - name: config
              mountPath: /etc/prometheus/prometheus.yml
              subPath: prometheus.yml
            - name: config
              mountPath: /etc/prometheus/rules/rules.yml
              subPath: rules.yml
            - name: data
              mountPath: /prometheus
      volumes:
        - name: tmp
          emptyDir:
            sizeLimit: 128Mi
        - name: config
          configMap:
            name: prometheus-config
        - name: data
          persistentVolumeClaim:
            claimName: prometheus-data
---
apiVersion: v1
kind: Service
metadata:
  name: prometheus
  namespace: monitoring
spec:
  type: ClusterIP
  selector:
    app.kubernetes.io/name: prometheus
  ports:
    - name: http
      port: 9090
      targetPort: http