44713e4ca0
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_018BAUeCFpDkRD6tU5wGsc1C
352 lines
11 KiB
YAML
352 lines
11 KiB
YAML
# =============================================================================
|
|
# Prometheus -- metriques et regles d'alerte
|
|
# =============================================================================
|
|
# Perimetre volontairement reduit : kubelet/cAdvisor, node-exporter, Traefik,
|
|
# cert-manager et postgres-exporter. Pas de kube-state-metrics ni d'operateur :
|
|
# a un noeud et une dizaine de pods, ils coutent plus de RAM qu'ils n'apportent.
|
|
---
|
|
apiVersion: v1
|
|
kind: ServiceAccount
|
|
metadata:
|
|
name: prometheus
|
|
namespace: monitoring
|
|
---
|
|
apiVersion: rbac.authorization.k8s.io/v1
|
|
kind: ClusterRole
|
|
metadata:
|
|
name: prometheus
|
|
rules:
|
|
- apiGroups: [""]
|
|
resources: [nodes, nodes/metrics, nodes/proxy, services, endpoints, pods]
|
|
verbs: [get, list, watch]
|
|
- nonResourceURLs: ["/metrics", "/metrics/cadvisor"]
|
|
verbs: [get]
|
|
---
|
|
apiVersion: rbac.authorization.k8s.io/v1
|
|
kind: ClusterRoleBinding
|
|
metadata:
|
|
name: prometheus
|
|
roleRef:
|
|
apiGroup: rbac.authorization.k8s.io
|
|
kind: ClusterRole
|
|
name: prometheus
|
|
subjects:
|
|
- kind: ServiceAccount
|
|
name: prometheus
|
|
namespace: monitoring
|
|
---
|
|
apiVersion: v1
|
|
kind: ConfigMap
|
|
metadata:
|
|
name: prometheus-config
|
|
namespace: monitoring
|
|
data:
|
|
prometheus.yml: |
|
|
global:
|
|
scrape_interval: 30s
|
|
evaluation_interval: 30s
|
|
external_labels:
|
|
cluster: xpeditis-prod
|
|
|
|
rule_files:
|
|
- /etc/prometheus/rules/*.yml
|
|
|
|
alerting:
|
|
alertmanagers:
|
|
- static_configs:
|
|
- targets: ['alertmanager:9093']
|
|
|
|
scrape_configs:
|
|
- job_name: prometheus
|
|
static_configs:
|
|
- targets: ['localhost:9090']
|
|
|
|
- job_name: node-exporter
|
|
kubernetes_sd_configs:
|
|
- role: endpoints
|
|
relabel_configs:
|
|
- source_labels: [__meta_kubernetes_service_name]
|
|
regex: node-exporter
|
|
action: keep
|
|
|
|
# cAdvisor : consommation CPU/memoire par conteneur.
|
|
- job_name: kubelet-cadvisor
|
|
scheme: https
|
|
tls_config:
|
|
ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
|
|
insecure_skip_verify: true
|
|
bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
|
|
kubernetes_sd_configs:
|
|
- role: node
|
|
relabel_configs:
|
|
- action: labelmap
|
|
regex: __meta_kubernetes_node_label_(.+)
|
|
- target_label: __address__
|
|
replacement: kubernetes.default.svc:443
|
|
- source_labels: [__meta_kubernetes_node_name]
|
|
regex: (.+)
|
|
target_label: __metrics_path__
|
|
replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor
|
|
|
|
- job_name: traefik
|
|
kubernetes_sd_configs:
|
|
- role: pod
|
|
namespaces:
|
|
names: [kube-system]
|
|
relabel_configs:
|
|
- source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_name]
|
|
regex: traefik
|
|
action: keep
|
|
- source_labels: [__address__]
|
|
regex: '(.+?)(?::\d+)?'
|
|
target_label: __address__
|
|
replacement: '${1}:9100'
|
|
|
|
- job_name: cert-manager
|
|
kubernetes_sd_configs:
|
|
- role: pod
|
|
namespaces:
|
|
names: [cert-manager]
|
|
relabel_configs:
|
|
- source_labels: [__meta_kubernetes_pod_container_port_number]
|
|
regex: "9402"
|
|
action: keep
|
|
|
|
# PostgreSQL : l'exportateur tourne sur db-01, hors du cluster.
|
|
- job_name: postgres
|
|
static_configs:
|
|
- targets: ['10.10.1.20:9187']
|
|
labels:
|
|
instance: xpeditis-prod-db-01
|
|
|
|
- job_name: loki
|
|
static_configs:
|
|
- targets: ['loki:3100']
|
|
|
|
rules.yml: |
|
|
groups:
|
|
- name: xpeditis-disponibilite
|
|
rules:
|
|
- alert: BackendIndisponible
|
|
# Traefik ne voit plus aucune instance saine derriere le service :
|
|
# l'API est hors ligne pour les utilisateurs, quoi que dise k8s.
|
|
expr: |
|
|
sum by (service) (traefik_service_server_up{service=~"xpeditis-prod-xpeditis-backend.*"}) == 0
|
|
for: 2m
|
|
labels:
|
|
severity: critique
|
|
annotations:
|
|
summary: "Aucune instance backend saine derriere l'ingress"
|
|
description: "L'API Xpeditis ne repond plus. Verifier : kubectl -n xpeditis-prod get pods"
|
|
|
|
- alert: FrontendIndisponible
|
|
expr: |
|
|
sum by (service) (traefik_service_server_up{service=~"xpeditis-prod-xpeditis-frontend.*"}) == 0
|
|
for: 2m
|
|
labels:
|
|
severity: critique
|
|
annotations:
|
|
summary: "Aucune instance frontend saine derriere l'ingress"
|
|
|
|
- alert: TauxErreur5xxEleve
|
|
expr: |
|
|
sum(rate(traefik_service_requests_total{code=~"5.."}[5m]))
|
|
/ clamp_min(sum(rate(traefik_service_requests_total[5m])), 0.001) > 0.05
|
|
for: 5m
|
|
labels:
|
|
severity: critique
|
|
annotations:
|
|
summary: "Plus de 5% de reponses 5xx"
|
|
description: "Taux d'erreur serveur anormal sur les 5 dernieres minutes."
|
|
|
|
- alert: LatenceElevee
|
|
expr: |
|
|
histogram_quantile(0.95,
|
|
sum(rate(traefik_service_request_duration_seconds_bucket[5m])) by (le, service)) > 2
|
|
for: 10m
|
|
labels:
|
|
severity: avertissement
|
|
annotations:
|
|
summary: "P95 au-dessus de 2 s sur {{ $labels.service }}"
|
|
|
|
- name: xpeditis-ressources
|
|
rules:
|
|
- alert: DisqueBientotPlein
|
|
# Un disque plein arrete PostgreSQL et k3s. Seuil a 15% restants
|
|
# pour laisser le temps d'intervenir.
|
|
expr: |
|
|
node_filesystem_avail_bytes{mountpoint="/",fstype!="tmpfs"}
|
|
/ node_filesystem_size_bytes{mountpoint="/",fstype!="tmpfs"} < 0.15
|
|
for: 10m
|
|
labels:
|
|
severity: critique
|
|
annotations:
|
|
summary: "Moins de 15% d'espace disque libre sur {{ $labels.instance }}"
|
|
|
|
- alert: MemoireNoeudSaturee
|
|
expr: |
|
|
(1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) > 0.90
|
|
for: 10m
|
|
labels:
|
|
severity: avertissement
|
|
annotations:
|
|
summary: "Memoire du noeud a plus de 90%"
|
|
|
|
- alert: ConteneurRedemarreEnBoucle
|
|
expr: |
|
|
increase(container_start_time_seconds{namespace="xpeditis-prod"}[15m]) > 3
|
|
for: 5m
|
|
labels:
|
|
severity: critique
|
|
annotations:
|
|
summary: "Le conteneur {{ $labels.container }} redemarre en boucle"
|
|
|
|
- name: xpeditis-donnees
|
|
rules:
|
|
- alert: PostgresInjoignable
|
|
expr: pg_up == 0
|
|
for: 2m
|
|
labels:
|
|
severity: critique
|
|
annotations:
|
|
summary: "PostgreSQL ne repond plus sur db-01"
|
|
description: "Verifier db-01 : docker compose ps, journalctl -u docker"
|
|
|
|
- alert: ConnexionsPostgresSaturees
|
|
expr: |
|
|
sum(pg_stat_database_numbackends) / on() pg_settings_max_connections > 0.80
|
|
for: 5m
|
|
labels:
|
|
severity: avertissement
|
|
annotations:
|
|
summary: "Plus de 80% des connexions PostgreSQL consommees"
|
|
|
|
# NOTE -- la surveillance des sauvegardes ne passe PAS par Prometheus.
|
|
# Une alerte "la sauvegarde n'a pas tourne" evaluee par un composant
|
|
# heberge sur la meme machine ne se declenche pas quand la machine est
|
|
# eteinte, c'est-a-dire precisement quand elle serait utile.
|
|
# Elle repose donc sur un battement de coeur externe : pg-backup.sh
|
|
# appelle BACKUP_HEARTBEAT_URL a chaque succes, et le service externe
|
|
# (BetterStack / healthchecks.io) alerte en cas de silence.
|
|
# Cf. docs/mise-en-prod/12-sauvegardes-restauration.md.
|
|
|
|
- name: xpeditis-tls
|
|
rules:
|
|
- alert: CertificatBientotExpire
|
|
expr: |
|
|
(certmanager_certificate_expiration_timestamp_seconds - time()) / 86400 < 15
|
|
for: 1h
|
|
labels:
|
|
severity: critique
|
|
annotations:
|
|
summary: "Certificat {{ $labels.name }} expire dans moins de 15 jours"
|
|
description: "Le renouvellement automatique ne fonctionne plus. Verifier : kubectl describe certificate -A"
|
|
---
|
|
apiVersion: v1
|
|
kind: PersistentVolumeClaim
|
|
metadata:
|
|
name: prometheus-data
|
|
namespace: monitoring
|
|
spec:
|
|
accessModes: [ReadWriteOnce]
|
|
storageClassName: local-path
|
|
resources:
|
|
requests:
|
|
storage: 15Gi
|
|
---
|
|
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata:
|
|
name: prometheus
|
|
namespace: monitoring
|
|
labels:
|
|
app.kubernetes.io/name: prometheus
|
|
spec:
|
|
replicas: 1
|
|
strategy:
|
|
type: Recreate
|
|
selector:
|
|
matchLabels:
|
|
app.kubernetes.io/name: prometheus
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app.kubernetes.io/name: prometheus
|
|
annotations:
|
|
# Redemarre Prometheus quand les regles changent.
|
|
checksum/config: "PLACEHOLDER"
|
|
spec:
|
|
serviceAccountName: prometheus
|
|
securityContext:
|
|
runAsNonRoot: true
|
|
runAsUser: 65534
|
|
runAsGroup: 65534
|
|
fsGroup: 65534
|
|
seccompProfile:
|
|
type: RuntimeDefault
|
|
containers:
|
|
- name: prometheus
|
|
image: prom/prometheus:v3.1.0
|
|
args:
|
|
- --config.file=/etc/prometheus/prometheus.yml
|
|
- --storage.tsdb.path=/prometheus
|
|
# 15 jours : suffisant pour l'analyse d'incident, tient dans 15 Go.
|
|
- --storage.tsdb.retention.time=15d
|
|
- --web.enable-lifecycle
|
|
- --web.listen-address=:9090
|
|
ports:
|
|
- name: http
|
|
containerPort: 9090
|
|
readinessProbe:
|
|
httpGet:
|
|
path: /-/ready
|
|
port: http
|
|
initialDelaySeconds: 20
|
|
livenessProbe:
|
|
httpGet:
|
|
path: /-/healthy
|
|
port: http
|
|
initialDelaySeconds: 60
|
|
periodSeconds: 30
|
|
resources:
|
|
requests:
|
|
cpu: 150m
|
|
memory: 512Mi
|
|
limits:
|
|
cpu: 1000m
|
|
memory: 1536Mi
|
|
securityContext:
|
|
allowPrivilegeEscalation: false
|
|
capabilities:
|
|
drop: ["ALL"]
|
|
volumeMounts:
|
|
- name: config
|
|
mountPath: /etc/prometheus/prometheus.yml
|
|
subPath: prometheus.yml
|
|
- name: config
|
|
mountPath: /etc/prometheus/rules/rules.yml
|
|
subPath: rules.yml
|
|
- name: data
|
|
mountPath: /prometheus
|
|
volumes:
|
|
- name: config
|
|
configMap:
|
|
name: prometheus-config
|
|
- name: data
|
|
persistentVolumeClaim:
|
|
claimName: prometheus-data
|
|
---
|
|
apiVersion: v1
|
|
kind: Service
|
|
metadata:
|
|
name: prometheus
|
|
namespace: monitoring
|
|
spec:
|
|
type: ClusterIP
|
|
selector:
|
|
app.kubernetes.io/name: prometheus
|
|
ports:
|
|
- name: http
|
|
port: 9090
|
|
targetPort: http
|