KI-Server-Monitoring: Prometheus, Grafana und DCGM

version: '3.8'
services:
  # Prometheus: Metriken-Sammlung
  prometheus:
    image: prom/prometheus:latest
    volumes:
      - ./prometheus.yml:/etc/prometheus/prometheus.yml
      - prometheus_data:/prometheus
    ports:
      - "9090:9090"

  # Grafana: Visualisierung
  grafana:
    image: grafana/grafana:latest
    ports:
      - "3000:3000"
    volumes:
      - grafana_data:/var/lib/grafana

  # DCGM: GPU-Metriken
  dcgm-exporter:
    image: nvcr.io/nvidia/k8s/dcgm-exporter:latest
    runtime: nvidia
    ports:
      - "9400:9400"

  # Node Exporter: System-Metriken
  node-exporter:
    image: prom/node-exporter:latest
    pid: host
    volumes:
      - /proc:/host/proc:ro
      - /sys:/host/sys:ro

  # cAdvisor: Container-Metriken
  cadvisor:
    image: gcr.io/cadvisor/cadvisor:latest
    volumes:
      - /:/rootfs:ro
      - /var/run:/var/run:ro

volumes:
  prometheus_data:
  grafana_data:

# prometheus/alert_rules.yml
groups:
  - name: ki-server-alerts
    rules:
      - alert: GPUP过热
        expr: dcgm_gpu_temp > 85
        for: 5m
        annotations:
          summary: "GPU überhitzt"
          description: "GPU {{ $labels.gpu }} Temperatur: {{ $value }}°C"

      - alert: APIHighLatency
        expr: http_request_duration_seconds > 5
        for: 2m
        annotations:
          summary: "Hohe API-Latenz"
          description: "Latenz >5s für {{ $labels.job }}"