Prometheus 指标与监控体系

# prometheus.yml
scrape_configs:
  # HAMi Scheduler 指标
  - job_name: 'hami-scheduler'
    kubernetes_sd_configs:
      - role: pod
    relabel_configs:
      - source_labels: [__meta_kubernetes_pod_label_app]
        regex: hami-scheduler
        action: keep
      - source_labels: [__meta_kubernetes_pod_ip]
        target_label: __address__
        replacement: '$1:31993'

  # HAMi vGPU Monitor 指标
  - job_name: 'hami-vgpu-monitor'
    kubernetes_sd_configs:
      - role: pod
    relabel_configs:
      - source_labels: [__meta_kubernetes_pod_label_app]
        regex: hami-device-plugin
        action: keep
      - source_labels: [__meta_kubernetes_pod_ip]
        target_label: __address__
        replacement: '$1:9394'

# 集群总显存
sum(GPUDeviceMemoryLimit)

# 集群已分配显存
sum(GPUDeviceMemoryAllocated)

# 集群 GPU 分配率
sum(GPUDeviceMemoryAllocated) / sum(GPUDeviceMemoryLimit) * 100

# 各节点物理显存使用率
HostGPUMemoryUsage / on(deviceuuid) group_left GPUDeviceMemoryLimit * 100

# 各节点 GPU 核心利用率
HostCoreUtilization

# 容器 vGPU 显存使用率
vGPU_device_memory_usage_in_bytes / vGPU_device_memory_limit_in_bytes * 100

# 容器设备利用率
Device_utilization_desc_of_container

# 容器空闲检测（距最后 kernel 执行超过 5 分钟的容器）
Device_last_kernel_of_container > 300

# 每个 GPU 上的容器数
GPUDeviceSharedNum

# GPU 显存碎片率（分配但未实际使用的比例）
(vGPU_device_memory_limit_in_bytes - vGPU_device_memory_usage_in_bytes)
  / vGPU_device_memory_limit_in_bytes * 100

# gpu-alerts.yaml
groups:
  - name: hami-gpu-alerts
    rules:
      # GPU 物理显存使用率超过 90%
      - alert: GPUMemoryUsageHigh
        expr: |
          HostGPUMemoryUsage / on(deviceuuid)
          group_left(devicetype) GPUDeviceMemoryLimit > 0.9
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "GPU 显存使用率过高"
          description: |
            节点 {{ $labels.nodeid }} 上的 GPU {{ $labels.deviceuuid }}
            ({{ $labels.devicetype }}) 显存使用率超过 90%，
            当前值: {{ $value | humanizePercentage }}

      # GPU 核心利用率持续 100%
      - alert: GPUCoreUtilizationSaturated
        expr: HostCoreUtilization == 100
        for: 15m
        labels:
          severity: warning
        annotations:
          summary: "GPU 核心利用率饱和"
          description: |
            GPU {{ $labels.deviceuuid }} 核心利用率持续 15 分钟保持 100%

      # 容器 vGPU 显存超过限制的 95%
      - alert: ContainerVGPUMemoryNearLimit
        expr: |
          vGPU_device_memory_usage_in_bytes
          / vGPU_device_memory_limit_in_bytes > 0.95
        for: 2m
        labels:
          severity: warning
        annotations:
          summary: "容器 vGPU 显存接近限制"
          description: |
            Pod {{ $labels.podnamespace }}/{{ $labels.podname }}
            容器 {{ $labels.ctrname }} 的 vGPU 显存使用率超过 95%

      # 容器长时间没有 GPU kernel 执行
      - alert: ContainerGPUIdle
        expr: Device_last_kernel_of_container > 1800
        for: 5m
        labels:
          severity: info
        annotations:
          summary: "容器 GPU 长时间空闲"
          description: |
            Pod {{ $labels.podnamespace }}/{{ $labels.podname }}
            容器 {{ $labels.ctrname }} 已超过 30 分钟无 GPU kernel 执行，
            可能存在资源浪费

      # 集群 GPU 显存分配率过高
      - alert: ClusterGPUMemoryAllocationHigh
        expr: |
          sum(GPUDeviceMemoryAllocated)
          / sum(GPUDeviceMemoryLimit) > 0.85
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "集群 GPU 显存分配率过高"
          description: |
            集群 GPU 显存分配率超过 85%，
            当前值: {{ $value | humanizePercentage }}

      # 单 GPU 上共享容器数过多
      - alert: GPUOversharing
        expr: GPUDeviceSharedNum > 10
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "GPU 共享容器数过多"
          description: |
            节点 {{ $labels.nodeid }} GPU {{ $labels.deviceuuid }}
            上运行了 {{ $value }} 个容器，可能导致性能下降

      # vGPU Monitor 指标端点不可达
      - alert: VGPUMonitorDown
        expr: up{job="hami-vgpu-monitor"} == 0
        for: 2m
        labels:
          severity: critical
        annotations:
          summary: "vGPU Monitor 不可达"
          description: |
            节点上的 vGPU Monitor 指标采集端点已不可达超过 2 分钟