监控与 Metrics 指南

custom:
  metrics_enable: true                  # 全局 Metrics 开关
  scheduler_metrics_enable: true        # 调度器 Metrics
  controller_metrics_enable: true       # 控制器 Metrics

apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: volcano-scheduler
  namespace: volcano-system
  labels:
    app: volcano
spec:
  selector:
    matchLabels:
      app: volcano-scheduler
  endpoints:
    - port: metrics
      interval: 15s
      path: /metrics
---
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: volcano-controller
  namespace: volcano-system
  labels:
    app: volcano
spec:
  selector:
    matchLabels:
      app: volcano-controller-manager
  endpoints:
    - port: metrics
      interval: 15s
      path: /metrics

# prometheus.yml
scrape_configs:
  - job_name: 'volcano-scheduler'
    kubernetes_sd_configs:
      - role: pod
    relabel_configs:
      - source_labels: [__meta_kubernetes_namespace]
        regex: volcano-system
        action: keep
      - source_labels: [__meta_kubernetes_pod_label_app]
        regex: volcano-scheduler
        action: keep
    scrape_interval: 15s

  - job_name: 'volcano-controller'
    kubernetes_sd_configs:
      - role: pod
    relabel_configs:
      - source_labels: [__meta_kubernetes_namespace]
        regex: volcano-system
        action: keep
      - source_labels: [__meta_kubernetes_pod_label_app]
        regex: volcano-controller-manager
        action: keep
    scrape_interval: 15s

# P99 端到端调度延迟
histogram_quantile(0.99,
  sum(rate(volcano_e2e_scheduling_latency_milliseconds_bucket[5m])) by (le)
)

# P50 调度延迟
histogram_quantile(0.50,
  sum(rate(volcano_e2e_scheduling_latency_milliseconds_bucket[5m])) by (le)
)

# 平均调度延迟
sum(rate(volcano_e2e_scheduling_latency_milliseconds_sum[5m]))
/
sum(rate(volcano_e2e_scheduling_latency_milliseconds_count[5m]))

# 每个 Action 的 P99 延迟
histogram_quantile(0.99,
  sum(rate(volcano_action_scheduling_latency_milliseconds_bucket[5m])) by (le, action)
)

# Action 执行频率（每秒次数）
sum(rate(volcano_action_scheduling_latency_milliseconds_count[5m])) by (action)

# 各 Plugin 的 P99 延迟
histogram_quantile(0.99,
  sum(rate(volcano_plugin_scheduling_latency_milliseconds_bucket[5m])) by (le, plugin)
)

# OnSessionOpen 阶段各 Plugin 耗时
histogram_quantile(0.99,
  sum(rate(volcano_plugin_scheduling_latency_milliseconds_bucket{OnSession="OnSessionOpen"}[5m])) by (le, plugin)
)

# 调度成功率
sum(rate(volcano_schedule_attempts_total{result="Success"}[5m]))
/
sum(rate(volcano_schedule_attempts_total[5m]))

# 调度失败率
sum(rate(volcano_schedule_attempts_total{result="Error"}[5m]))
/
sum(rate(volcano_schedule_attempts_total[5m]))

# 不可调度 Job 数量
volcano_unschedule_job_count

# 不可调度 Task 数量（按 Job）
sum(volcano_unschedule_task_count) by (job_id)

# 抢占 Victim 数量
volcano_pod_preemption_victims

# 抢占尝试速率
rate(volcano_total_preemption_attempts[5m])

{
  "title": "E2E Scheduling Latency (P99)",
  "type": "timeseries",
  "targets": [
    {
      "expr": "histogram_quantile(0.99, sum(rate(volcano_e2e_scheduling_latency_milliseconds_bucket[5m])) by (le))",
      "legendFormat": "P99"
    },
    {
      "expr": "histogram_quantile(0.50, sum(rate(volcano_e2e_scheduling_latency_milliseconds_bucket[5m])) by (le))",
      "legendFormat": "P50"
    }
  ],
  "fieldConfig": {
    "defaults": {
      "unit": "ms"
    }
  }
}

{
  "title": "Scheduling Success Rate",
  "type": "stat",
  "targets": [
    {
      "expr": "sum(rate(volcano_schedule_attempts_total{result='Success'}[5m])) / sum(rate(volcano_schedule_attempts_total[5m])) * 100",
      "legendFormat": "Success Rate"
    }
  ],
  "fieldConfig": {
    "defaults": {
      "unit": "percent",
      "thresholds": {
        "steps": [
          {"color": "red", "value": 80},
          {"color": "yellow", "value": 95},
          {"color": "green", "value": 99}
        ]
      }
    }
  }
}

apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: volcano-alerts
  namespace: volcano-system
spec:
  groups:
    - name: volcano.scheduler
      rules:
        # 调度延迟过高
        - alert: VolcanoSchedulingLatencyHigh
          expr: |
            histogram_quantile(0.99,
              sum(rate(volcano_e2e_scheduling_latency_milliseconds_bucket[5m])) by (le)
            ) > 5000
          for: 5m
          labels:
            severity: warning
          annotations:
            summary: "Volcano P99 调度延迟超过 5 秒"
            description: "当前 P99 延迟: {{ $value }}ms"

        # 大量不可调度 Job
        - alert: VolcanoUnschedulableJobsHigh
          expr: volcano_unschedule_job_count > 50
          for: 10m
          labels:
            severity: warning
          annotations:
            summary: "不可调度 Job 数量过多"
            description: "当前不可调度 Job: {{ $value }}"

        # 调度成功率过低
        - alert: VolcanoSchedulingSuccessRateLow
          expr: |
            sum(rate(volcano_schedule_attempts_total{result="Success"}[5m]))
            /
            sum(rate(volcano_schedule_attempts_total[5m]))
            < 0.9
          for: 5m
          labels:
            severity: critical
          annotations:
            summary: "Volcano 调度成功率低于 90%"

        # 抢占频繁
        - alert: VolcanoPreemptionFrequent
          expr: rate(volcano_total_preemption_attempts[5m]) > 10
          for: 5m
          labels:
            severity: warning
          annotations:
            summary: "Volcano 抢占频率过高"
            description: "每秒抢占尝试: {{ $value }}"

    - name: volcano.component
      rules:
        # 调度器 Pod 重启
        - alert: VolcanoSchedulerRestart
          expr: |
            changes(kube_pod_container_status_restarts_total{
              namespace="volcano-system",
              container="volcano-scheduler"
            }[1h]) > 2
          labels:
            severity: critical
          annotations:
            summary: "Volcano Scheduler 频繁重启"