etcd 性能调优

# etcd 配置文件示例
# /etc/etcd/etcd.conf.yaml

# 心跳间隔 (毫秒)
# 默认: 100ms
# 推荐: 根据网络延迟调整
heartbeat-interval: 100

# 选举超时 (毫秒)
# 默认: 1000ms
# 推荐: heartbeat-interval 的 10 倍
election-timeout: 1000

# 存储配额 (字节)
# 默认: 2GB
# 推荐: 根据数据量调整，最大 8GB
quota-backend-bytes: 8589934592

# 自动压缩模式
# periodic: 按时间周期
# revision: 按版本数量
auto-compaction-mode: periodic

# 自动压缩保留
# periodic 模式: 保留时间 (如 "1h")
# revision 模式: 保留版本数
auto-compaction-retention: "1h"

# 后端批量提交间隔 (毫秒)
# 更小的值提高一致性，但降低吞吐量
backend-batch-interval: 100

# 后端批量限制
backend-batch-limit: 10000

# 最大请求大小 (字节)
# 默认: 1.5MB
max-request-bytes: 1572864

# gRPC 保活参数
# grpc-keepalive-min-time: 服务端允许的最小保活间隔
# grpc-keepalive-interval: 客户端保活间隔
# grpc-keepalive-timeout: 保活超时

// API Server 的 etcd 配置
// staging/src/k8s.io/apiserver/pkg/storage/storagebackend/config.go

type TransportConfig struct {
    // etcd 服务器列表
    ServerList []string

    // TLS 配置
    CertFile   string
    KeyFile    string
    CAFile     string

    // 连接池大小
    // 推荐: CPU 核数 * 2
    MaxIdleConns int

    // 单个请求超时
    // 推荐: 较短超时以快速失败
    RequestTimeout time.Duration

    // Watch 缓存大小
    // staging/src/k8s.io/apiserver/pkg/storage/cacher/cacher.go
    CacheCapacity int
}

// 推荐配置
config := &TransportConfig{
    MaxIdleConns:   100,
    RequestTimeout: 30 * time.Second,
    CacheCapacity:  1000,
}

// etcd 客户端连接池
// client/v3/config.go

type Config struct {
    Endpoints   []string
    DialTimeout time.Duration

    // 连接池大小
    MaxCallSendMsgSize int
    MaxCallRecvMsgSize int

    // 自动同步端点
    AutoSyncInterval time.Duration

    // 重试策略
    // client/v3/retry_interceptor.go
}

// 推荐配置
cfg := clientv3.Config{
    Endpoints:   []string{"etcd-1:2379", "etcd-2:2379", "etcd-3:2379"},
    DialTimeout: 5 * time.Second,

    // 自动同步端点列表 (应对集群变更)
    AutoSyncInterval: 30 * time.Second,
}

// 使用事务进行批量操作
func batchPut(client *clientv3.Client, kvs map[string]string) error {
    ops := make([]clientv3.Op, 0, len(kvs))
    for k, v := range kvs {
        ops = append(ops, clientv3.OpPut(k, v))
    }

    // 使用事务
    _, err := client.Txn(context.Background()).
        Then(ops...).
        Commit()

    return err
}

// 分批处理大量操作
func batchPutLarge(client *clientv3.Client, kvs map[string]string) error {
    const batchSize = 100

    keys := make([]string, 0, len(kvs))
    for k := range kvs {
        keys = append(keys, k)
    }

    for i := 0; i < len(keys); i += batchSize {
        end := i + batchSize
        if end > len(keys) {
            end = len(keys)
        }

        batch := make(map[string]string)
        for _, k := range keys[i:end] {
            batch[k] = kvs[k]
        }

        if err := batchPut(client, batch); err != nil {
            return err
        }
    }

    return nil
}

// Watch 优化最佳实践
func optimizedWatch(client *clientv3.Client, prefix string) {
    ctx := context.Background()

    // 1. 使用前缀 Watch
    ch := client.Watch(ctx, prefix,
        clientv3.WithPrefix(),
        clientv3.WithProgressNotify(), // 启用进度通知
        clientv3.WithPrevKV(),         // 需要时才启用
    )

    // 2. 批量处理事件
    eventBatch := make([]*clientv3.Event, 0, 100)
    ticker := time.NewTicker(100 * time.Millisecond)

    for {
        select {
        case resp := <-ch:
            if resp.Err() != nil {
                // 处理错误
                continue
            }

            if resp.IsProgressNotify() {
                // 处理进度通知
                continue
            }

            eventBatch = append(eventBatch, resp.Events...)

        case <-ticker.C:
            if len(eventBatch) > 0 {
                processEvents(eventBatch)
                eventBatch = eventBatch[:0]
            }
        }
    }
}

// 带重试的操作
func putWithRetry(client *clientv3.Client, key, value string) error {
    backoff := wait.Backoff{
        Duration: 100 * time.Millisecond,
        Factor:   2,
        Jitter:   0.1,
        Steps:    5,
        Cap:      5 * time.Second,
    }

    return wait.ExponentialBackoff(backoff, func() (bool, error) {
        ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
        defer cancel()

        _, err := client.Put(ctx, key, value)
        if err != nil {
            // 检查是否可重试
            if isRetryable(err) {
                return false, nil // 继续重试
            }
            return false, err // 不可恢复的错误
        }

        return true, nil // 成功
    })
}

func isRetryable(err error) bool {
    // 超时、连接错误等可重试
    return err == context.DeadlineExceeded ||
           status.Code(err) == codes.Unavailable
}

# 跨区域配置建议
# Region A (主区域): 2 节点
# Region B (备区域): 1 节点

# 优化心跳和选举超时以适应跨区域延迟
heartbeat-interval: 300    # 增加心跳间隔
election-timeout: 3000     # 增加选举超时

# etcd 告警规则
groups:
- name: etcd
  rules:
  - alert: EtcdNoLeader
    expr: etcd_server_has_leader == 0
    for: 1m
    labels:
      severity: critical
    annotations:
      summary: "etcd 集群没有 Leader"

  - alert: EtcdHighFsyncLatency
    expr: histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket[5m])) > 0.01
    for: 5m
    labels:
      severity: warning
    annotations:
      summary: "etcd WAL fsync 延迟过高"

  - alert: EtcdHighCommitLatency
    expr: histogram_quantile(0.99, rate(etcd_disk_backend_commit_duration_seconds_bucket[5m])) > 0.025
    for: 5m
    labels:
      severity: warning
    annotations:
      summary: "etcd 后端提交延迟过高"

  - alert: EtcdQuotaNearLimit
    expr: etcd_mvcc_db_total_size_in_bytes / etcd_server_quota_backend_bytes > 0.8
    for: 5m
    labels:
      severity: warning
    annotations:
      summary: "etcd 存储配额即将用尽"

  - alert: EtcdMemberUnhealthy
    expr: etcd_server_health_failures > 0
    for: 3m
    labels:
      severity: warning
    annotations:
      summary: "etcd 成员健康检查失败"

{
  "title": "etcd Overview",
  "panels": [
    {
      "title": "Leader",
      "targets": [{
        "expr": "etcd_server_is_leader"
      }]
    },
    {
      "title": "DB Size",
      "targets": [{
        "expr": "etcd_mvcc_db_total_size_in_bytes"
      }]
    },
    {
      "title": "WAL Fsync Latency",
      "targets": [{
        "expr": "histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket[5m]))"
      }]
    },
    {
      "title": "Client Traffic In",
      "targets": [{
        "expr": "rate(etcd_network_client_grpc_received_bytes_total[5m])"
      }]
    },
    {
      "title": "Proposals",
      "targets": [
        {"expr": "rate(etcd_server_proposals_committed_total[5m])", "legendFormat": "committed"},
        {"expr": "rate(etcd_server_proposals_applied_total[5m])", "legendFormat": "applied"},
        {"expr": "rate(etcd_server_proposals_failed_total[5m])", "legendFormat": "failed"}
      ]
    }
  ]
}