Kubernetes可观测性实战：Metrics、Logging与Tracing全栈方案

apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: api-server-metrics
  namespace: monitoring
spec:
  namespaceSelector:
    matchNames:
      - production
  selector:
    matchLabels:
      monitoring: enabled
  endpoints:
    - port: metrics          # Service 中的端口名
      path: /metrics
      interval: 15s
      scrapeTimeout: 10s
      honorLabels: true

# 节点 CPU 使用率
100 - (avg by(instance) (irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100)

# 节点内存使用率
100 * (1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)

# Pod CPU 使用率（相对 limit）
rate(container_cpu_usage_seconds_total[5m]) / container_spec_cpu_quota * 100

# 容器内存使用率（相对 limit）
container_memory_working_set_bytes / container_spec_memory_limit_bytes

# 节点磁盘使用率
100 - (node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"} * 100)

# Promtail（日志采集）
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: promtail
spec:
  template:
    spec:
      containers:
        - name: promtail
          image: grafana/promtail:latest
          args:
            - -config.file=/etc/promtail/promtail.yaml
          volumeMounts:
            - name: logs
              mountPath: /var/log
            - name: containers
              mountPath: /var/log/containers
            - name: pods
              mountPath: /var/log/pods
      volumes:
        - name: logs
          hostPath:
            path: /var/log
        - name: containers
          hostPath:
            path: /var/log/containers

# OpenTelemetry Operator 注入 sidecar
apiVersion: opentelemetry.io/v1alpha1
kind: OpenTelemetryCollector
metadata:
  name: production
spec:
  mode: deployment
  config: |
    receivers:
      otlp:
        protocols:
          grpc:
            endpoint: 0.0.0.0:4317
          http:
            endpoint: 0.0.0.0:4318
    exporters:
      otlp/jaeger:
        endpoint: jaeger-collector:4317
        tls:
          insecure: true
    service:
      pipelines:
        traces:
          receivers: [otlp]
          exporters: [otlp/jaeger]

metadata:
  annotations:
    instrumentation.opentelemetry.io/inject-sdk: "true"

# 找到慢请求
jaeger-query:16686 → Service: api-server → Operation: POST /api/orders

# 查看某 trace 的完整调用链
trace_id: 4f7d3c8a9b2e1f (横跨 api-server → order-service → payment-service → postgres)

apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: kubernetes-alerts
  namespace: monitoring
spec:
  groups:
    - name: kubernetes-resources
      rules:
        - alert: KubernetesNodeNotReady
          expr: kube_node_status_condition{condition="Ready",status="true"} == 0
          for: 10m
          labels:
            severity: critical
          annotations:
            summary: "节点 {{ $labels.node }} 不可用"
            description: "节点已持续 10 分钟报告 NotReady 状态"

        - alert: KubernetesPodCrashLooping
          expr: rate(kube_pod_container_status_restarts_total[15m]) > 0
          for: 5m
          labels:
            severity: warning
          annotations:
            summary: "Pod {{ $labels.pod }} 频繁重启"

        - alert: KubernetesHighMemoryUsage
          expr: |
            container_memory_working_set_bytes{container!=""} /
            container_spec_memory_limit_bytes > 0.9
          for: 5m
          labels:
            severity: warning
          annotations:
            summary: "Pod {{ $labels.pod }} 内存使用率超过 90%"

        - alert: KubernetesPersistentVolumeFillingUp
          expr: |
            kubelet_volume_stats_used_bytes /
            kubelet_volume_stats_capacity_bytes > 0.85
          for: 5m
          labels:
            severity: warning
          annotations:
            summary: "PVC {{ $labels.persistentvolumeclaim }} 使用率超过 85%"

apiVersion: monitoring.coreos.com/v1
kind: AlertmanagerConfig
metadata:
  name: main-routes
  namespace: monitoring
spec:
  route:
    groupBy: ['alertname', 'namespace']
    groupWait: 30s
    groupInterval: 5m
    repeatInterval: 12h
    receiver: default
    routes:
      - match:
          severity: critical
        receiver: pagerduty
        continue: true
      - match:
          severity: warning
        receiver: slack
  receivers:
    - name: default
      slackConfigs:
        - apiURL:
            key: url
            name: slack-webhook
          channel: '#alerts'
    - name: pagerduty
      pagerdutyConfigs:
        - serviceKey:
            key: key
            name: pagerduty-key

# 可用性（成功案例 / 总请求）
sum(rate(http_requests_total{status=~"2.."}[1h]))
/
sum(rate(http_requests_total[1h]))

# 延迟百分位
histogram_quantile(0.99,
  sum(rate(http_request_duration_seconds_bucket[1h])) by (le)
)