Kubernetes调度器原理与高级调度策略

# 查看调度队列中的 Pod
kubectl get pods --field-selector status.phase=Pending \
  -o custom-columns='NAME:.metadata.name,NODE:.spec.nodeName'

# 调度配置示例
apiVersion: kubescheduler.config.k8s.io/v1
kind: KubeSchedulerConfiguration
profiles:
  - schedulerName: default-scheduler
    plugins:
      score:
        disabled:
          - name: NodeResourcesFit
        enabled:
          - name: NodeResourcesFit
            weight: 5
    pluginConfig:
      - name: NodeResourcesFit
        args:
          # 评分策略：LeastAllocated 或 MostAllocated
          scoringStrategy:
            type: LeastAllocated
            resources:
              - name: cpu
                weight: 1
              - name: memory
                weight: 1

apiVersion: v1
kind: Pod
spec:
  topologySpreadConstraints:
    - maxSkew: 1                 # 拓扑域间最多相差 1 个 Pod
      topologyKey: topology.kubernetes.io/zone   # 可用区级别
      whenUnsatisfiable: DoNotSchedule  # 或 ScheduleAnyway
      labelSelector:
        matchLabels:
          app: api-server
    - maxSkew: 1
      topologyKey: kubernetes.io/hostname       # 节点级别
      whenUnsatisfiable: ScheduleAnyway
      labelSelector:
        matchLabels:
          app: api-server

resources:
  requests:
    cpu: 250m       # 调度判定依据
    memory: 256Mi   # 调度判定依据
  limits:
    cpu: 1000m      # CFS 配额限制
    memory: 512Mi   # OOM Killer 阈值

# 节点压力驱逐：Kubelet 在资源不足时按优先级驱逐 Pod
kubectl describe node <node> | grep -A5 "Allocated resources"

# /var/lib/kubelet/config.yaml
evictionHard:
  memory.available: "100Mi"      # 可用内存 < 100Mi 触发驱逐
  nodefs.available: "10%"         # 节点文件系统 < 10% 触发驱逐
  imagefs.available: "15%"        # 镜像文件系统 < 15% 触发驱逐
evictionSoft:
  memory.available: "500Mi"       # 软阈值，有宽限期
evictionSoftGracePeriod:
  memory.available: "1m"

apiVersion: v1
kind: Pod
spec:
  affinity:
    nodeAffinity:
      # 必须满足（硬性要求）
      requiredDuringSchedulingIgnoredDuringExecution:
        nodeSelectorTerms:
          - matchExpressions:
              - key: node-type
                operator: In
                values: ["compute"]
              - key: topology.kubernetes.io/zone
                operator: In
                values: ["cn-beijing-a", "cn-beijing-b"]
      # 尽量满足（软性偏好）
      preferredDuringSchedulingIgnoredDuringExecution:
        - weight: 100
          preference:
            matchExpressions:
              - key: disk-type
                operator: In
                values: ["ssd"]
        - weight: 50
          preference:
            matchExpressions:
              - key: node.kubernetes.io/instance-type
                operator: In
                values: ["ecs.g7.xlarge"]

apiVersion: apps/v1
kind: Deployment
spec:
  template:
    spec:
      affinity:
        # Pod 反亲和性：副本分散到不同节点
        podAntiAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            - labelSelector:
                matchExpressions:
                  - key: app
                    operator: In
                    values: ["api-server"]
              topologyKey: kubernetes.io/hostname   # 不同节点
        # Pod 亲和性：Redis 跟随 API Server
        podAffinity:
          preferredDuringSchedulingIgnoredDuringExecution:
            - weight: 100
              podAffinityTerm:
                labelSelector:
                  matchLabels:
                    app: api-server
                topologyKey: kubernetes.io/hostname   # 同一节点

apiVersion: apps/v1
kind: Deployment
spec:
  replicas: 6
  template:
    spec:
      topologySpreadConstraints:
        - maxSkew: 1
          minDomains: 3          # 域名至少需要 3 个，不足则等待
          topologyKey: topology.kubernetes.io/zone
          whenUnsatisfiable: DoNotSchedule
          labelSelector:
            matchLabels:
              app: api-server
          matchLabelKeys:           # K8s 1.27+ 新增
            - pod-template-hash     # 自动引用 Pod 标签

# 给节点添加污点
kubectl taint nodes gpu-node-1 dedicated=gpu:NoSchedule

# 节点维护时驱逐所有 Pod
kubectl taint nodes node-1 maintenance=true:NoExecute

# 移除污点
kubectl taint nodes gpu-node-1 dedicated=gpu:NoSchedule-

apiVersion: v1
kind: Pod
spec:
  tolerations:
    # 完全匹配污点
    - key: dedicated
      operator: Equal
      value: gpu
      effect: NoSchedule
    # 匹配任意值
    - key: dedicated
      operator: Exists
      effect: NoSchedule
    # 匹配所有 NoExecute 污点（带宽限期）
    - operator: Exists
      effect: NoExecute
      tolerationSeconds: 3600   # 被驱逐前等待 1 小时
    # 匹配特定节点（绕过调度器直接绑定）
    - key: node-role.kubernetes.io/master
      operator: Exists
      effect: NoSchedule

apiVersion: scheduling.k8s.io/v1
kind: PriorityClass
metadata:
  name: high-priority
value: 1000000           # 越大越优先，数值无上限但建议有范围
preemptionPolicy: PreemptLowerPriority   # 或 Never
globalDefault: false
description: "核心业务服务，可驱逐低优先级 Pod"
---
apiVersion: scheduling.k8s.io/v1
kind: PriorityClass
metadata:
  name: low-priority
value: 1000
preemptionPolicy: Never               # 永不抢占，但可被高优先级抢占
description: "批处理任务，不重要"

apiVersion: v1
kind: Pod
spec:
  priorityClassName: high-priority

# Pod 指定调度器
apiVersion: v1
kind: Pod
spec:
  schedulerName: gpu-scheduler   # 不是默认的 default-scheduler

package myscheduler

import (
    "context"
    "k8s.io/kubernetes/pkg/scheduler/framework"
)

type CustomPlugin struct{}

var _ framework.ScorePlugin = &CustomPlugin{}

func (p *CustomPlugin) Name() string { return "CustomPlugin" }

func (p *CustomPlugin) Score(ctx context.Context, state *framework.CycleState,
    pod *v1.Pod, nodeName string) (int64, *framework.Status) {
    // 自定义评分逻辑：如节点上的 GPU 利用率、网络延迟
    return 50, nil  // 0-100
}

func (p *CustomPlugin) ScoreExtensions() framework.ScoreExtensions {
    return nil
}

# 1. 查看 Pod 事件（核心）
kubectl describe pod <pod-name>

# 2. 典型输出分析
Events:
  Type     Reason            Age   From          Message
  ----     ------            ----  ----          -------
  Warning  FailedScheduling  10s   scheduler     0/3 nodes are available:
    1 node(s) had taint {dedicated=gpu:NoSchedule}
    1 node(s) had insufficient memory
    1 node(s) had volume node affinity conflict

# 原生 K8s
kubectl logs -n kube-system -l component=kube-scheduler

# 特定调度器 Pod
kubectl logs -n kube-system kube-scheduler-master-1

# K8s 1.28+ 的 dry-run 调度（不实际创建 Pod）
kubectl apply -f pod.yaml --dry-run=server

resources:
  requests:
    cpu: "100m"       # 保证调度时使用
    memory: "128Mi"   # 保证调度时使用
  limits:
    cpu: "500m"       # CFS 配额，可选设置
    memory: "256Mi"   # 必须设置，防止 OOM 导致节点不稳定

spec:
  affinity:
    podAntiAffinity:
      preferredDuringSchedulingIgnoredDuringExecution:
        - weight: 100
          podAffinityTerm:
            labelSelector:
              matchLabels: { app: api-server }
            topologyKey: topology.kubernetes.io/zone    # 跨可用区
  topologySpreadConstraints:
    - maxSkew: 1
      topologyKey: kubernetes.io/hostname              # 跨节点
      whenUnsatisfiable: ScheduleAnyway
      labelSelector:
        matchLabels: { app: api-server }

# GPU 节点专用
kubectl taint nodes gpu-node-1 nvidia.com/gpu=true:NoSchedule
kubectl label nodes gpu-node-1 node-type=gpu

# Pod 配置
tolerations:
  - key: nvidia.com/gpu
    operator: Equal
    value: "true"
    effect: NoSchedule
nodeSelector:
  node-type: gpu