Kubernetes网络模型与CNI插件深度解析

# /etc/cni/net.d/10-calico.conflist
{
  "cniVersion": "0.3.1",
  "name": "k8s-pod-network",
  "plugins": [
    {
      "type": "calico",
      "log_level": "info",
      "datastore_type": "kubernetes",
      "nodename": "node-1",
      "ipam": {
        "type": "calico-ipam",
        "assign_ipv4": "true",
        "ipv4_pools": ["10.1.0.0/16"]
      }
    },
    {
      "type": "portmap",
      "snat": true,
      "capabilities": {"portMappings": true}
    }
  ]
}

apiVersion: v1
kind: ConfigMap
metadata:
  name: kube-flannel-cfg
  namespace: kube-flannel
data:
  cni-conf.json: |
    {
      "name": "cbr0",
      "cniVersion": "0.3.1",
      "plugins": [
        {
          "type": "flannel",
          "delegate": {
            "hairpinMode": true,
            "isDefaultGateway": true
          }
        }
      ]
    }
  net-conf.json: |
    {
      "Network": "10.1.0.0/16",
      "Backend": {
        "Type": "vxlan",
        "VNI": 1,
        "Port": 8472
      }
    }

apiVersion: projectcalico.org/v3
kind: IPPool
metadata:
  name: default-ipv4-ippool
spec:
  cidr: 10.1.0.0/16
  blockSize: 26              # 每个节点分配的 IP 块大小（64 个 IP）
  natOutgoing: true
  disabled: false
  nodeSelector: all()

# 启用 eBPF 模式
kubectl patch installation default --type=merge -p '{"spec": {"calicoNetwork": {"linuxDataplane": "BPF"}}}'

apiVersion: cilium.io/v2alpha1
kind: CiliumClusterwideNetworkPolicy
metadata:
  name: default-deny-ingress
spec:
  endpointSelector: {}
  ingressDeny:
    - {}
---
# 允许 frontend → backend 的 HTTP GET /api
apiVersion: cilium.io/v2
kind: CiliumNetworkPolicy
metadata:
  name: backend-policy
  namespace: production
spec:
  endpointSelector:
    matchLabels:
      app: backend
  ingress:
    - fromEndpoints:
        - matchLabels:
            app: frontend
      toPorts:
        - ports:
            - port: "8080"
              protocol: TCP
          rules:
            http:
              - method: GET
                path: "/api/.*"

# 安装 Hubble CLI
hubble status
hubble observe --namespace production --from-pod frontend
hubble observe --protocol http --http-status 500

# 查看 iptables 规则
iptables -t nat -L KUBE-SERVICES -n | grep api-service
# 输出：KUBE-MARK-MASQ + KUBE-SVC-... 链 + 概率分配到各 Pod

# ipvs 支持的负载均衡算法
ipvsadm -Ln | grep Scheduling
# 输出：rr (轮询) / lc (最少连接) / dh (目标哈希) / sh (源哈希) / sed (最短期望延迟) / nq (永不排队)

mode: "nftables"

spec:
  externalTrafficPolicy: Local   # 保留源 IP，但可能导致负载不均
  # 或
  externalTrafficPolicy: Cluster  # 默认，负载均衡但丢失源 IP

# GatewayClass：集群级别，定义控制器类型
apiVersion: gateway.networking.k8s.io/v1
kind: GatewayClass
metadata:
  name: nginx
spec:
  controllerName: gateway.nginx.org/nginx-gateway-controller

---
# Gateway：定义监听端口和 TLS
apiVersion: gateway.networking.k8s.io/v1
kind: Gateway
metadata:
  name: public-gateway
  namespace: ingress-nginx
spec:
  gatewayClassName: nginx
  listeners:
    - name: https
      protocol: HTTPS
      port: 443
      tls:
        mode: Terminate
        certificateRefs:
          - name: wildcard-cert
      allowedRoutes:
        namespaces:
          from: All   # 允许所有 namespace 绑定路由

---
# HTTPRoute：应用团队定义路由规则
apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata:
  name: api-routes
  namespace: production
spec:
  parentRefs:
    - name: public-gateway
      namespace: ingress-nginx
  hostnames:
    - api.example.com
  rules:
    - matches:
        - path:
            type: PathPrefix
            value: /orders
      backendRefs:
        - name: order-service
          port: 80
    - matches:
        - path:
            type: PathPrefix
            value: /users
      backendRefs:
        - name: user-service
          port: 80

apiVersion: v1
kind: ConfigMap
metadata:
  name: coredns
  namespace: kube-system
data:
  Corefile: |
    .:53 {
      errors
      health {
        lameduck 5s
      }
      ready
      # K8s 插件配置
      kubernetes cluster.local in-addr.arpa ip6.arpa {
        pods insecure       # 创建 Pod A 记录
        fallthrough in-addr.arpa ip6.arpa
        ttl 30              # 缓存 TTL
      }
      # 缓存配置
      cache 30 {
        success 9984 300    # 成功响应缓存 5 分钟
        denial 9984 60      # NXDOMAIN 缓存 1 分钟
      }
      # 转发到外部 DNS
      forward . /etc/resolv.conf {
        max_concurrent 1000
      }
      # 循环避免热点
      loadbalance round_robin
      prometheus :9153
    }

# 1. 查看 Pod IP 和所在节点
kubectl get pods -o wide

# 2. Pod 内执行网络诊断
kubectl exec -it <pod> -- /bin/sh
  # Ping 测试
  ping <target-pod-ip>
  # DNS 解析
  nslookup kubernetes.default.svc.cluster.local
  # 端口连通
  nc -zv <service-ip> 80

# 3. 查看 Service 的 Endpoint
kubectl get endpoints <service>

# 4. 跨节点抓包
kubectl debug node/<node> -it --image=nicolaka/netshoot
  tcpdump -i any -n host <pod-ip>

# Calico
calicoctl node status              # 查看 BGP 状态
calicoctl get ippool -o wide      # 查看 IP 池
kubectl logs -n calico-system -l k8s-app=calico-node

# Cilium
cilium status                      # Cilium 健康状态
cilium endpoint list               # 查看端点
cilium policy get                  # 查看策略
cilium monitor --type drop         # 查看被丢弃的包

# Flannel
kubectl logs -n kube-flannel -l app=flannel
# 查看子网分配
cat /run/flannel/subnet.env