EKS Pod 卡在 Pending？從 Too Many Pods 到 ENI / CNI 限制全面解析

$ kubectl get pods -n default
NAME                                    READY   STATUS    RESTARTS   AGE
mycompany-strapi-prod-695854fbd4-dzw66  0/1     Pending   0          3h42m

$ kubectl describe pod mycompany-strapi-prod-695854fbd4-dzw66 -n default

Name:           mycompany-strapi-prod-695854fbd4-dzw66
Namespace:      default
Priority:       0
Node:           <none>
Labels:         app=strapi
                pod-template-hash=695854fbd4
Annotations:    <none>
Status:         Pending
IP:
IPs:            <none>
Controlled By:  ReplicaSet/mycompany-strapi-prod-695854fbd4

Containers:
  strapi:
    Image:      mycompany/strapi:v1.2.3
    Port:       1337/TCP
    Host Port:  0/TCP
    Requests:
      cpu:      250m
      memory:   512Mi
    Limits:
      cpu:      500m
      memory:   1Gi

Events:
  Type     Reason            Age   From               Message
  ----     ------            ----  ----               -------
  Warning  FailedScheduling  3h    default-scheduler  0/1 nodes are available:
                                                       1 Too many pods.

$ kubectl describe node ip-10-0-1-xxx.ap-northeast-1.compute.internal

Name:               ip-10-0-1-xxx.ap-northeast-1.compute.internal
Roles:              <none>
Labels:             beta.kubernetes.io/arch=amd64
                    beta.kubernetes.io/instance-type=t3.medium
                    beta.kubernetes.io/os=linux
                    kubernetes.io/arch=amd64
                    kubernetes.io/hostname=ip-10-0-1-xxx.ap-northeast-1.compute.internal
                    kubernetes.io/os=linux
                    node.kubernetes.io/instance-type=t3.medium
                    topology.kubernetes.io/region=ap-northeast-1
                    topology.kubernetes.io/zone=ap-northeast-1a

Allocated resources:
  (Total limits may be over 100 percent, i.e., overcommitted.)
  Resource           Requests      Limits
  --------           --------      ------
  cpu                680m (35%)    1200m (61%)
  memory             756Mi (22%)   1536Mi (44%)
  ephemeral-storage  0 (0%)        0 (0%)
  hugepages-1Gi      0 (0%)        0 (0%)
  hugepages-2Mi      0 (0%)        0 (0%)

$ kubectl describe node <node-name> | grep -A 10 "Capacity"

Capacity:
  attachable-volumes-aws-ebs:  25
  cpu:                         2
  ephemeral-storage:           20959212Ki
  hugepages-1Gi:               0
  hugepages-2Mi:               0
  memory:                      3943376Ki
  pods:                        17          # 👈 注意這裡！

Allocatable:
  attachable-volumes-aws-ebs:  25
  cpu:                         1930m
  ephemeral-storage:           19316009748
  hugepages-1Gi:               0
  hugepages-2Mi:               0
  memory:                      3388496Ki
  pods:                        17          # 👈 最大 Pod 數量是 17！

$ kubectl get pods -A --field-selector spec.nodeName=ip-10-0-1-xxx --no-headers | wc -l
17

#!/bin/bash
# quick-diagnosis.sh - EKS Pod Capacity 快速診斷腳本

echo "======================================"
echo "EKS Pod Capacity 診斷報告"
echo "======================================"
echo ""

# 取得所有 Node
for node in $(kubectl get nodes -o name); do
    node_name=$(echo $node | cut -d'/' -f2)

    echo "📍 Node: $node_name"
    echo "-----------------------------------"

    # 取得 Instance 類型
    instance_type=$(kubectl get $node -o jsonpath='{.metadata.labels.node\.kubernetes\.io/instance-type}')
    echo "   Instance Type: $instance_type"

    # 取得 Pod 容量
    max_pods=$(kubectl get $node -o jsonpath='{.status.allocatable.pods}')
    echo "   Max Pods: $max_pods"

    # 取得目前 Pod 數
    current_pods=$(kubectl get pods -A --field-selector spec.nodeName=$node_name --no-headers 2>/dev/null | wc -l)
    echo "   Current Pods: $current_pods"

    # 計算使用率
    if [ "$max_pods" -gt 0 ]; then
        usage=$((current_pods * 100 / max_pods))
        echo "   Usage: ${usage}%"

        # 警告訊息
        if [ $usage -ge 90 ]; then
            echo "   ⚠️  WARNING: Pod 容量即將耗盡！"
        elif [ $usage -ge 80 ]; then
            echo "   ⚡ NOTICE: Pod 容量使用率較高"
        else
            echo "   ✅ OK: Pod 容量充足"
        fi
    fi

    echo ""
done

echo "======================================"
echo "診斷完成"
echo "======================================"

# 查看目前的 Node Group
eksctl get nodegroup --cluster=my-cluster

# 擴展 Node 數量
eksctl scale nodegroup \
  --cluster=my-cluster \
  --name=my-nodegroup \
  --nodes=3 \
  --nodes-min=2 \
  --nodes-max=5

aws eks update-nodegroup-config \
  --cluster-name my-cluster \
  --nodegroup-name my-nodegroup \
  --scaling-config minSize=2,maxSize=5,desiredSize=3

# nodegroup-config.yaml
apiVersion: eksctl.io/v1alpha5
kind: ClusterConfig

metadata:
  name: my-cluster
  region: ap-northeast-1

managedNodeGroups:
  - name: production-ng
    instanceType: t3.large    # 從 t3.medium 升級
    desiredCapacity: 2
    minSize: 1
    maxSize: 4
    volumeSize: 50
    volumeType: gp3
    labels:
      role: production
      environment: prod
    tags:
      Owner: DevOps
      Project: MyApp
    iam:
      withAddonPolicies:
        autoScaler: true
        cloudWatch: true

# 建立新的 Node Group
eksctl create nodegroup -f nodegroup-config.yaml

# 刪除舊的 Node Group（確保 Pod 已遷移後）
eksctl delete nodegroup \
  --cluster=my-cluster \
  --name=old-nodegroup

# 在 AWS Console 或使用 CLI 為 Auto Scaling Group 加上這些 tag
aws autoscaling create-or-update-tags \
  --tags \
    "ResourceId=my-asg-name,ResourceType=auto-scaling-group,Key=k8s.io/cluster-autoscaler/enabled,Value=true,PropagateAtLaunch=true" \
    "ResourceId=my-asg-name,ResourceType=auto-scaling-group,Key=k8s.io/cluster-autoscaler/my-cluster,Value=owned,PropagateAtLaunch=true"

# cluster-autoscaler.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: cluster-autoscaler
  namespace: kube-system
  labels:
    app: cluster-autoscaler
spec:
  replicas: 1
  selector:
    matchLabels:
      app: cluster-autoscaler
  template:
    metadata:
      labels:
        app: cluster-autoscaler
    spec:
      serviceAccountName: cluster-autoscaler
      containers:
        - name: cluster-autoscaler
          image: registry.k8s.io/autoscaling/cluster-autoscaler:v1.28.0
          command:
            - ./cluster-autoscaler
            - --v=4
            - --stderrthreshold=info
            - --cloud-provider=aws
            - --skip-nodes-with-local-storage=false
            - --expander=least-waste
            - --node-group-auto-discovery=asg:tag=k8s.io/cluster-autoscaler/enabled,k8s.io/cluster-autoscaler/my-cluster
            - --balance-similar-node-groups
            - --skip-nodes-with-system-pods=false
          resources:
            limits:
              cpu: 100m
              memory: 600Mi
            requests:
              cpu: 100m
              memory: 600Mi
          env:
            - name: AWS_REGION
              value: ap-northeast-1

kubectl apply -f cluster-autoscaler.yaml

{
  "Version": "2012-10-17",
  "Statement": [
    {
      "Effect": "Allow",
      "Action": [
        "autoscaling:DescribeAutoScalingGroups",
        "autoscaling:DescribeAutoScalingInstances",
        "autoscaling:DescribeLaunchConfigurations",
        "autoscaling:DescribeScalingActivities",
        "autoscaling:DescribeTags",
        "autoscaling:SetDesiredCapacity",
        "autoscaling:TerminateInstanceInAutoScalingGroup",
        "ec2:DescribeLaunchTemplateVersions",
        "ec2:DescribeInstanceTypes"
      ],
      "Resource": "*"
    }
  ]
}

# Step 1: 更新 VPC CNI 到最新版本
kubectl describe daemonset aws-node -n kube-system | grep Image

# Step 2: 啟用 Prefix Delegation
kubectl set env daemonset aws-node \
  -n kube-system \
  ENABLE_PREFIX_DELEGATION=true

# Step 3: 設定 warm pool（預先分配的 IP prefix 數量）
kubectl set env daemonset aws-node \
  -n kube-system \
  WARM_PREFIX_TARGET=1

# Step 4: 驗證設定
kubectl get ds aws-node -n kube-system -o yaml | grep -A 5 "env:"

#!/bin/bash
# verify-prefix-delegation.sh

echo "檢查 VPC CNI 設定..."

# 檢查 CNI 版本
cni_version=$(kubectl get ds aws-node -n kube-system -o jsonpath='{.spec.template.spec.containers[0].image}' | grep -oP 'v\d+\.\d+\.\d+')
echo "VPC CNI 版本: $cni_version"

# 檢查 Prefix Delegation 狀態
prefix_enabled=$(kubectl get ds aws-node -n kube-system -o jsonpath='{.spec.template.spec.containers[0].env[?(@.name=="ENABLE_PREFIX_DELEGATION")].value}')
echo "Prefix Delegation: ${prefix_enabled:-未啟用}"

# 檢查 Node 的 max-pods
for node in $(kubectl get nodes -o name); do
    node_name=$(echo $node | cut -d'/' -f2)
    max_pods=$(kubectl get $node -o jsonpath='{.status.allocatable.pods}')
    echo "Node $node_name - Max Pods: $max_pods"
done

# fargate-profile.yaml
apiVersion: eksctl.io/v1alpha5
kind: ClusterConfig

metadata:
  name: my-cluster
  region: ap-northeast-1

fargateProfiles:
  - name: fp-default
    selectors:
      # Namespace 選擇器
      - namespace: staging
        labels:
          compute-type: fargate
      - namespace: dev
      # 特定 workload 選擇器
      - namespace: production
        labels:
          workload-type: stateless

eksctl create fargateprofile -f fargate-profile.yaml

# deployment-fargate.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: my-app
  namespace: staging    # 匹配 Fargate Profile 的 namespace
spec:
  replicas: 3
  selector:
    matchLabels:
      app: my-app
  template:
    metadata:
      labels:
        app: my-app
        compute-type: fargate    # 匹配 Fargate Profile 的 label
    spec:
      containers:
        - name: app
          image: my-app:latest
          resources:
            requests:
              cpu: "256m"
              memory: "512Mi"
            limits:
              cpu: "512m"
              memory: "1Gi"

# 查看所有 Pod 的資源使用狀況（按 memory 排序）
kubectl top pods -A --sort-by=memory

# 查看所有 Pod（按啟動時間排序）
kubectl get pods -A -o wide --sort-by='.status.startTime'

# 找出長時間運行的 Pod
kubectl get pods -A -o custom-columns=\
NAMESPACE:.metadata.namespace,\
NAME:.metadata.name,\
STATUS:.status.phase,\
AGE:.metadata.creationTimestamp \
--sort-by='.metadata.creationTimestamp'

# 刪除特定 Pod（如果有 Deployment，會自動重建）
kubectl delete pod <pod-name> -n <namespace>

# 完全移除一個服務
kubectl delete deployment <deployment-name> -n <namespace>

# 檢查是否有 Completed 或 Failed 的 Pod 可以清理
kubectl get pods -A | grep -E 'Completed|Error|Failed'
kubectl delete pods -A --field-selector=status.phase==Succeeded
kubectl delete pods -A --field-selector=status.phase==Failed

# 使用 AWS CLI 建立告警
aws cloudwatch put-metric-alarm \
  --alarm-name "EKS-Pod-Capacity-Warning" \
  --alarm-description "Pod capacity usage exceeds 80%" \
  --metric-name "pod_count" \
  --namespace "ContainerInsights" \
  --dimensions Name=ClusterName,Value=my-cluster \
  --statistic Average \
  --period 300 \
  --threshold 80 \
  --comparison-operator GreaterThanThreshold \
  --evaluation-periods 2 \
  --alarm-actions arn:aws:sns:ap-northeast-1:123456789012:alerts

# prometheus-rules.yaml
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: pod-capacity-alerts
  namespace: monitoring
spec:
  groups:
    - name: pod-capacity
      rules:
        - alert: HighPodUtilization
          expr: |
            sum(kube_pod_status_phase{phase="Running"}) by (node)
            /
            sum(kube_node_status_allocatable{resource="pods"}) by (node)
            > 0.8
          for: 5m
          labels:
            severity: warning
          annotations:
            summary: "Node {{ $labels.node }} Pod 使用率超過 80%"
            description: "Node {{ $labels.node }} 的 Pod 使用率已達 {{ $value | humanizePercentage }}，請考慮擴展。"

        - alert: CriticalPodUtilization
          expr: |
            sum(kube_pod_status_phase{phase="Running"}) by (node)
            /
            sum(kube_node_status_allocatable{resource="pods"}) by (node)
            > 0.95
          for: 2m
          labels:
            severity: critical
          annotations:
            summary: "Node {{ $labels.node }} Pod 容量即將耗盡！"
            description: "Node {{ $labels.node }} 的 Pod 使用率已達 {{ $value | humanizePercentage }}，可能導致新 Pod 無法排程。"

#!/bin/bash
# /usr/local/bin/check-pod-capacity.sh

LOG_FILE="/var/log/pod-capacity-check.log"
SLACK_WEBHOOK="https://hooks.slack.com/services/xxx/yyy/zzz"

echo "$(date): Starting pod capacity check" >> $LOG_FILE

WARNING_NODES=""

for node in $(kubectl get nodes -o name); do
    node_name=$(echo $node | cut -d'/' -f2)
    max_pods=$(kubectl get $node -o jsonpath='{.status.allocatable.pods}')
    current_pods=$(kubectl get pods -A --field-selector spec.nodeName=$node_name --no-headers | wc -l)

    if [ "$max_pods" -gt 0 ]; then
        usage=$((current_pods * 100 / max_pods))

        if [ $usage -ge 80 ]; then
            WARNING_NODES="$WARNING_NODES\n- $node_name: ${usage}% ($current_pods/$max_pods)"
        fi
    fi
done

if [ -n "$WARNING_NODES" ]; then
    # 發送 Slack 通知
    curl -X POST -H 'Content-type: application/json' \
        --data "{\"text\":\"⚠️ EKS Pod Capacity Warning\n\nThe following nodes have high pod utilization:$WARNING_NODES\"}" \
        $SLACK_WEBHOOK

    echo "$(date): Warning sent for nodes:$WARNING_NODES" >> $LOG_FILE
fi

echo "$(date): Check completed" >> $LOG_FILE

# 加入 crontab，每 15 分鐘檢查一次
*/15 * * * * /usr/local/bin/check-pod-capacity.sh