┌─────────────────────────────────────────────────────────────┐
│                    Master 节点 (192.168.174.128)            │
│  ┌─────────────┐ ┌─────────────┐ ┌─────────────┐          │
│  │ API Server  │ │ Controller  │ │  Scheduler  │          │
│  │  (6443)     │ │  Manager    │ │ (10259)     │          │
│  └─────────────┘ └─────────────┘ └─────────────┘          │
│         │                │                │                  │
│         └────────────────┼────────────────┘                  │
│                          │                                   │
│                    ┌─────┴─────┐                            │
│                    │    etcd   │                            │
│                    │  (2379)   │                            │
│                    └───────────┘                            │
│                                                             │
│  ┌─────────────┐ ┌─────────────┐                           │
│  │   kubelet   │ │ kube-proxy  │                           │
│  │  (10250)    │ │             │                           │
│  └─────────────┘ └─────────────┘                           │
└─────────────────────────────────────────────────────────────┘
                          │
                    ┌─────┴─────┐
                    │  Flannel  │
                    │   CNI     │
                    └───────────┘
                          │
                    ┌─────┴─────┐
                    │   Docker  │
                    │ (containerd)│
                    └───────────┘

# 临时禁用 Swap
# 为什么：立即生效，不需要重启
sudo swapoff -a

# 永久禁用 Swap
# 为什么：重启后仍然禁用
# 怎么做：注释掉 /etc/fstab 中的 swap 行
sudo sed -i '/ swap / s/^\(.*\)$/#\1/g' /etc/fstab

# 验证 Swap 已禁用
# 预期输出：空（无任何内容）
sudo swapon --show

# 加载必要的内核模块
# 为什么：让内核支持容器网络功能
sudo modprobe overlay
sudo modprobe br_netfilter

# 永久加载
# 为什么：重启后自动加载
cat <<EOF | sudo tee /etc/modules-load.d/k8s.conf
overlay
br_netfilter
EOF

# 验证模块已加载
# 预期输出：显示 overlay 和 br_netfilter 模块
lsmod | grep -E "overlay|br_netfilter"

# 配置网络参数
# 为什么：让 Kubernetes 的网络功能正常工作
cat <<EOF | sudo tee /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-iptables  = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.ipv4.ip_forward                 = 1
EOF

# 应用配置
# 为什么：立即生效，不需要重启
sudo sysctl --system

# 验证配置
# 预期输出：三个参数都是 1
sysctl net.bridge.bridge-nf-call-iptables net.bridge.bridge-nf-call-ip6tables net.ipv4.ip_forward

# 安装依赖包
# 为什么：Docker 需要这些工具来添加仓库和管理包
sudo yum install -y yum-utils device-mapper-persistent-data lvm2

# 添加 Docker 仓库
# 为什么：从官方源安装最新稳定版
sudo yum-config-manager --add-repo https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo

# 安装 Docker
# 为什么：Docker 是容器运行时，K8s 需要它来运行容器
sudo yum install -y docker-ce docker-ce-cli containerd.io

# 启动 Docker
# 为什么：让 Docker 服务在后台运行
sudo systemctl daemon-reload
sudo systemctl enable docker
sudo systemctl start docker

# 创建 Docker 配置目录
# 为什么：Docker 的配置文件需要放在这个目录
sudo mkdir -p /etc/docker

# 创建 Docker 配置文件
# 为什么：让 Docker 与 Kubernetes 兼容
cat <<EOF | sudo tee /etc/docker/daemon.json
{
  "exec-opts": ["native.cgroupdriver=systemd"],
  "log-driver": "json-file",
  "log-opts": {
    "max-size": "100m"
  },
  "storage-driver": "overlay2"
}
EOF

# 重启 Docker
# 为什么：让配置生效
sudo systemctl daemon-reload
sudo systemctl restart docker

# 检查 Docker 版本
# 预期输出：显示 Docker 版本信息
docker --version

# 检查 Docker 信息
# 预期输出：Cgroup Driver: systemd
docker info | grep -i cgroup

# 测试 Docker 运行（需要联网，离线环境可跳过）
# 预期输出：Hello from Docker!
docker run hello-world

# 创建临时目录
# 为什么：下载的文件需要先存放在这里
cd /tmp

# 下载 etcd
# 为什么：etcd 是 K8s 的数据库，存储集群状态
curl -L "https://github.com/etcd-io/etcd/releases/download/v3.5.12/etcd-v3.5.12-linux-amd64.tar.gz" -o etcd.tar.gz

# 解压
# 为什么：获取二进制文件
tar xzf etcd.tar.gz

# 安装到系统路径
# 为什么：让系统可以全局访问这些命令
cp etcd-v3.5.12-linux-amd64/etcd /usr/local/bin/
cp etcd-v3.5.12-linux-amd64/etcdctl /usr/local/bin/

# 验证安装
# 预期输出：显示 etcd 版本信息
etcd --version

# 定义版本变量
# 为什么：方便后续使用和维护
K8S_VERSION="1.28.15"

# 下载核心组件
# 为什么：这些是 K8s 的核心二进制文件
for bin in kube-apiserver kube-controller-manager kube-scheduler kubelet kube-proxy kubectl; do
    echo "Downloading ${bin}..."
    curl -sL "https://dl.k8s.io/release/v${K8S_VERSION}/bin/linux/amd64/${bin}" -o /usr/local/bin/${bin}
    chmod +x /usr/local/bin/${bin}
done

# 验证安装
# 预期输出：显示各组件版本信息
kubectl version --client

# 创建证书目录
# 为什么：存放所有证书文件
mkdir -p /etc/kubernetes/pki

# 生成 CA 私钥
# 为什么：私钥用于签发其他证书，必须保密
openssl genrsa -out /etc/kubernetes/pki/ca.key 2048

# 生成 CA 证书
# 为什么：CA 证书用于验证其他证书的合法性
openssl req -new -x509 -days 3650 -key /etc/kubernetes/pki/ca.key \
    -out /etc/kubernetes/pki/ca.crt -subj "/CN=kubernetes-ca"

# 生成 API Server 私钥
openssl genrsa -out /etc/kubernetes/pki/apiserver.key 2048

# 生成证书签名请求
# 为什么：需要指定证书的使用者和备用名称
cat <<CSR > /tmp/apiserver-csr.conf
[req]
req_extensions = v3_req
distinguished_name = req_distinguished_name
[req_distinguished_name]
[v3_req]
keyUsage = digitalSignature, keyEncipherment
extendedKeyUsage = serverAuth
subjectAltName = @alt_names
[alt_names]
DNS.1 = kubernetes
DNS.2 = kubernetes.default
DNS.3 = kubernetes.default.svc.cluster.local
IP.1 = 10.96.0.1
IP.2 = 192.168.174.128
IP.3 = 127.0.0.1
CSR
openssl req -new -key /etc/kubernetes/pki/apiserver.key \
    -out /etc/kubernetes/pki/apiserver.csr -subj "/CN=kube-apiserver" -config /tmp/apiserver-csr.conf

# 创建扩展配置文件
cat <<EXT > /tmp/apiserver-ext.conf
[v3_req]
keyUsage = digitalSignature, keyEncipherment
extendedKeyUsage = serverAuth
subjectAltName = @alt_names
[alt_names]
DNS.1 = kubernetes
DNS.2 = kubernetes.default
DNS.3 = kubernetes.default.svc.cluster.local
IP.1 = 10.96.0.1
IP.2 = 192.168.174.128
IP.3 = 127.0.0.1
EXT

# 签发证书
# 为什么：使用 CA 私钥签发，证明这个证书是合法的
openssl x509 -req -days 3650 -in /etc/kubernetes/pki/apiserver.csr \
    -CA /etc/kubernetes/pki/ca.crt -CAkey /etc/kubernetes/pki/ca.key \
    -CAcreateserial -out /etc/kubernetes/pki/apiserver.crt \
    -extensions v3_req -extfile /tmp/apiserver-ext.conf

# Service Account 密钥
# 为什么：用于创建和验证 Service Account Token
openssl genrsa -out /etc/kubernetes/pki/sa.key 2048
openssl req -new -x509 -days 3650 -key /etc/kubernetes/pki/sa.key \
    -out /etc/kubernetes/pki/sa.crt -subj "/CN=service-accounts"

# Admin 证书（必须由 CA 签发！）
# 为什么：kubectl 使用这个证书连接 API Server
# 注意：如果使用自签名证书，kubectl 会报 "Unauthorized" 错误
openssl genrsa -out /etc/kubernetes/pki/admin.key 2048
openssl req -new -key /etc/kubernetes/pki/admin.key \
    -out /etc/kubernetes/pki/admin.csr -subj "/CN=kubernetes-admin/O=system:masters"
openssl x509 -req -days 3650 -in /etc/kubernetes/pki/admin.csr \
    -CA /etc/kubernetes/pki/ca.crt -CAkey /etc/kubernetes/pki/ca.key \
    -CAcreateserial -out /etc/kubernetes/pki/admin.crt

# Controller Manager 证书（必须由 CA 签发！）
# 为什么：Controller Manager 使用这个证书连接 API Server
openssl genrsa -out /etc/kubernetes/pki/controller-manager.key 2048
openssl req -new -key /etc/kubernetes/pki/controller-manager.key \
    -out /etc/kubernetes/pki/controller-manager.csr \
    -subj "/CN=system:kube-controller-manager/O=system:kube-controller-managers"
openssl x509 -req -days 3650 -in /etc/kubernetes/pki/controller-manager.csr \
    -CA /etc/kubernetes/pki/ca.crt -CAkey /etc/kubernetes/pki/ca.key \
    -CAcreateserial -out /etc/kubernetes/pki/controller-manager.crt

# Scheduler 证书（必须由 CA 签发！）
# 为什么：Scheduler 使用这个证书连接 API Server
openssl genrsa -out /etc/kubernetes/pki/scheduler.key 2048
openssl req -new -key /etc/kubernetes/pki/scheduler.key \
    -out /etc/kubernetes/pki/scheduler.csr \
    -subj "/CN=system:kube-scheduler/O=system:kube-schedulers"
openssl x509 -req -days 3650 -in /etc/kubernetes/pki/scheduler.csr \
    -CA /etc/kubernetes/pki/ca.crt -CAkey /etc/kubernetes/pki/ca.key \
    -CAcreateserial -out /etc/kubernetes/pki/scheduler.crt

# 生成随机 Token
# 为什么：用于 kubelet 的初始认证
TOKEN=$(head -c 16 /dev/urandom | od -An -tx1 | tr -d ' \n')

# 创建 Token 文件
# 为什么：API Server 需要这个文件来验证 kubelet 的身份
cat <<EOF > /etc/kubernetes/token.csv
${TOKEN},kubelet-bootstrap,10001,"system:kubelet-bootstrap"
EOF

# 创建 Bootstrap Kubeconfig
# 为什么：kubelet 使用这个文件向 API Server 注册
cat <<EOF > /etc/kubernetes/bootstrap-kubelet.conf
apiVersion: v1
kind: Config
clusters:
- cluster:
    certificate-authority: /etc/kubernetes/pki/ca.crt
    server: https://192.168.174.128:6443
  name: kubernetes
contexts:
- context:
    cluster: kubernetes
    user: kubelet-bootstrap
  name: kubelet-bootstrap@kubernetes
current-context: kubelet-bootstrap@kubernetes
users:
- name: kubelet-bootstrap
  user:
    token: ${TOKEN}
EOF

# 创建 Admin Kubeconfig
# 为什么：kubectl 使用这个文件连接 API Server
cat <<EOF > /etc/kubernetes/admin.conf
apiVersion: v1
kind: Config
clusters:
- cluster:
    certificate-authority: /etc/kubernetes/pki/ca.crt
    server: https://192.168.174.128:6443
  name: kubernetes
contexts:
- context:
    cluster: kubernetes
    user: kubernetes-admin
  name: kubernetes-admin@kubernetes
current-context: kubernetes-admin@kubernetes
users:
- name: kubernetes-admin
  user:
    client-certificate: /etc/kubernetes/pki/admin.crt
    client-key: /etc/kubernetes/pki/admin.key
EOF

# 创建 Controller Manager Kubeconfig
# 为什么：Controller Manager 使用这个文件连接 API Server
cat <<EOF > /etc/kubernetes/controller-manager.conf
apiVersion: v1
kind: Config
clusters:
- cluster:
    certificate-authority: /etc/kubernetes/pki/ca.crt
    server: https://127.0.0.1:6443
  name: kubernetes
contexts:
- context:
    cluster: kubernetes
    user: system:kube-controller-manager
  name: system:kube-controller-manager@kubernetes
current-context: system:kube-controller-manager@kubernetes
users:
- name: system:kube-controller-manager
  user:
    client-certificate: /etc/kubernetes/pki/controller-manager.crt
    client-key: /etc/kubernetes/pki/controller-manager.key
EOF

# 创建 Scheduler Kubeconfig
# 为什么：Scheduler 使用这个文件连接 API Server
cat <<EOF > /etc/kubernetes/scheduler.conf
apiVersion: v1
kind: Config
clusters:
- cluster:
    certificate-authority: /etc/kubernetes/pki/ca.crt
    server: https://127.0.0.1:6443
  name: kubernetes
contexts:
- context:
    cluster: kubernetes
    user: system:kube-scheduler
  name: system:kube-scheduler@kubernetes
current-context: system:kube-scheduler@kubernetes
users:
- name: system:kube-scheduler
  user:
    client-certificate: /etc/kubernetes/pki/scheduler.crt
    client-key: /etc/kubernetes/pki/scheduler.key
EOF

# 复制到 kubectl 默认路径
# 为什么：让 kubectl 能自动使用正确的配置
mkdir -p $HOME/.kube
cp /etc/kubernetes/admin.conf $HOME/.kube/config

cat <<'EOF' > /etc/systemd/system/etcd.service
[Unit]
Description=etcd
After=network-online.target
Wants=network-online.target

[Service]
ExecStart=/usr/local/bin/etcd \
  --name=etcd \
  --data-dir=/var/lib/etcd \
  --listen-client-urls=http://0.0.0.0:2379 \
  --advertise-client-urls=http://192.168.174.128:2379 \
  --listen-peer-urls=http://0.0.0.0:2380 \
  --initial-advertise-peer-urls=http://192.168.174.128:2380 \
  --initial-cluster=etcd=http://192.168.174.128:2380
Restart=always
RestartSec=10

[Install]
WantedBy=multi-user.target
EOF

cat <<'EOF' > /etc/systemd/system/kube-apiserver.service
[Unit]
Description=Kubernetes API Server
After=network-online.target etcd.service
Wants=network-online.target

[Service]
ExecStart=/usr/local/bin/kube-apiserver \
  --etcd-servers=http://127.0.0.1:2379 \
  --service-cluster-ip-range=10.96.0.0/12 \
  --bind-address=0.0.0.0 \
  --secure-port=6443 \
  --advertise-address=192.168.174.128 \
  --allow-privileged=true \
  --service-node-port-range=30000-32767 \
  --enable-admission-plugins=NodeRestriction \
  --authorization-mode=Node,RBAC \
  --enable-bootstrap-token-auth=true \
  --token-auth-file=/etc/kubernetes/token.csv \
  --tls-cert-file=/etc/kubernetes/pki/apiserver.crt \
  --tls-private-key-file=/etc/kubernetes/pki/apiserver.key \
  --client-ca-file=/etc/kubernetes/pki/ca.crt \
  --service-account-key-file=/etc/kubernetes/pki/sa.key \
  --service-account-signing-key-file=/etc/kubernetes/pki/sa.key \
  --service-account-issuer=https://kubernetes.default.svc \
  --kubelet-client-certificate=/etc/kubernetes/pki/admin.crt \
  --kubelet-client-key=/etc/kubernetes/pki/admin.key
Restart=always
RestartSec=10

[Install]
WantedBy=multi-user.target
EOF

cat <<'EOF' > /etc/systemd/system/kube-controller-manager.service
[Unit]
Description=Kubernetes Controller Manager
After=network-online.target kube-apiserver.service
Wants=network-online.target

[Service]
ExecStart=/usr/local/bin/kube-controller-manager \
  --kubeconfig=/etc/kubernetes/controller-manager.conf \
  --cluster-cidr=10.244.0.0/16 \
  --service-cluster-ip-range=10.96.0.0/12 \
  --cluster-name=kubernetes \
  --cluster-signing-cert-file=/etc/kubernetes/pki/ca.crt \
  --cluster-signing-key-file=/etc/kubernetes/pki/ca.key \
  --service-account-private-key-file=/etc/kubernetes/pki/sa.key \
  --root-ca-file=/etc/kubernetes/pki/ca.crt \
  --leader-elect=true \
  --allocate-node-cidrs=true
Restart=always
RestartSec=10

[Install]
WantedBy=multi-user.target
EOF

cat <<'EOF' > /etc/systemd/system/kube-scheduler.service
[Unit]
Description=Kubernetes Scheduler
After=network-online.target kube-apiserver.service
Wants=network-online.target

[Service]
ExecStart=/usr/local/bin/kube-scheduler \
  --kubeconfig=/etc/kubernetes/scheduler.conf \
  --leader-elect=true
Restart=always
RestartSec=10

[Install]
WantedBy=multi-user.target
EOF

# 创建 kubelet 配置文件
# 为什么：kubelet 需要配置 cgroup、DNS 等参数
mkdir -p /var/lib/kubelet

cat <<EOF > /var/lib/kubelet/config.yaml
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
cgroupDriver: systemd
clusterDNS:
- 10.96.0.10
clusterDomain: cluster.local
resolvConf: /run/systemd/resolve/resolv.conf
rotateCertificates: true
serverTLSBootstrap: true
EOF

# 创建 kubelet 服务
cat <<'EOF' > /etc/systemd/system/kubelet.service
[Unit]
Description=kubelet
After=network-online.target
Wants=network-online.target

[Service]
ExecStart=/usr/local/bin/kubelet \
  --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \
  --kubeconfig=/var/lib/kubelet/kubeconfig \
  --config=/var/lib/kubelet/config.yaml \
  --pod-infra-container-image=registry.aliyuncs.com/google_containers/pause:3.8 \
  --node-ip=192.168.174.128 \
  --client-ca-file=/etc/kubernetes/pki/ca.crt
Restart=always
RestartSec=10

[Install]
WantedBy=multi-user.target
EOF

cat <<'EOF' > /etc/systemd/system/kube-proxy.service
[Unit]
Description=Kubernetes Kube-Proxy
After=network-online.target kube-apiserver.service
Wants=network-online.target

[Service]
ExecStart=/usr/local/bin/kube-proxy \
  --kubeconfig=/etc/kubernetes/admin.conf \
  --cluster-cidr=10.244.0.0/16 \
  --proxy-mode=iptables
Restart=always
RestartSec=10

[Install]
WantedBy=multi-user.target
EOF

# 重新加载 systemd 配置
# 为什么：让 systemd 识别新创建的服务文件
systemctl daemon-reload

# 启动 containerd（容器运行时）
# 为什么：kubelet 需要容器运行时来管理容器
systemctl enable containerd
systemctl start containerd

# 启动 etcd
# 为什么：etcd 是基础，其他组件都依赖它
systemctl enable etcd
systemctl start etcd
sleep 3

# 验证 etcd 启动
# 预期输出：显示 etcd 版本信息
curl -s http://localhost:2379/version

# 启动 kube-apiserver
# 为什么：API Server 是所有组件的入口
systemctl enable kube-apiserver
systemctl start kube-apiserver
sleep 5

# 验证 API Server 启动
# 预期输出：显示 K8s 版本信息
curl -k https://localhost:6443/version

# 启动 Controller Manager 和 Scheduler
# 为什么：它们都依赖 API Server
systemctl enable kube-controller-manager kube-scheduler
systemctl start kube-controller-manager kube-scheduler
sleep 3

# 验证组件状态
# 预期输出：所有组件都是 Healthy
kubectl get componentstatuses

# 创建 resolv.conf
# 为什么：kubelet 和容器需要 DNS 解析
mkdir -p /run/systemd/resolve
cp /etc/resolv.conf /run/systemd/resolve/resolv.conf

# :warning: 清理旧的 pod sandbox（重要！）
# 为什么：如果有之前的部署，旧的挂载点会导致 kubelet 无法正常工作
# 解决方案：停止 containerd，杀死 shim 进程，lazy unmount，删除目录
systemctl stop containerd 2>/dev/null || true
pkill -9 -f "containerd-shim" 2>/dev/null || true
for mount in $(mount 2>/dev/null | grep "kubelet/pods.*projected" | awk '{print $3}'); do
    umount -l "$mount" 2>/dev/null || true
done
rm -rf /var/lib/kubelet/pods 2>/dev/null || true

# 重新部署时必须清理旧的 kubelet 证书
# 为什么：旧证书会导致 kubelet 无法向 API Server 注册，报 "Unauthorized" 错误
rm -rf /var/lib/kubelet/pki 2>/dev/null || true

# 重启 containerd
systemctl start containerd

# 创建 RBAC 绑定（必须在启动 kubelet 之前！）
# 为什么：让 kubelet、controller-manager、scheduler 有权限操作资源
# :warning: 如果跳过这一步，kubelet 会报 "forbidden" 错误
kubectl create clusterrolebinding kubelet-bootstrap --clusterrole=system:node-bootstrapper --user=kubelet-bootstrap 2>/dev/null || true
kubectl create clusterrolebinding kubelet-anon --clusterrole=system:node --user=system:anonymous 2>/dev/null || true
kubectl create clusterrolebinding kubelet-nodes --clusterrole=system:node --group=system:nodes 2>/dev/null || true
kubectl create clusterrolebinding cm-admin --clusterrole=cluster-admin --user=system:kube-controller-manager 2>/dev/null || true
kubectl create clusterrolebinding sched-admin --clusterrole=cluster-admin --user=system:kube-scheduler 2>/dev/null || true

# 重启服务以加载新证书
systemctl restart kube-controller-manager kube-scheduler
sleep 5

# 启动 kubelet
# 为什么：kubelet 是节点的代理，管理 Pod
systemctl enable kubelet
systemctl start kubelet

# 等待 kubelet 启动
sleep 15

# 批准 CSR
# 为什么：kubelet 需要证书才能与 API Server 通信
kubectl get csr
for csr in $(kubectl get csr --no-headers | grep Pending | awk '{print $1}'); do
    kubectl certificate approve $csr
done

# 启动 kube-proxy
# 为什么：kube-proxy 设置 ClusterIP 路由，flannel 需要它才能访问 API Server
# :warning: 必须在 flannel 之前启动！
systemctl enable kube-proxy
systemctl start kube-proxy

# 等待节点 Ready
# 预期输出：STATUS 显示 Ready
for i in {1..30}; do
    if kubectl get nodes 2>/dev/null | grep -q "Ready"; then
        echo "节点已 Ready"
        break
    fi
    sleep 2
done

# 验证 ClusterIP 可访问
# 预期输出：显示 K8s 版本信息
curl -sk https://10.96.0.1:443/version

# 安装 CNI 插件
# 为什么：kubelet 需要这些插件来配置容器网络
yum install -y containernetworking-plugins

# 复制到 kubelet 期望的路径
# 为什么：kubelet 在 /opt/cni/bin/ 查找 CNI 插件
mkdir -p /opt/cni/bin
cp /usr/libexec/cni/* /opt/cni/bin/

# 验证安装
# 预期输出：显示 flannel, bridge 等插件
ls -la /opt/cni/bin/

# :warning: 创建 CNI 配置文件（必须在 Flannel 之前！）
# 为什么：Flannel 的 init 容器需要 CNI 配置已存在才能正常工作
# 如果跳过这一步，Flannel 会卡在 Init 状态
mkdir -p /etc/cni/net.d
cat <<EOF > /etc/cni/net.d/10-flannel.conflist
{
  "name": "cbr0",
  "cniVersion": "0.3.1",
  "plugins": [
    {
      "type": "flannel",
      "delegate": {
        "hairpinMode": true,
        "isDefaultGateway": true
      }
    },
    {
      "type": "portmap",
      "capabilities": {
        "portMappings": true
      }
    }
  ]
}
EOF

# 安装 Flannel（内联 manifest，无需联网下载）
# 为什么：使用镜像站的 Flannel 镜像，兼容离线环境
cat <<'EOF' | kubectl apply -f -
apiVersion: v1
kind: Namespace
metadata:
  name: kube-flannel
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: flannel
  namespace: kube-flannel
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: flannel
rules:
- apiGroups: [""]
  resources: ["pods"]
  verbs: ["get"]
- apiGroups: [""]
  resources: ["nodes"]
  verbs: ["list"]
- apiGroups: [""]
  resources: ["nodes/status"]
  verbs: ["patch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: flannel
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: flannel
subjects:
- kind: ServiceAccount
  name: flannel
  namespace: kube-flannel
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: kube-flannel-cfg
  namespace: kube-flannel
  labels:
    tier: node
    app: flannel
data:
  cni-conf.json: |
    {
      "name": "cbr0",
      "cniVersion": "0.3.1",
      "plugins": [
        {
          "type": "flannel",
          "delegate": {
            "hairpinMode": true,
            "isDefaultGateway": true
          }
        },
        {
          "type": "portmap",
          "capabilities": {
            "portMappings": true
          }
        }
      ]
    }
  net-conf.json: |
    {
      "Network": "10.244.0.0/16",
      "Backend": {
        "Type": "vxlan"
      }
    }
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: kube-flannel-ds
  namespace: kube-flannel
  labels:
    tier: node
    app: flannel
spec:
  selector:
    matchLabels:
      app: flannel
  template:
    metadata:
      labels:
        tier: node
        app: flannel
    spec:
      serviceAccountName: flannel
      initContainers:
      - name: install-cni-plugin
        image: docker.m.daocloud.io/flannel/flannel-cni-plugin:v1.4.0-flannel1
        command: ["cp"]
        args: ["-f", "/flannel", "/opt/cni/bin/flannel"]
        volumeMounts:
        - name: cni-plugin
          mountPath: /opt/cni/bin
      - name: install-cni
        image: docker.m.daocloud.io/flannel/flannel:v0.24.2
        command: ["cp"]
        args: ["-f", "/etc/kube-flannel/cni-conf.json", "/etc/cni/net.d/10-flannel.conflist"]
        volumeMounts:
        - name: cni
          mountPath: /etc/cni/net.d
        - name: flannel-cfg
          mountPath: /etc/kube-flannel
      containers:
      - name: kube-flannel
        image: docker.m.daocloud.io/flannel/flannel:v0.24.2
        command: ["/opt/bin/flanneld"]
        args: ["--ip-masq", "--kube-subnet-mgr"]
        env:
        - name: POD_NAME
          valueFrom:
            fieldRef:
              fieldPath: metadata.name
        - name: POD_NAMESPACE
          valueFrom:
            fieldRef:
              fieldPath: metadata.namespace
        resources:
          requests:
            cpu: 100m
            memory: 50Mi
        securityContext:
          capabilities:
            add: ["NET_ADMIN", "NET_RAW"]
        volumeMounts:
        - name: run
          mountPath: /run/flannel
        - name: flannel-cfg
          mountPath: /etc/kube-flannel
        - name: cni
          mountPath: /etc/cni/net.d
      tolerations:
      - operator: Exists
      volumes:
      - name: run
        hostPath:
          path: /run/flannel
      - name: cni
        hostPath:
          path: /etc/cni/net.d
      - name: flannel-cfg
        configMap:
          name: kube-flannel-cfg
      - name: cni-plugin
        hostPath:
          path: /opt/cni/bin
EOF

# 创建 CoreDNS RBAC 权限
# 为什么：CoreDNS 需要读取 Service、Endpoint 等资源来提供 DNS 解析
# 如果跳过这一步，CoreDNS 会报 "forbidden" 错误
kubectl create clusterrole coredns --verb=list,watch --resource=services,endpointslices,endpoints,namespaces 2>/dev/null || true
kubectl create clusterrolebinding coredns --clusterrole=coredns --serviceaccount=kube-system:default 2>/dev/null || true

# 获取节点 DNS 服务器（避免 DNS 循环）
# 为什么：必须使用节点的 DNS 服务器，否则 CoreDNS 会形成 DNS 循环
NODE_DNS=$(grep nameserver /etc/resolv.conf | head -1 | awk '{print $2}')
echo "使用节点 DNS 服务器：${NODE_DNS}"

# 部署 CoreDNS
# 为什么：K8s 需要 DNS 服务来实现服务发现
cat <<EOF | kubectl apply -f -
apiVersion: v1
kind: ConfigMap
metadata:
  name: coredns
  namespace: kube-system
data:
  Corefile: |
    .:53 {
        errors
        health {
            lameduck 5s
        }
        ready
        kubernetes cluster.local in-addr.arpa ip6.arpa {
            pods insecure
            fallthrough in-addr.arpa ip6.arpa
            ttl 30
        }
        prometheus :9153
        forward . ${NODE_DNS} {
            max_concurrent 1000
        }
        cache 30
        loop
        reload
        loadbalance
    }
---
apiVersion: v1
kind: Service
metadata:
  name: kube-dns
  namespace: kube-system
  labels:
    k8s-app: kube-dns
spec:
  selector:
    k8s-app: kube-dns
  ports:
  - name: dns
    port: 53
    protocol: UDP
  - name: dns-tcp
    port: 53
    protocol: TCP
  clusterIP: 10.96.0.10
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: coredns
  namespace: kube-system
  labels:
    k8s-app: kube-dns
spec:
  replicas: 1
  selector:
    matchLabels:
      k8s-app: kube-dns
  template:
    metadata:
      labels:
        k8s-app: kube-dns
    spec:
      priorityClassName: system-cluster-critical
      containers:
      - name: coredns
        image: registry.aliyuncs.com/google_containers/coredns:v1.10.1
        args:
        - -conf
        - /etc/coredns/Corefile
        ports:
        - containerPort: 53
          name: dns
          protocol: UDP
        - containerPort: 53
          name: dns-tcp
          protocol: TCP
        resources:
          limits:
            memory: 170Mi
          requests:
            cpu: 100m
            memory: 70Mi
        volumeMounts:
        - name: config-volume
          mountPath: /etc/coredns
          readOnly: true
      volumes:
      - name: config-volume
        configMap:
          name: coredns
          items:
          - key: Corefile
            path: Corefile
EOF

# 查看所有节点
# 预期输出：STATUS 为 Ready
kubectl get nodes -o wide

NAME   STATUS   ROLES    AGE     VERSION    INTERNAL-IP       EXTERNAL-IP   OS-IMAGE             KERNEL-VERSION      CONTAINER-RUNTIME
test   Ready    <none>   XXm     v1.28.15   192.168.174.128   <none>        CentOS Linux 7       3.10.0-1160.el7     containerd://1.6.33

# 查看所有 Pod
# 预期输出：所有 Pod 都是 Running 状态
kubectl get pods -A

NAMESPACE      NAME                       READY   STATUS    RESTARTS   AGE
kube-flannel   kube-flannel-ds-xxxxx      1/1     Running   0          XXm
kube-system    coredns-xxxxx              1/1     Running   0          XXm

# 创建测试部署
# 为什么：验证集群可以正常调度和运行 Pod
kubectl create deployment nginx-test --image=nginx:latest --replicas=2

# 等待 Pod 就绪
kubectl wait --for=condition=ready pod -l app=nginx-test --timeout=120s

# 检查部署状态
kubectl get deployment nginx-test

# 检查 Pod 状态
kubectl get pods -o wide

NAME         READY   UP-TO-DATE   AVAILABLE   AGE
nginx-test   2/2     2            2           XXs

# 创建 Service
# 为什么：验证 Service 可以正常工作
kubectl expose deployment nginx-test --port=80 --type=NodePort

# 获取 Service 信息
kubectl get svc nginx-test

# 测试访问
curl -s http://10.x.x.x:80 | head -5

kubectl delete deployment nginx-test
kubectl delete svc nginx-test

etcd → kube-apiserver → kube-controller-manager → kube-scheduler → kubelet → kube-proxy → flannel → CoreDNS

# 查看节点
kubectl get nodes -o wide

# 查看 Pod
kubectl get pods -A

# 查看日志
kubectl logs -f <pod-name> -n <namespace>

# 进入容器
kubectl exec -it <pod-name> -- /bin/bash

# 查看事件
kubectl get events --sort-by='.lastTimestamp'

# 查看集群信息
kubectl cluster-info

# 创建 Worker 节点部署包目录
mkdir -p /root/k8s-worker-pack/bin
mkdir -p /root/k8s-worker-pack/configs
mkdir -p /root/k8s-worker-pack/packages
mkdir -p /root/k8s-worker-pack/scripts

# 复制 Worker 节点需要的二进制文件
cp /usr/local/bin/kubelet /root/k8s-worker-pack/bin/
cp /usr/local/bin/kube-proxy /root/k8s-worker-pack/bin/
cp /usr/local/bin/kubectl /root/k8s-worker-pack/bin/

# 复制证书文件
cp /etc/kubernetes/pki/ca.crt /root/k8s-worker-pack/configs/
cp /etc/kubernetes/pki/admin.crt /root/k8s-worker-pack/configs/
cp /etc/kubernetes/pki/admin.key /root/k8s-worker-pack/configs/

# 复制 kubeconfig 文件
cp /etc/kubernetes/admin.conf /root/k8s-worker-pack/configs/
cp /etc/kubernetes/bootstrap-kubelet.conf /root/k8s-worker-pack/configs/

# 复制 kubelet 配置
cp /var/lib/kubelet/config.yaml /root/k8s-worker-pack/configs/

# 复制 RPM 包（Docker、containerd、CNI 插件）
cp /root/work/k8s-offline-pack/packages/*.rpm /root/k8s-worker-pack/packages/

echo "Worker 节点部署包已生成：/root/k8s-worker-pack/"
ls -la /root/k8s-worker-pack/

# 在 Master 节点执行，将部署包传输到 Worker 节点
# 将 WORKER_IP 替换为 Worker 节点的实际 IP 地址
scp -r /root/k8s-worker-pack root@WORKER_IP:/root/

# 进入部署包目录
cd /root/k8s-worker-pack

# 禁用 Swap
swapoff -a
sed -i '/ swap / s/^\(.*\)$/#\1/g' /etc/fstab

# 加载内核模块
modprobe overlay
modprobe br_netfilter
cat <<EOF > /etc/modules-load.d/k8s.conf
overlay
br_netfilter
EOF

# 配置系统参数
cat <<EOF > /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-iptables  = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.ipv4.ip_forward                 = 1
EOF
sysctl --system

# 创建 resolv.conf
mkdir -p /run/systemd/resolve
cp /etc/resolv.conf /run/systemd/resolve/resolv.conf

# 从离线包安装
rpm -ivh --force packages/*.rpm

# 配置 Docker
mkdir -p /etc/docker
cat <<EOF > /etc/docker/daemon.json
{
  "exec-opts": ["native.cgroupdriver=systemd"],
  "log-driver": "json-file",
  "log-opts": {
    "max-size": "100m"
  },
  "storage-driver": "overlay2"
}
EOF

# 启动 Docker 和 containerd
systemctl daemon-reload
systemctl enable docker containerd
systemctl start docker containerd

# 安装二进制文件
cp bin/* /usr/local/bin/
chmod +x /usr/local/bin/kubelet /usr/local/bin/kube-proxy /usr/local/bin/kubectl

# 安装 CNI 插件
rpm -ivh --force packages/containernetworking-plugins*.rpm 2>/dev/null || true
mkdir -p /opt/cni/bin
cp /usr/libexec/cni/* /opt/cni/bin/ 2>/dev/null || true

# 创建证书目录
mkdir -p /etc/kubernetes/pki
cp configs/ca.crt /etc/kubernetes/pki/
cp configs/admin.crt /etc/kubernetes/pki/
cp configs/admin.key /etc/kubernetes/pki/

# 复制 kubeconfig 文件
cp configs/admin.conf /etc/kubernetes/
cp configs/bootstrap-kubelet.conf /etc/kubernetes/

# 创建 kubelet 配置
mkdir -p /var/lib/kubelet
cp configs/config.yaml /var/lib/kubelet/

# :warning: 修改 kubelet 配置中的 node-ip
# 将 WORKER_IP 替换为 Worker 节点的实际 IP
# 找到当前的 node-ip 配置并修改

# kubelet 服务
cat <<'EOF' > /etc/systemd/system/kubelet.service
[Unit]
Description=kubelet
After=network-online.target
Wants=network-online.target

[Service]
ExecStart=/usr/local/bin/kubelet \
  --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \
  --kubeconfig=/var/lib/kubelet/kubeconfig \
  --config=/var/lib/kubelet/config.yaml \
  --pod-infra-container-image=registry.aliyuncs.com/google_containers/pause:3.8 \
  --node-ip=WORKER_IP \
  --client-ca-file=/etc/kubernetes/pki/ca.crt
Restart=always
RestartSec=10

[Install]
WantedBy=multi-user.target
EOF

# 将 WORKER_IP 替换为 Worker 节点的实际 IP
sed -i "s/WORKER_IP/$(hostname -I | awk '{print $1}')/g" /etc/systemd/system/kubelet.service

# kube-proxy 服务
cat <<'EOF' > /etc/systemd/system/kube-proxy.service
[Unit]
Description=Kubernetes Kube-Proxy
After=network-online.target kube-apiserver.service
Wants=network-online.target

[Service]
ExecStart=/usr/local/bin/kube-proxy \
  --kubeconfig=/etc/kubernetes/admin.conf \
  --cluster-cidr=10.244.0.0/16 \
  --proxy-mode=iptables
Restart=always
RestartSec=10

[Install]
WantedBy=multi-user.target
EOF

# 重新加载 systemd
systemctl daemon-reload

# 启动 kubelet
systemctl enable kubelet
systemctl start kubelet

# 等待 kubelet 启动并发送 CSR 请求
sleep 15

# 查看 CSR 请求
kubectl get csr

# 查看待批准的 CSR
kubectl get csr

# 批准所有待处理的 CSR
for csr in $(kubectl get csr --no-headers | grep Pending | awk '{print $1}'); do
    kubectl certificate approve $csr
    echo "已批准: $csr"
done

# 回到 Worker 节点启动 kube-proxy
# （在 Worker 节点执行）
systemctl enable kube-proxy
systemctl start kube-proxy

# 等待节点 Ready
for i in {1..30}; do
    if kubectl get nodes 2>/dev/null | grep -q "Ready"; then
        echo "节点已 Ready"
        break
    fi
    sleep 2
done

# 在 Master 节点执行
kubectl get nodes -o wide

# 预期输出：
# NAME     STATUS   ROLES    AGE     VERSION    INTERNAL-IP       ...
# master   Ready    <none>   XXm     v1.28.15   192.168.174.128   ...
# worker1  Ready    <none>   XXs     v1.28.15   192.168.174.129   ...

# 测试跨节点 Pod 调度
kubectl run test-cross-node --image=nginx:latest --replicas=2
kubectl get pods -o wide  # 检查 Pod 是否分布在不同节点

# 1. 传输部署包到目标机器
scp k8s-offline-pack-v1.28.15.tar.gz root@目标IP:/root/

# 2. 解压并执行
tar xzf k8s-offline-pack-v1.28.15.tar.gz
cd k8s-offline-pack

# 3. 一键部署
./scripts/install.sh

# 或指定 IP 部署
NODE_IP=192.168.1.100 ./scripts/install.sh

containerd → etcd → kube-apiserver → kube-controller-manager → kube-scheduler → kubelet → kube-proxy → flannel → CoreDNS