环境说明

银河麒麟v10 x86架构，cgroup v2启用

系统内核：5.4.x  源码编译安装

kubeadm 1.31.2 自编译二进制文件，证书有效期100年

containerd 版本：2.0.0

部署准备

 1. 参数模块调整和依赖安装

# 配置hostname

hostnamectl set-hostname k8s-master
hostnamectl set-hostname k8s-node1
hostnamectl set-hostname k8s-node2

# 配置hosts

192.168.10.101 k8s-node1
192.168.10.102 k8s-node2
192.168.10.100 k8s-master

# 配置时间同步chrony 略

# 关闭swap
swapoff -a

# 内核模块加载

cat << EOF > /etc/modules-load.d/containerd.conf
overlay
br_netfilter
EOF

modprobe overlay
modprobe br_netfilter

lsmod | grep br_netfilter

cat > /etc/modules-load.d/ipvs.conf << EOF
ip_vs
ip_vs_rr
ip_vs_wrr
ip_vs_sh
nf_conntrack
ip_tables
ip_set
xt_set
ipt_set
ipt_rpfilter
ipt_REJECT
ipip
EOF

# 配置内核参数/etc/sysctl.conf

net.ipv4.ip_forward=1
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
vm.swappiness = 0

# 安装依赖
yum install socat libnetfilter_queue libnetfilter_cttimeout conntrack-tools libnetfilter_cthelper

2. 安装containerd

# 安装runc
wget https://github.com/opencontainers/runc/releases/download/v1.2.2/runc.amd64

install -m 755 runc.amd64 /usr/sbin/runc

# 安装cni插件
wget https://github.com/containernetworking/plugins/releases/download/v1.6.0/cni-plugins-linux-amd64-v1.6.0.tgz

mkdir -p /opt/cni/bin

tar Cxzvf /opt/cni/bin cni-plugins-linux-amd64-v1.6.0.tgz

# 下载二进制包

wget https://github.com/containerd/containerd/releases/download/v2.0.0/containerd-2.0.0-
linux-amd64.tar.gz

或者

wget https://github.com/containerd/containerd/releases/download/v2.0.0/containerd-static-2.0.0-linux-amd64.tar.gz

两者区别：

containerd-<VERSION>-<OS>-<ARCH>.tar.gz: ✅Recommended. Dynamically linked with glibc 2.31 (Ubuntu 20.04).
containerd-static-<VERSION>-<OS>-<ARCH>.tar.gz: Statically linked. Expected to be used on non-glibc Linux distributions. Not position-independent.

# 解压

tar Cxzvf /usr/bin/ containerd-2.0.0-linux-amd64.tar.gz

# 编写service文件
cat > /usr/lib/systemd/system/containerd.service << 'EOF'
# Copyright The containerd Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

[Unit]
Description=containerd container runtime
Documentation=https://containerd.io
After=network.target local-fs.target dbus.service

[Service]
ExecStartPre=-/sbin/modprobe overlay
ExecStart=/usr/bin/containerd

Type=notify
Delegate=yes
KillMode=process
Restart=always
RestartSec=5

# Having non-zero Limit*s causes performance problems due to accounting overhead
# in the kernel. We recommend using cgroups to do container-local accounting.
LimitNPROC=infinity
LimitCORE=infinity

# Comment TasksMax if your systemd version does not supports it.
# Only systemd 226 and above support this version.
TasksMax=infinity
OOMScoreAdjust=-999

[Install]
WantedBy=multi-user.target
EOF

systemctl daemon-reload && systemctl enable containerd

# 生成containerd的配置文件
mkdir /etc/containerd
containerd config default > /etc/containerd/config.toml

# 编辑containerd的配置文件/etc/containerd/config.toml，主要修改container的数据目录，并启用systemd的cgroup

# 修改数据存储目录
root = "/data/lib/containerd"

# 对于使用systemd作为init system的linux发行版，官方建议用systemd作为容器cgroup driver
# false改成true
SystemdCgroup = true

[plugins."io.containerd.grpc.v1.cri".containerd]
  cgroup = "cgroupv2"

[plugins."io.containerd.grpc.v1.cri"]
  [plugins."io.containerd.grpc.v1.cri".containerd]
    # 设置 cgroup 驱动为 systemd
    cgroup_driver = "systemd"


systemctl daemon-reload && systemctl start containerd && systemctl status containerd

# 安装nerdctl (可选)

wget https://github.com/containerd/nerdctl/releases/download/v2.0.0/nerdctl-2.0.0-linux-amd64.tar.gz

tar Czxvf /usr/bin nerdctl-2.0.0-linux-amd64.tar.gz

Enabling cgroup v2 is highly recommended for rootless mode, see https://rootlesscontaine.rs/getting-started/common/cgroup2/ 

安装kubeadm

将编译的二进制文件kubeadm kubectl   kubelet拷贝到目录/usr/bin下

PS：官方二进制文件下载地址：https://github.com/kubernetes/kubernetes/releases

cat > /usr/lib/systemd/system/kubelet.service << EOF
[Unit]
Description=kubelet: The Kubernetes Node Agent
Documentation=https://kubernetes.io/docs/
Wants=network-online.target
After=network-online.target

[Service]
ExecStart=/usr/bin/kubelet
Restart=always
StartLimitInterval=0
RestartSec=10

[Install]
WantedBy=multi-user.target
EOF

systemctl daemon-reload && systemctl enable kubelet

mkdir -p  /usr/lib/systemd/system/kubelet.service.d/

cat > /usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf << 'EOF'
[Service]
Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf"
Environment="KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml"
# This is a file that "kubeadm init" and "kubeadm join" generates at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically
EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env
# This is a file that the user can use for overrides of the kubelet args as a last resort. Preferably, the user should use
# the .NodeRegistration.KubeletExtraArgs object in the configuration files instead. KUBELET_EXTRA_ARGS should be sourced from this file.
EnvironmentFile=-/etc/sysconfig/kubelet
ExecStart=
ExecStart=/usr/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS
EOF

systemctl start kubelet

# 初始化集群

# 拉取所需镜像
 
kubeadm config images pull --image-repository registry.aliyuncs.com/google_containers
 
# 初始化
 
kubeadm init --kubernetes-version=v1.31.2 \
--apiserver-advertise-address=192.168.100.100 \
--pod-network-cidr=10.244.0.0/16 \
--service-cidr=10.96.0.0/12 \
--token-ttl=0 \
--image-repository registry.aliyuncs.com/google_containers

# 接下来在worker节点上执行相关的操作，worker节点与master节点的操作步骤的唯一区别是：master节点执行kubeadm init操作，woker节点执行kubeadm join操作，因此上面的步骤除了kubeadm init步骤之外，其他所有的步骤woker节点同样也需要执行。
 
# 执行kubeadm init 成功之后输出的 最后一行kubeadm join 命令
 
kubeadm config images pull --cri-socket unix:///var/run/cri-dockerd.sock --image-repository registry.aliyuncs.com/google_containers
 
kubeadm join 192.168.100.100:6443 --token o4zf8w.xxxx --discovery-token-ca-cert-hash sha256:376e215a51620ac6ccc

部署flannel

# 部署flannel插件
 
cat > flannel.yaml << EOF
#---
#kind: Namespace
#apiVersion: v1
#metadata:
#  name: kube-flannel
#  labels:
#    k8s-app: flannel
#    pod-security.kubernetes.io/enforce: privileged
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  labels:
    k8s-app: flannel
  name: flannel
rules:
- apiGroups:
  - ""
  resources:
  - pods
  verbs:
  - get
- apiGroups:
  - ""
  resources:
  - nodes
  verbs:
  - get
  - list
  - watch
- apiGroups:
  - ""
  resources:
  - nodes/status
  verbs:
  - patch
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  labels:
    k8s-app: flannel
  name: flannel
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: flannel
subjects:
- kind: ServiceAccount
  name: flannel
  namespace: kube-system
---
apiVersion: v1
kind: ServiceAccount
metadata:
  labels:
    k8s-app: flannel
  name: flannel
  namespace: kube-system
---
kind: ConfigMap
apiVersion: v1
metadata:
  name: kube-flannel-cfg
  namespace: kube-system
  labels:
    tier: node
    k8s-app: flannel
    app: flannel
data:
  cni-conf.json: |
    {
      "name": "cbr0",
      "cniVersion": "0.3.1",
      "plugins": [
        {
          "type": "flannel",
          "delegate": {
            "hairpinMode": true,
            "isDefaultGateway": true
          }
        },
        {
          "type": "portmap",
          "capabilities": {
            "portMappings": true
          }
        }
      ]
    }
  net-conf.json: |
    {
      "Network": "10.244.0.0/16",
      "EnableNFTables": false,
      "Backend": {
        "Type": "vxlan"
      }
    }
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: kube-flannel-ds
  namespace: kube-system
  labels:
    tier: node
    app: flannel
    k8s-app: flannel
spec:
  selector:
    matchLabels:
      app: flannel
  template:
    metadata:
      labels:
        tier: node
        app: flannel
    spec:
      affinity:
        nodeAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:
            - matchExpressions:
              - key: kubernetes.io/os
                operator: In
                values:
                - linux
      hostNetwork: true
      priorityClassName: system-node-critical
      tolerations:
      - operator: Exists
        effect: NoSchedule
      serviceAccountName: flannel
      initContainers:
      - name: install-cni-plugin
        image: docker.io/flannel/flannel-cni-plugin:v1.6.0-flannel1
        command:
        - cp
        args:
        - -f
        - /flannel
        - /opt/cni/bin/flannel
        volumeMounts:
        - name: cni-plugin
          mountPath: /opt/cni/bin
      - name: install-cni
        image: docker.io/flannel/flannel:v0.26.1
        command:
        - cp
        args:
        - -f
        - /etc/kube-flannel/cni-conf.json
        - /etc/cni/net.d/10-flannel.conflist
        volumeMounts:
        - name: cni
          mountPath: /etc/cni/net.d
        - name: flannel-cfg
          mountPath: /etc/kube-flannel/
      containers:
      - name: kube-flannel
        image: docker.io/flannel/flannel:v0.26.1
        command:
        - /opt/bin/flanneld
        args:
        - --ip-masq
        - --kube-subnet-mgr
        resources:
          requests:
            cpu: "100m"
            memory: "50Mi"
        securityContext:
          privileged: false
          capabilities:
            add: ["NET_ADMIN", "NET_RAW"]
        env:
        - name: POD_NAME
          valueFrom:
            fieldRef:
              fieldPath: metadata.name
        - name: POD_NAMESPACE
          valueFrom:
            fieldRef:
              fieldPath: metadata.namespace
        - name: EVENT_QUEUE_DEPTH
          value: "5000"
        volumeMounts:
        - name: run
          mountPath: /run/flannel
        - name: flannel-cfg
          mountPath: /etc/kube-flannel/
        - name: xtables-lock
          mountPath: /run/xtables.lock
      volumes:
      - name: run
        hostPath:
          path: /run/flannel
      - name: cni-plugin
        hostPath:
          path: /opt/cni/bin
      - name: cni
        hostPath:
          path: /etc/cni/net.d
      - name: flannel-cfg
        configMap:
          name: kube-flannel-cfg
      - name: xtables-lock
        hostPath:
          path: /run/xtables.lock
          type: FileOrCreate
EOF
 
kubectl apply -f flannel.yaml

测试验证

# 检查证书过期时间
kubeadm certs check-expiration

# 检查node pod
kubectl get node  

kubectl get pod -A

# 创建一个pod。nginx需要提前docker pull
kubectl create deployment nginx --image=nginx

# 暴露端口
kubectl expose deployment nginx --port=80 --type=NodePort

# 查看状态
kubectl get pods,svc

安装 kubeadm | Kubernetes