由于centos停更，yum、docker等不支持，采用阿里云仓库搭建K8S

一：准备

服务器信息	主机名	IP地址
Centos7.9	node1-master	192.168.35.130
Centos7.9	node2	192.168.35.131

# 查看系统版本
cat /etc/centos-release
# 查看内核版本
uname -sr

二：服务器前置操作
每个节点都需要操作

#使用 hostnamectl set-hostname设置主机名
hostnamectl set-hostname node01-master

#编辑hosts文件，配置域名映射
vi /etc/hosts 
192.168.35.130  node01-master
192.168.35.131  node02

# 关闭防火墙
systemctl stop firewalld
# 关闭开机自启
systemctl disable firewalld
# 查看防火墙状态 显示not running 表示关闭成功
firewall-cmd --state

#将 SELinux 设置为 permissive 模式（相当于将其禁用）
sudo setenforce 0
sudo sed -i 's/^SELINUX=enforcing$/SELINUX=permissive/' /etc/selinux/config

#关闭swap
swapoff -a  # 临时关闭，主机重启后k8s无法自动重启，需要重新关闭swap
vim /etc/fstab  # 永久关闭
注释行 #/dev/mapper/cl_fedora_swap

#允许 iptables 检查桥接流量
cat <<EOF | sudo tee /etc/modules-load.d/k8s.conf
br_netfilter
EOF

cat <<EOF | sudo tee /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
EOF

sudo sysctl --system

由于centos停更，yum无法下载docker，需要更换yum仓库为阿里云仓库

cd /etc/yum.repos.d
vi CentOS-Base.repo
替换全部内容更为下面的
[base]
name=CentOS-$releasever - Base - mirrors.aliyun.com
baseurl=http://mirrors.aliyun.com/centos/$releasever/os/$basearch/
gpgcheck=1
gpgkey=http://mirrors.aliyun.com/centos/RPM-GPG-KEY-CentOS-7
 
[updates]
name=CentOS-$releasever - Updates - mirrors.aliyun.com
baseurl=http://mirrors.aliyun.com/centos/$releasever/updates/$basearch/
gpgcheck=1
gpgkey=http://mirrors.aliyun.com/centos/RPM-GPG-KEY-CentOS-7
 
[extras]
name=CentOS-$releasever - Extras - mirrors.aliyun.com
baseurl=http://mirrors.aliyun.com/centos/$releasever/extras/$basearch/
gpgcheck=1
gpgkey=http://mirrors.aliyun.com/centos/RPM-GPG-KEY-CentOS-7
 
[centosplus]
name=CentOS-$releasever - Plus - mirrors.aliyun.com
baseurl=http://mirrors.aliyun.com/centos/$releasever/centosplus/$basearch/
gpgcheck=1
enabled=0
gpgkey=http://mirrors.aliyun.com/centos/RPM-GPG-KEY-CentOS-7

保存完成执行
yum clean all
yum makecache
yum update

#下载docker 根据仓库情况建议选择docker版本 
# yum -y install docker-ce 执行不成功选择yum -y install docker
yum -y install docker-ce
systemctl enable docker && systemctl start docker
# 配置阿里云镜像加速 另外k8s 1.2版本以上需要修改native.cgroupdriver=systemd
sudo mkdir -p /etc/docker
sudo tee /etc/docker/daemon.json <<-'EOF'
{
  "registry-mirrors": ["https://iedolof4.mirror.aliyuncs.com"],
  "exec-opts": ["native.cgroupdriver=systemd"]
}
EOF
sudo systemctl daemon-reload
sudo systemctl restart docker
#验证
docker version

三：开始搭建
每个节点都需要操作安装kubeadm、kubelet、kubectl

#添加阿里的yum软件源，这里需要注意baseurl的地址是否是你虚拟机对应的版本  
# centos7.9使用kubernetes-el7-x86_64 可通过uname -sr命令查看
#访问地址 https://mirrors.aliyun.com/kubernetes/yum/repos/ 看对应版本
cat > /etc/yum.repos.d/kubernetes.repo << EOF
[kubernetes]
name=Kubernetes
baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64
enabled=1
gpgcheck=0
repo_gpgcheck=0
gpgkey=https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg https://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
EOF

#安装 kubelet kubeadm  kubectl 注意版本号
yum install -y kubelet-1.21.0 kubeadm-1.21.0 kubectl-1.21.0
systemctl enable kubelet

只需要在node1-master执行

#执行kubeadm 初始化 注意更改apiserver-advertise-address 的 ip地址为node1-master地址
kubeadm init \
      --apiserver-advertise-address=192.168.35.130 \
      --image-repository registry.aliyuncs.com/google_containers \
      --kubernetes-version v1.21.0 \
      --service-cidr=10.96.0.0/12 \
      --pod-network-cidr=10.244.0.0/16 \
      --ignore-preflight-errors=all      
#-–apiserver-advertise-address 集群通告地址（master内网） 注意修改为master节点的address
#–-image-repository 由于默认拉取镜像地址k8s.gcr.io国内无法访问，这里指定阿里云镜像仓库地址
#–-kubernetes-version K8s版本，与上面安装的一致
#–-service-cidr 集群内部虚拟网络，Pod统一访问入口
#-–pod-network-cidr Pod网络，与下面部署的CNI网络组件yaml中保持一致```

1：执行如果发现报错，docker版本过高，可以根据推荐版本降级，不降级也没多大影响
2：执行如果发现docker pull 拉取不下来镜像 原因是centos停更，docker不支持
需要通过三方进行手动拉取命令行执行 根据缺失的docker镜像执行
假如拉取原始镜像命令如下：docker pull whyour/qinglong:latest
改为 docker pull dockerpull.org/whyour/qinglong:latest
拉去完成后 docker tag 改为的镜像名（dockerpull.org/whyour/qinglong:latest）  需要的镜像名
例如docker tag dockerpull.org/whyour/qinglong:latest whyour/qinglong:latest
执行完成后删除dockerpull.org/whyour/qinglong:latest

#初始化完成后：注意最后一行 保存下
#kubeadm join 192.168.35.130:6443 --token ms6617.5hbjusrd37ovnbh1 \ --discovery-token-ca-cert-hash sha256:1161e6266f482d5fdf436aa115a941657a2aee5ff0cc739d8c58adfc6a8441fb

mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config
export KUBECONFIG=/etc/kubernetes/admin.conf

只需要在node2等子节点执行

#node1-master初始化完成后的最后一行执行
kubeadm join 192.168.35.130:6443 --token ms6617.5hbjusrd37ovnbh1 \ --discovery-token-ca-cert-hash sha256:1161e6266f482d5fdf436aa115a941657a2aee5ff0cc739d8c58adfc6a8441fb

#token有效期为24小时，如果过期，需要重新创建token
kubeadm token create --print-join-command

#验证  能看到所有服务器
kubectl get nodes
kubectl get nodes -owide

只需要在node1-master执行

#本地下载 https://docs.projectcalico.org/manifests/calico.yaml 
#如果404请执行https://raw.githubusercontent.com/projectcalico/calico/v3.25.0/manifests/calico.yaml
mkdir -p /z/k8s/calico
cd /z/k8s/calico
#文件calico.yaml 复制到/z/k8s/calico目录下
kubectl apply -f calico.yaml
# 查看运行状态
kubectl get pods -n kube-system
#状态非Runnning的都是没有创建成功的，关注在那个NODE下 查看具体的详细pod信息 ，
#那个NODE有问题去那个NODE执行或所有集群都执行一边
#可以使用kubectl describe pod -n kube-system podname 命令查看容器运行情况，如果是因为镜像下载失败，可以对应的集群节点使用docker pull命令将镜像下载下来
kubectl describe pod -n NAMESPACE名称 name名称
#执行如果发现docker pull 拉取不下来镜像 原因是centos停更，docker不支持
#需要通过三方进行手动拉取命令行执行 根据缺失的docker镜像执行
#假如拉取原始镜像命令如下：docker pull whyour/qinglong:latest
#改为 docker pull dockerpull.org/whyour/qinglong:latest
#拉去完成后 docker tag 改为的镜像名（dockerpull.org/whyour/qinglong:latest）  需要的镜像名
#例如docker tag dockerpull.org/whyour/qinglong:latest whyour/qinglong:latest
#执行完成后删除dockerpull.org/whyour/qinglong:latest
#等待几秒钟等待pod自动刷新，然后再次执行是否状态变为Runnning
kubectl get pods -n kube-system

四：（可选）搭建可视化页面----Dashboard

cd /z/k8s
mkdir dashboard
cd /z/k8s/dashboard
vi recommended.yaml
#编写内容
# Copyright 2017 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

apiVersion: v1
kind: Namespace
metadata:
  name: kubernetes-dashboard

---

apiVersion: v1
kind: ServiceAccount
metadata:
  labels:
    k8s-app: kubernetes-dashboard
  name: kubernetes-dashboard
  namespace: kubernetes-dashboard

---

kind: Service
apiVersion: v1
metadata:
  labels:
    k8s-app: kubernetes-dashboard
  name: kubernetes-dashboard
  namespace: kubernetes-dashboard
spec:
  ports:
    - port: 443
      targetPort: 8443
  selector:
    k8s-app: kubernetes-dashboard

---

apiVersion: v1
kind: Secret
metadata:
  labels:
    k8s-app: kubernetes-dashboard
  name: kubernetes-dashboard-certs
  namespace: kubernetes-dashboard
type: Opaque

---

apiVersion: v1
kind: Secret
metadata:
  labels:
    k8s-app: kubernetes-dashboard
  name: kubernetes-dashboard-csrf
  namespace: kubernetes-dashboard
type: Opaque
data:
  csrf: ""

---

apiVersion: v1
kind: Secret
metadata:
  labels:
    k8s-app: kubernetes-dashboard
  name: kubernetes-dashboard-key-holder
  namespace: kubernetes-dashboard
type: Opaque

---

kind: ConfigMap
apiVersion: v1
metadata:
  labels:
    k8s-app: kubernetes-dashboard
  name: kubernetes-dashboard-settings
  namespace: kubernetes-dashboard

---

kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  labels:
    k8s-app: kubernetes-dashboard
  name: kubernetes-dashboard
  namespace: kubernetes-dashboard
rules:
  # Allow Dashboard to get, update and delete Dashboard exclusive secrets.
  - apiGroups: [""]
    resources: ["secrets"]
    resourceNames: ["kubernetes-dashboard-key-holder", "kubernetes-dashboard-certs", "kubernetes-dashboard-csrf"]
    verbs: ["get", "update", "delete"]
    # Allow Dashboard to get and update 'kubernetes-dashboard-settings' config map.
  - apiGroups: [""]
    resources: ["configmaps"]
    resourceNames: ["kubernetes-dashboard-settings"]
    verbs: ["get", "update"]
    # Allow Dashboard to get metrics.
  - apiGroups: [""]
    resources: ["services"]
    resourceNames: ["heapster", "dashboard-metrics-scraper"]
    verbs: ["proxy"]
  - apiGroups: [""]
    resources: ["services/proxy"]
    resourceNames: ["heapster", "http:heapster:", "https:heapster:", "dashboard-metrics-scraper", "http:dashboard-metrics-scraper"]
    verbs: ["get"]

---

kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  labels:
    k8s-app: kubernetes-dashboard
  name: kubernetes-dashboard
rules:
  # Allow Metrics Scraper to get metrics from the Metrics server
  - apiGroups: ["metrics.k8s.io"]
    resources: ["pods", "nodes"]
    verbs: ["get", "list", "watch"]

---

apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  labels:
    k8s-app: kubernetes-dashboard
  name: kubernetes-dashboard
  namespace: kubernetes-dashboard
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: kubernetes-dashboard
subjects:
  - kind: ServiceAccount
    name: kubernetes-dashboard
    namespace: kubernetes-dashboard

---

apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: kubernetes-dashboard
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: kubernetes-dashboard
subjects:
  - kind: ServiceAccount
    name: kubernetes-dashboard
    namespace: kubernetes-dashboard

---

kind: Deployment
apiVersion: apps/v1
metadata:
  labels:
    k8s-app: kubernetes-dashboard
  name: kubernetes-dashboard
  namespace: kubernetes-dashboard
spec:
  replicas: 1
  revisionHistoryLimit: 10
  selector:
    matchLabels:
      k8s-app: kubernetes-dashboard
  template:
    metadata:
      labels:
        k8s-app: kubernetes-dashboard
    spec:
      containers:
        - name: kubernetes-dashboard
          image: kubernetesui/dashboard:v2.3.1
          imagePullPolicy: IfNotPresent
          ports:
            - containerPort: 8443
              protocol: TCP
          args:
            - --auto-generate-certificates
            - --namespace=kubernetes-dashboard
            # Uncomment the following line to manually specify Kubernetes API server Host
            # If not specified, Dashboard will attempt to auto discover the API server and connect
            # to it. Uncomment only if the default does not work.
            # - --apiserver-host=http://my-address:port
          volumeMounts:
            - name: kubernetes-dashboard-certs
              mountPath: /certs
              # Create on-disk volume to store exec logs
            - mountPath: /tmp
              name: tmp-volume
          livenessProbe:
            httpGet:
              scheme: HTTPS
              path: /
              port: 8443
            initialDelaySeconds: 30
            timeoutSeconds: 30
          securityContext:
            allowPrivilegeEscalation: false
            readOnlyRootFilesystem: true
            runAsUser: 1001
            runAsGroup: 2001
      volumes:
        - name: kubernetes-dashboard-certs
          secret:
            secretName: kubernetes-dashboard-certs
        - name: tmp-volume
          emptyDir: {}
      serviceAccountName: kubernetes-dashboard
      nodeSelector:
        "kubernetes.io/os": linux
      # Comment the following tolerations if Dashboard must not be deployed on master
      tolerations:
        - key: node-role.kubernetes.io/master
          effect: NoSchedule

---

kind: Service
apiVersion: v1
metadata:
  labels:
    k8s-app: dashboard-metrics-scraper
  name: dashboard-metrics-scraper
  namespace: kubernetes-dashboard
spec:
  ports:
    - port: 8000
      targetPort: 8000
  selector:
    k8s-app: dashboard-metrics-scraper

---

kind: Deployment
apiVersion: apps/v1
metadata:
  labels:
    k8s-app: dashboard-metrics-scraper
  name: dashboard-metrics-scraper
  namespace: kubernetes-dashboard
spec:
  replicas: 1
  revisionHistoryLimit: 10
  selector:
    matchLabels:
      k8s-app: dashboard-metrics-scraper
  template:
    metadata:
      labels:
        k8s-app: dashboard-metrics-scraper
      annotations:
        seccomp.security.alpha.kubernetes.io/pod: 'runtime/default'
    spec:
      containers:
        - name: dashboard-metrics-scraper
          image: kubernetesui/metrics-scraper:v1.0.6
          imagePullPolicy: IfNotPresent
          ports:
            - containerPort: 8000
              protocol: TCP
          livenessProbe:
            httpGet:
              scheme: HTTP
              path: /
              port: 8000
            initialDelaySeconds: 30
            timeoutSeconds: 30
          volumeMounts:
          - mountPath: /tmp
            name: tmp-volume
          securityContext:
            allowPrivilegeEscalation: false
            readOnlyRootFilesystem: true
            runAsUser: 1001
            runAsGroup: 2001
      serviceAccountName: kubernetes-dashboard
      nodeSelector:
        "kubernetes.io/os": linux
      # Comment the following tolerations if Dashboard must not be deployed on master
      tolerations:
        - key: node-role.kubernetes.io/master
          effect: NoSchedule
      volumes:
        - name: tmp-volume
          emptyDir: {}

#方案一：
#由于docker问题需要手动拉取镜像
docker pull kubernetesui/metrics-scraper:v1.0.6
docker pull kubernetesui/dashboard:v2.3.1
#需要通过三方进行手动拉取命令行执行 根据缺失的docker镜像执行
#假如拉取原始镜像命令如下：docker pull whyour/qinglong:latest
#改为 docker pull dockerpull.org/whyour/qinglong:latest
#拉去完成后 docker tag 改为的镜像名（dockerpull.org/whyour/qinglong:latest）  需要的镜像名
#例如docker tag dockerpull.org/whyour/qinglong:latest whyour/qinglong:latest
#执行完成后删除dockerpull.org/whyour/qinglong:latest

#方案二（需要建立到方案一的基础上，如果方案一执行完成，还是启动无法加载镜像采用方案二继续执行）：
docker save kubernetesui/dashboard:v2.3.1 -o dashboard.tar
docker save kubernetesui/metrics-scraper:v1.0.6 -o metrics-scraper.tar
sudo ctr -n k8s.io images import dashboard.tar
sudo ctr -n k8s.io images import metrics-scraper.tar

#部署：
kubectl apply -f recommended.yaml
#更改外部访问
kubectl edit svc kubernetes-dashboard -n kubernetes-dashboard
#type: ClusterIP 改为 type: NodePort

kubectl get pod,svc -n kubernetes-dashboard
#pod如果非RUNNING都是没有成功，可以使用kubectl describe pod -n kube-system podname 命令查看容器运行情况
#可以看到443映射给新端口，通过master ip进行访问
#注意一定是https://ip:端口
#由于dashboard证书问题，需要临时使用火狐浏览器进行访问

#创建用户
cd /z/k8s/dashboard
mkdir user
cd /z/k8s/dashboard/user
#创建用户yaml
vi dashboard-root.yaml
#编写内容
apiVersion: v1
kind: ServiceAccount
metadata:
  name: dashboard-root
  namespace: kube-system
#创建用户角色yaml
vi dashboard-root-role.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: dashboard-admin-binding
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: cluster-admin
subjects:
- kind: ServiceAccount
  name: dashboard-root
  namespace: kube-system

#部署：
kubectl apply -f dashboard-root.yaml
kubectl apply -f dashboard-root-role.yaml
#获取token
kubectl -n kube-system get secret $(kubectl -n kube-system get sa dashboard-root -o jsonpath='{.secrets[0].name}') -o jsonpath='{.data.token}' | base64 --decode
#注意保存token
eyJhbGciOiJSUzI1NiIsImtpZCI6ImRvRXBBcS1hTGVOb1VjTF95YzliSUtleDlOOEpSZndDSTE5ajBXX2xXNlUifQ.eyJpc3MiOiJrdWJlcm5ldGVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJrdWJlLXN5c3RlbSIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VjcmV0Lm5hbWUiOiJkYXNoYm9hcmQtcm9vdC10b2tlbi10cmt4dCIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VydmljZS1hY2NvdW50Lm5hbWUiOiJkYXNoYm9hcmQtcm9vdCIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VydmljZS1hY2NvdW50LnVpZCI6IjkwYzdkYjZlLTRmNDEtNGRlNy1iMzc5LWU2YzIyODczYjE3NSIsInN1YiI6InN5c3RlbTpzZXJ2aWNlYWNjb3VudDprdWJlLXN5c3RlbTpkYXNoYm9hcmQtcm9vdCJ9.C59IlVhFAd8k9t7TqR6z9mwexzkBh86-peJLda2ELYbn72mS8gBsRkfIzfSA64qHW2ajMPFgFes-RZDSjsuekNg2xJe-k952lmJqUdThxtauGzemK0OCCy6UKSHPkkWzXaHtH6juMOGPX4k5C0tUv_9ro5LfnotjPNvCrUsk-BqJssT6Pe62jFHcxpxLzaM50IIpbR1gTMU9OKaN80cb-e4HXx633oXCsO3zy_8tUELX3jDmW7NTIq-7CACMGXZW6xChgSGtBxbueYENUJ06X1Z7p8nix7uOhoxYVw0QaUBc6sgBP0HMqbLPg9HWqvJe_XqZTp2GkgjBOgrFLCyC_g

#注意：kubernetes-dashboard可视化页面使用token进行登录, 观看右上角是否有消息，如果为空代表成功，如果不为空，根据情况自行修改yaml文件

由于centos停更，yum、docker等不支持，采用阿里云仓库搭建K8S

悦读