k8s部署微服务

一、资源准备

1.ECS

ECS服务器	操作系统	硬件配置
master *1	CentOS7.9 内核5.4	CPU：8C 内存：16G 系统盘：50GiB 数据盘：100GiB
worker *3	CentOS7.9 内核5.4	CPU：16C 内存：64G 系统盘：100GiB 数据盘：500GiB
RDS *1	主备	CPU：4C 内存：16G 存储：500GiB

2.ELB、弹性IP

ELB负载均衡	规格
负载均衡	四层（TCP/UDP）
弹性公网IP	50M

3.组件及功能软件、中间件

组件名称	版本
kube-apiserver	v1.22.17
kube-scheduler	v1.22.17
kube-controller-manager	v1.22.17
kubelet	v1.22.17
kubectl	v1.22.17
kube-proxy	v1.22.17
etcd	v3.5.1
flannel	v0.16.1
coredns	v0.16.1
docker	20.10.99
ingress-nginx-controller	v1.0.0
cfssl	-
nacos-server	v2.1.0
seata-server	1.4.2
nginx	1.22.0
mysql	8.0.29
redis	6.0.16
openjdk	11
docker-distribution	2.6.2

二、部署Kubernetes集群

二进制部署文档

# 相关操作已经上传博客跳转查看

三、部署MySQL

下载MySQL

# 下载MySQL
# 创建RPM目录
mkdir ~/MySQL_RPM

# 解压压缩包
tar xf mysql-8.0.27-1.el7.x86_64.rpm-bundle.tar -C ~/MySQL_RPM
cd MySQL_RPM

# yum安装MySQL
yum -y install *.rpm

# 修改/etc/my.cnf
[mysqld]

default-storage-engine=INNODB
character_set_server=utf8mb4

port=3306   # 端口监听
datadir=/data/store/mysql/data  # 指定mysql数据目录
log-output=FILE
log-error="mysql.err"
lower_case_table_names=1
secure-file-priv=''
#skip-grant-tables
max_connections=1500

# 开启MySQL
systemctl enable mysqld --now

# 初始密码存放在/var/log/mysqld.log内
cat /var/log/mysqld.log | grep password
#...[Note] A temporary password is generated for root@localhost: GG,?o)hxv1%h

# 登陆MySQL
mysql -uroot -p'GG,?o)hxv1%h'

# 修改mysql-root密码
alter user  localhost@'root' identified by 'xxxxxx';

# 修改mysql-root连接权限
use mysql;
update user set host='%' where user='root';
flush privileges;

# 导入nacos和服务所需库表
# nacos建表sql语句在https://github.com/alibaba/nacos/blob/master/distribution/conf/mysql-schema.sql

# 在mysql-schema.sql的16行处添加建库sql语句
CREATE DATABASE `nacos` CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_bin';
USE nacos;

# 导入sql
mysql -uroot -p'xxxxxx' < mysql-schema.sql

# 导入服务应用库表
mysql -uroot -p'xxxxxx' < create-db-user.sql
mysql -uroot -p'xxxxxx' < capp_pre.sql

四、部署Redis

下载Redis

# CentOS7.9版本自带gcc版本为4.8.5 redis6.0+需要gcc版本大于4.9所以需要升级gcc
yum -y install centos-release-scl
yum -y install devtoolset-9-gcc devtoolset-9-gcc-c++ devtoolset-9-binutils
scl enable devtoolset-9 bash
echo "source /opt/rh/devtoolset-9/enable" >>/etc/profile

# 编译redis
make && make install

# 修改redis.conf
bind 0.0.0.0
protected-mode yes # 启动保护模式
daemonize yes  # 启动守护进程模式
cluster-enabled yes # 启动集群模式
cluster-config-file /var/lib/redis/nodes.conf
cluster-node-timeout 5000
appendonly yes 启用aof持久化

# 启动redis创建集群
bin/redis-cli --cluster create  192.168.xx.xx:6379 192.168.xx.xx:6379 192.168.xx.xx:6379 --cluster-replicas 0

# 修改redis.conf添加密码 # 每台redis都需操作
redis-cli 
127.0.0.1:6379> config set requirepass Redis@Pass
127.0.0.1:6379> config set masterauth Redis@Pass
127.0.0.1:6379> config rewrite

五、部署Naco

# 下载nacos镜像
# 这里用的是docker-distribution作为私仓
docker pull nacos/nacos-server:v2.1.0
docker tag nacos/nacos-server:v2.1.0 registr:5000/nacos/nacos-server:v2.1.0

# nacos的yaml文件
---
kind: Namespace
apiVersion: v1
metadata:
  name: nacos

---
kind: StatefulSet
apiVersion: apps/v1
metadata:
  name: nacos
  namespace: nacos
  labels:
    app: nacos
spec:
  replicas: 3
  selector:
    matchLabels:
      app: nacos
  template:
    metadata:
      creationTimestamp: null
      labels:
        app: nacos
      annotations:
        pod.alpha.kubernetes.io/initialized: 'true'
    spec:
      containers:
        - name: k8s-nacos
          image: registry:5000/nacos/nacos-server:v2.1.0
          ports:
            - name: server
              containerPort: 8848
              protocol: TCP
            - name: client-rpc
              containerPort: 9848
              protocol: TCP
            - name: raft-rpc
              containerPort: 9849
              protocol: TCP
            - name: old-raft-rpc
              containerPort: 7848
              protocol: TCP
          env:
            - name: NACOS_REPLICAS
              value: '3'
            - name: MYSQL_SERVICE_HOST
              valueFrom:
                configMapKeyRef:
                  name: nacos-cm
                  key: mysql.host
            - name: MYSQL_SERVICE_DB_NAME
              valueFrom:
                configMapKeyRef:
                  name: nacos-cm
                  key: mysql.db.name
            - name: MYSQL_SERVICE_PORT
              valueFrom:
                configMapKeyRef:
                  name: nacos-cm
                  key: mysql.port
            - name: MYSQL_SERVICE_USER
              valueFrom:
                configMapKeyRef:
                  name: nacos-cm
                  key: mysql.user
            - name: MYSQL_SERVICE_PASSWORD
              valueFrom:
                configMapKeyRef:
                  name: nacos-cm
                  key: mysql.password
            - name: MODE
              value: cluster
            - name: NACOS_SERVER_PORT
              value: '8848'
            - name: PREFER_HOST_MODE
              value: hostname
            - name: MYSQL_SERVICE_DB_PARAM
              value: >-
                characterEncoding=utf8&connectTimeout=1000&socketTimeout=3000&autoReconnect=true&useSSL=false
            - name: NACOS_SERVERS
              value: >-
                nacos-0.nacos-headless.nacos.svc.cluster.local:8848
                nacos-1.nacos-headless.nacos.svc.cluster.local:8848
                nacos-2.nacos-headless.nacos.svc.cluster.local:8848
          resources:
            limits:
              memory: 1Gi
            requests:
              cpu: 500m
              memory: 512Mi
          terminationMessagePath: /dev/termination-log
          terminationMessagePolicy: File
          imagePullPolicy: IfNotPresent
      restartPolicy: Always
      terminationGracePeriodSeconds: 5
      dnsPolicy: ClusterFirst
      securityContext: {}
      affinity:
        podAntiAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            - labelSelector:
                matchExpressions:
                  - key: app
                    operator: In
                    values:
                      - nacos
              topologyKey: kubernetes.io/hostname
      schedulerName: default-scheduler
  serviceName: nacos-headless
  podManagementPolicy: OrderedReady
  updateStrategy:
    type: RollingUpdate
    rollingUpdate:
      partition: 0
  revisionHistoryLimit: 10

---
kind: Service
apiVersion: v1
metadata:
  name: nacos-headless
  namespace: nacos
  labels:
    app: nacos
    app: nacos-headless
spec:
  selector:
    app: nacos
  clusterIP: 10.0.0.66
  type: NodePort
  sessionAffinity: None
  ports:
    - name: server
      protocol: TCP
      port: 8848
      targetPort: 8848
    - name: client-rpc
      protocol: TCP
      port: 9848
      targetPort: 9848
    - name: raft-rpc
      protocol: TCP
      port: 9849
      targetPort: 9849
    - name: old-raft-rpc
      protocol: TCP
      port: 7848
      targetPort: 7848



---
kind: Ingress
apiVersion: networking.k8s.io/v1
metadata:
  name: nacos-ingress
  namespace: nacos
  labels:
    app: nacos
spec:
  ingressClassName: nginx
  rules:

  - host: nacos.cn
    http:
      paths:
      - path: /nacos
        pathType: Prefix
        backend:
          service:
            name: nacos-headless
            port:
              number: 8848


---
kind: ConfigMap
apiVersion: v1
metadata:
  name: nacos-cm
  namespace: nacos
data:
  mysql.db.name: nacos
  mysql.host: 192.168.xx.xx
  mysql.password: xxxxxx
  mysql.port: '3306'
  mysql.user: root

六、部署Seata

# 下载seata镜像
docker pull seataio/seata-server:1.4.2
docker tag seataio/seata-server:1.4.2 registry:5000/seata/seata-server:1.4.2

# seata的yaml文件
---
apiVersion: v1
kind: Namespace
metadata:
  name: seata
  labels:
    app: seata

---
apiVersion: v1
kind: ConfigMap
metadata:
  name: seata-cm
  namespace: seata
  annotations:
    version: 1.4.2 
data:
  registry.conf: |
    registry {
        type = "nacos"
        nacos {
          group="SEATA_GROUP"
          namespace =""
          application = "seata-server"
          serverAddr = "nacos-headless.nacos:8848"
          cluster = "default"
          username = "nacos"
          password = "nacos"
        }
    }
    config {
      type = "nacos"
      nacos {
        serverAddr = "nacos-headless.nacos:8848"
        group="SEATA_GROUP"
        namespace=""
        username = "nacos"
        password = "nacos"
        dataID = "seataServer.properties"
      }
    }

---
kind: Service
apiVersion: v1
metadata:
  namespace: seata
  name: seata-svc
  labels:
    app: seata
spec:
  ports:
    - name: http-8091
      protocol: TCP
      port: 8091
      targetPort: 8091
  selector:
    app: seata
  clusterIP: 10.0.0.91
  type: ClusterIP
  sessionAffinity: None

---
kind: Deployment
apiVersion: apps/v1
metadata:
  namespace: seata
  name: seata-app
  labels:
    app: seata
spec:
  replicas: 3
  selector:
    matchLabels:
      app: seata
  template:
    metadata:
      creationTimestamp: null
      labels:
        app: seata
    spec:
      volumes:
        - name: seata-cm
          configMap:
            name: seata-cm  
            defaultMode: 420
      containers:
        - name: seata
          image: 'registry:5000/seata/seata-server:1.4.2'
          ports:
            - name: http-8091
              containerPort: 8091
              protocol: TCP
          env:
            - name: SEATA_CONFIG_NAME
              value: 'file:/root/seata-config/registry'
          resources:
            limits:
              cpu: 500m
              memory: 500Mi
          volumeMounts:
            - name: seata-cm
              readOnly: true
              mountPath: /root/seata-config
      terminationGracePeriodSeconds: 10
      dnsPolicy: ClusterFirst
      serviceAccountName: default
      serviceAccount: default
      securityContext: {}
      affinity: {}
      schedulerName: default-scheduler
  strategy:
    type: RollingUpdate
    rollingUpdate:
      maxUnavailable: 25%
      maxSurge: 25%
  revisionHistoryLimit: 10
  progressDeadlineSeconds: 600

七、部署java服务

# 制作java环境镜像
docker pull openjdk:11
docker run -itd --name openjdk-11 openjdk:11
docker exec -it openjdk-11 /bin/bash
#> mkdir -p /data/{projects/capp/{logs/{console,logback},scripts,services},env}
#> exit
docker commit openjdk-11 openjdk-11:latest
docker rmi openjdk:11
docker tag openjdk-11:latest openjdk:11
docker rmi openjdk-11:latest

# 编写java应用启动相关脚本
mkdir capp/{auth,gateway,basic-data,user,....} -p
cat > capp/auth/env.sh <<EOF
#!/bin/bash
JAVA_OPTS=${OPTS:-"-Xms128m -Xmx256m -XX:MaxDirectMemorySize=128m -Dfile.encoding=utf-8"}
APP_ARGS=${ARGS:-"--spring.profiles.active=pre"}
APP_VERSION=${VERSION:-"0.0.1-SNAPSHOT"}

export JAVA_OPTS APP_ARGS APP_VERSION 
EOF

cat > capp/auth/start-service.sh <<EOF
#!/bin/bash
$JAVA_HOME/bin/java $JAVA_OPTS  -jar  ${INSTALL_DIR}/services/${APP_NAME}-${APP_VERSION}.jar  $APP_ARGS >${INSTALL_DIR}/logs/console/${APP_NAME}.log
EOF

cat > capp/auth/stop-service.sh <<EOF
#!/bin/bash
$JAVA_HOME/bin/jps | grep ${APP_NAME}-${APP_VERSION}.jar | awk '{print "kill -9 "$1}'|sh
EOF

cat > capp/auth/restart-service.sh <<EOF
#!/bin/bash
INSTALL_DIR=/data/projects/capp
source ${INSTALL_DIR}/scripts/env.sh

APP_NAME=capp-auth
export INSTALL_DIR APP_NAME

${INSTALL_DIR}/scripts/stop-service.sh
${INSTALL_DIR}/scripts/start-service.sh
EOF

# 把固定脚本打tar包
tar -zvcf scripts.tar env.sh start-service.sh stop-service.sh 

# 制作java应用镜像
# 编写Dockerfile
cat > Dockerfile <<EOF
FROM openjdk:11
ADD  scripts.tar /data/projects/capp/scripts/
COPY restart-service.sh /data/projects/capp/scripts/   #不同应用需要修改
COPY capp-auth-0.0.1-SNAPSHOT.jar /data/projects/capp/services/ #不同应用需要修改
EOF

docker build -t registry:5000/capp/capp-auth .
# 其他应用相同步骤

# 创建控制台和服务日志挂载路径
mkdir -p /data/capp/logs/{console,logback}


# 编写应用服务yaml文件
---
apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    app: capp-auth
  name: capp-auth
  namespace: capp
spec:
  replicas: 2
  selector:
    matchLabels:
      app: capp-auth
  template:
    metadata:
      creationTimestamp: null
      labels:
        app: capp-auth
    spec:
      volumes:
      - name: console
        hostPath:
          path: /data/capp/logs/console
      - name: logback
        hostPath:
          path: /data/capp/logs/logback     
      dnsPolicy: None
      dnsConfig:
        nameservers:
          - xxx.xxx.xxx.xxx
      containers:
      - image: registry:5000/capp/capp-auth
        name: capp-auth
        command: [ "/bin/bash","-c" ]
        args: [ "/data/projects/capp/scripts/restart-service.sh" ]
        volumeMounts:
        - name: console
          mountPath: /data/projects/capp/logs/console
        - name: logback
          mountPath: /data/projects/capp/logs/logback
        env:
        - name: OPTS
          value: "-Xms64m -Xmx128m -XX:MaxDirectMemorySize=64m -Dfile.encoding=utf-8"

k8s部署微服务

一、资源准备

1.ECS

2.ELB、弹性IP

3.组件及功能软件、中间件

二、部署Kubernetes集群

三、部署MySQL

四、部署Redis

五、部署Naco

六、部署Seata

七、部署java服务

悦读