web高可用集群(lvs负载均衡+keepalved高可用)
| 主机 | IP地址 |
|---|---|
| VIP | 192.168.88.88 |
| DS1(MASTER) | 192.168.88.38 |
| DS2(BACKUP) | 192.168.88.66 |
| web1 | 192.168.88.10 |
| web2 | 192.168.88.20 |
|
+----------------+-----------------+
| |
192.168.88.38|---- VIP:192.168.88.88 ----|192.168.88.66
+-------+--------+ +--------+-------+
| DS1 | | DS2 |
| LVS+Keepalived | | LVS+Keepalived |
+-------+--------+ +--------+-------+
| |
+----------------+-----------------+
|
+------------+ | +------------+
| RS1 |192.168.88.10 | 192.168.88.20| RS2 |
| Web Server +--------------+---------------+ Web Server |
+------------+ +------------+
#架构图如上图所示。DS1、DS2 为两个 LB 节点，RS1、RS2 为两个真实的服务节点，通过一个虚拟的 IP 地址对外提供服务。
配置DS
#配置 Keepalived
[root@lvs1 ~]# yum install ipvsadm keepalived -y
DS1(MASTER节点)
[root@lvs1 ~]# vim /etc/keepalived/keepalived.conf
[root@lvs1 ~]# cat /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
notification_email {
[email protected]
[email protected]
[email protected]
}
notification_email_from [email protected]
smtp_server 127.0.0.1
smtp_connect_timeout 30
router_id lvs1
}
vrrp_instance VI_1 {
state MASTER # 两个 DS，一个为 MASTER 一个为 BACKUP
interface ens37 # 当前 IP 对应的网络接口，通过 ifconfig 查询
virtual_router_id 51 # 虚拟路由 ID(0-255)，在一个 VRRP 实例中主备服务器 ID 必须一样
priority 90 # 优先级值设定：MASTER 要比 BACKUP 的值大
advert_int 1 # 通告时间间隔：单位秒，主备要一致
authentication { # 认证机制，主从节点保持一致即可
auth_type PASS
auth_pass 1111
}
virtual_ipaddress {
192.168.88.88/24 # VIP，可配置多个
}
}
virtual_server 192.168.88.88 80 {
delay_loop 6 # 设置健康状态检查时间
lb_algo rr # 调度算法，这里用了 rr 轮询算法
lb_kind DR # 这里测试用了 Direct Route 模式
persistence_timeout 50 # 持久连接超时时间
protocol TCP
real_server 192.168.88.10 80 {
weight 1
TCP_CHECK {
connect_port 80
connect_timeout 3
nb_get_retry 3
delay_before_retry 3
}
}
real_server 192.168.88.20 80 {
weight 1
TCP_CHECK {
connect_port 80
connect_timeout 3
nb_get_retry 3
delay_before_retry 3
}
}
}
DS2(BACKUP) 节点
[root@proxy ~]# cat /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
notification_email {
[email protected]
[email protected]
[email protected]
}
notification_email_from [email protected]
smtp_server 127.0.0.1
smtp_connect_timeout 30
router_id proxy
}
vrrp_instance VI_1 {
state BACKUP
interface ens37
virtual_router_id 51
priority 80
advert_int 1
authentication {
auth_type PASS
auth_pass 1111
}
virtual_ipaddress {
192.168.88.88/24
}
}
virtual_server 192.168.88.88 80 {
delay_loop 6
lb_algo rr
lb_kind DR
persistence_timeout 0
protocol TCP
real_server 192.168.88.10 80 {
weight 1
TCP_CHECK {
connect_port 80
connect_timeout 3
nb_get_retry 3
delay_before_retry 3
}
}
real_server 192.168.88.20 80 {
weight 1
TCP_CHECK {
connect_port 80
connect_timeout 3
nb_get_retry 3
delay_before_retry 3
}
}
}
#配置完成后，分别重启 Keepalived 服务。
[root@proxy ~]# systemctl restart keepalived.service
配置 RS
需要在 RS 的网卡上配置 lo 为 VIP。配置脚本如下
#!/bin/bash
SNS_VIP=192.168.88.88
case "$1" in
start)
ifconfig lo:0 $SNS_VIP netmask 255.255.255.255 broadcast $SNS_VIP
/sbin/route add -host $SNS_VIP dev lo:0
echo "1" >/proc/sys/net/ipv4/conf/lo/arp_ignore
echo "2" >/proc/sys/net/ipv4/conf/lo/arp_announce
echo "1" >/proc/sys/net/ipv4/conf/all/arp_ignore
echo "2" >/proc/sys/net/ipv4/conf/all/arp_announce
sysctl -p >/dev/null 2>&1
echo "RealServer Start OK"
;;
stop)
ifconfig lo:0 down
route del $SNS_VIP >/dev/null 2>&1
echo "0" >/proc/sys/net/ipv4/conf/lo/arp_ignore
echo "0" >/proc/sys/net/ipv4/conf/lo/arp_announce
echo "0" >/proc/sys/net/ipv4/conf/all/arp_ignore
echo "0" >/proc/sys/net/ipv4/conf/all/arp_announce
echo "RealServer Stoped"
;;
*)
echo "Usage: $0 {start|stop}"
exit 1
esac
exit 0
本地创建完后，并执行。
[root@web ~]# chmod +x lo.sh
[root@web ~]# ./lo.sh start
配置完成后，通过 VIP 就可以访问到 RS 上的服务了。
HA 测试
配置完双机热备后，我们就可以测试下，节点发生故障后以及 LB 切换失败后，能否保证服务的 HA。
在 LB 的主节点上输入 ip a，可以看到 VIP 目前已经正确配置在网卡上。
[root@lvs1 ~]# ip a
3: ens37: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
link/ether 00:0c:29:48:b3:7b brd ff:ff:ff:ff:ff:ff
inet 192.168.88.38/24 brd 192.168.88.255 scope global noprefixroute ens37
valid_lft forever preferred_lft forever
inet 192.168.88.88/24 scope global secondary ens37
valid_lft forever preferred_lft forever
inet6 fe80::dbb5:b534:a44a:a21/64 scope link noprefixroute
valid_lft forever preferred_lft forever
# 查看负载均衡的配置
[root@lvs1 ~]# ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
-> RemoteAddress:Port Forward Weight ActiveConn InActConn
TCP 192.168.88.88:80 rr
-> 192.168.88.10:80 Route 1 0 0
-> 192.168.88.20:80 Route 1 0 0
访问一下 VIP。
[root@ceph01 ~]# curl 192.168.88.88
test1
[root@ceph02 ~]# curl 192.168.88.88
test2[root@ceph03 ~]# curl 192.168.88.88
test1
[root@ceph03 ~]# curl 192.168.88.88
test1
看到服务可正常轮询
#此时手动停止一个 RS，再次访问 VIP
[root@web ~]# systemctl stop nginx
#LVS 会自动剔除无法访问的服务
[root@lvs1 ~]# ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
-> RemoteAddress:Port Forward Weight ActiveConn InActConn
TCP 192.168.88.88:80 rr
-> 192.168.88.20:80 Route 1 0 1
[root@ceph01 ~]# curl 192.168.88.88
test2
[root@ceph02 ~]# curl 192.168.88.88
test2
[root@ceph03 ~]# curl 192.168.88.88
test2
访问请求全都发送到剩下的web节点上。
[root@web ~]# systemctl start nginx
重启后，服务会被自动添加。
[root@lvs1 ~]# ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
-> RemoteAddress:Port Forward Weight ActiveConn InActConn
TCP 192.168.88.88:80 rr
-> 192.168.88.10:80 Route 1 0 0
-> 192.168.88.20:80 Route 1 0 3
#如果此时，手动停止 MASTER 上的 Keepalived，模拟 LB MASTER 节点挂了，VIP 会自动飘到 BACKUP LB 上。
[root@lvs1 ~]# ip a
3: ens37: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
link/ether 00:0c:29:48:b3:7b brd ff:ff:ff:ff:ff:ff
inet 192.168.88.38/24 brd 192.168.88.255 scope global noprefixroute ens37
valid_lft forever preferred_lft forever
inet 192.168.88.88/24 scope global secondary ens37
valid_lft forever preferred_lft forever
inet6 fe80::dbb5:b534:a44a:a21/64 scope link noprefixroute
valid_lft forever preferred_lft forever
[root@lvs1 ~]# systemctl stop keepalived.service
[root@lvs1 ~]# ip a
3: ens37: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
link/ether 00:0c:29:48:b3:7b brd ff:ff:ff:ff:ff:ff
inet 192.168.88.38/24 brd 192.168.88.255 scope global noprefixroute ens37
valid_lft forever preferred_lft forever
inet6 fe80::dbb5:b534:a44a:a21/64 scope link noprefixroute
valid_lft forever preferred_lft forever
[root@proxy ~]# ip a
3: ens37: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
link/ether 00:0c:29:e4:cd:ac brd ff:ff:ff:ff:ff:ff
inet 192.168.88.66/24 brd 192.168.88.255 scope global noprefixroute ens37
valid_lft forever preferred_lft forever
inet 192.168.88.88/24 scope global secondary ens37
valid_lft forever preferred_lft forever
inet6 fe80::20c:29ff:fee4:cdac/64 scope link
valid_lft forever preferred_lft forever
[root@ceph01 ~]# curl 192.168.88.88
test1
[root@ceph02 ~]# curl 192.168.88.88
test2
[root@ceph03 ~]# curl 192.168.88.88
test1
web节点正常访问。
此时，如果重启 MASTER 后，VIP 又会飘回去。MASTER 的优先级高于 BACKUP，从而实现 HA。
[root@lvs1 ~]# systemctl start keepalived.service
[root@lvs1 ~]# ip a
3: ens37: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
link/ether 00:0c:29:48:b3:7b brd ff:ff:ff:ff:ff:ff
inet 192.168.88.38/24 brd 192.168.88.255 scope global noprefixroute ens37
valid_lft forever preferred_lft forever
inet 192.168.88.88/24 scope global secondary ens37
valid_lft forever preferred_lft forever
inet6 fe80::dbb5:b534:a44a:a21/64 scope link noprefixroute
valid_lft forever preferred_lft forever