postgresql 高可用 etcd + patroni 之八 haproxy + keepalived

时间 2020-08-22

标签 postgresql 可用 etcd patroni 之八 haproxy keepalived 栏目 Postgre SQL 繁體版

原文原文链接

os: ubuntu 16.04
postgresql: 9.6.9
etcd: 3.2.18
patroni: 1.4.4
haproxy: 1.6.3
keepalived: 1.2.24html

ip规划
192.168.56.101 node1 master
192.168.56.102 node2 slave
192.168.56.103 node3 slave
192.168.56.104 node4 slave
192.168.56.105 node5 slavenode

192.168.56.201 haproxy1
192.168.56.202 haproxy2web

192.168.56.210 keepalived vip
192.168.56.211 keepalived vip
192.168.56.212 keepalived vipsql

keepalived 是 ha 的一把利器，应用范围极广，这里主要用来对 haproxy 作高可用。关于 keepalived 的更多知识，能够自行baidu。ubuntu

安装、配置好haproxy

haproxy一、haproxy2节点都须要作
详细过程略，参考上一篇blogbash

安装 keepalived

# apt install keepalived
Reading package lists... Done
Building dependency tree       
Reading state information... Done
The following additional packages will be installed:
  ipvsadm libnl-route-3-200 libsensors4 libsnmp-base libsnmp30
Suggested packages:
  heartbeat ldirectord lm-sensors snmp-mibs-downloader
The following NEW packages will be installed:
  ipvsadm keepalived libnl-route-3-200 libsensors4 libsnmp-base libsnmp30
0 upgraded, 6 newly installed, 0 to remove and 171 not upgraded.
Need to get 1,427 kB of archives.
After this operation, 5,570 kB of additional disk space will be used.
Do you want to continue? [Y/n]

去除了 /usr/share/snmp、/usr/share/doc、/usr/share/man微信

# dpkg -L keepalived
/.
/usr
/usr/lib
/usr/lib/systemd
/usr/lib/systemd/system
/usr/lib/systemd/system/keepalived.service
/usr/bin
/usr/bin/genhash
/usr/share
/usr/sbin
/usr/sbin/keepalived
/lib
/lib/systemd
/lib/systemd/system
/lib/systemd/system/keepalived.service
/etc
/etc/keepalived
/etc/default
/etc/default/keepalived
/etc/init.d
/etc/init.d/keepalived

查看 keepalived.serviceide

# more /lib/systemd/system/keepalived.service 
[Unit]
Description=Keepalive Daemon (LVS and VRRP)
After=network-online.target
Wants=network-online.target
# Only start if there is a configuration file
ConditionFileNotEmpty=/etc/keepalived/keepalived.conf

[Service]
Type=forking
KillMode=process
PIDFile=/var/run/keepalived.pid
# Read configuration variable file if it is present
EnvironmentFile=-/etc/default/keepalived
ExecStart=/usr/sbin/keepalived $DAEMON_ARGS
ExecReload=/bin/kill -HUP $MAINPID

[Install]
WantedBy=multi-user.target

查看 /etc/init.d/keepalivedsvg

# more /etc/init.d/keepalived

#! /bin/sh
#
# keepalived   LVS cluster monitor daemon.
#
#              Written by Andres Salomon <dilinger@voxel.net>
#
### BEGIN INIT INFO
# Provides:          keepalived
# Required-Start:    $syslog $network $remote_fs
# Required-Stop:     $syslog $network $remote_fs
# Default-Start:     2 3 4 5
# Default-Stop:      0 1 6
# Short-Description: Starts keepalived
# Description:       Starts keepalived lvs loadbalancer
### END INIT INFO
PATH=/sbin:/bin:/usr/sbin:/usr/bin
DAEMON=/usr/sbin/keepalived
NAME=keepalived
DESC=keepalived
CONFIG=/etc/keepalived/keepalived.conf
TMPFILES="/tmp/.vrrp /tmp/.healthcheckers"

#includes lsb functions 
. /lib/lsb/init-functions

test -f $CONFIG || exit 0 
test -f $DAEMON || exit 0

# Read configuration variable file if it is present
[ -r /etc/default/$NAME ] && . /etc/default/$NAME

case "$1" in
  start)
	log_daemon_msg "Starting $DESC" "$NAME"
	for file in $TMPFILES
	do 
       		test -e $file && test ! -L $file && rm $file
	done
	if start-stop-daemon --start --quiet --pidfile /var/run/$NAME.pid \
               --exec $DAEMON -- $DAEMON_ARGS; then
		log_end_msg 0
	else
		log_end_msg 1
	fi
       ;;
  stop)
	log_daemon_msg "Stopping $DESC" "$NAME"
	if start-stop-daemon --oknodo --stop --quiet --pidfile /var/run/$NAME.pid \
        	--exec $DAEMON; then
		log_end_msg 0
	else
		log_end_msg 1
	fi
       ;;
  reload|force-reload)
	log_action_begin_msg "Reloading $DESC configuration..."
      	if start-stop-daemon --stop --quiet --signal 1 --pidfile \
               /var/run/$NAME.pid --exec $DAEMON; then
	       log_end_msg 0
	else
		log_action_end_msg 1
	fi
       ;;
  restart)
  	log_action_begin_msg "Restarting $DESC" "$NAME"

       	start-stop-daemon --stop --quiet --pidfile \
		/var/run/$NAME.pid --exec $DAEMON || true 
       sleep 1
       if start-stop-daemon --start --quiet --pidfile \
               /var/run/$NAME.pid --exec $DAEMON -- $DAEMON_ARGS; then
	       log_end_msg 0
	else
		log_end_msg 1
	fi
       ;;
  *)
       echo "Usage: /etc/init.d/$NAME {start|stop|restart|reload|force-reload}" >&2
       exit 1
       ;;
esac

exit 0

配置 keepalived

注意：能够修改 state为 MASTER、BACKUPpost

# vi /etc/keepalived/keepalived.conf


! Configuration File for keepalived    #! 表示注释
global_defs {
        router_id LVS_HAProxy1                  # 路由标示，相同局域网内惟一
}

vrrp_script chk_health {
        script "/etc/keepalived/scripts/health_check.sh"   # 监控脚本
        interval 5   # 每隔5s探测一次
        fall 5       # 连续检测5次失败后，则进行切换IP
        rise 5       # 5次检测成功后将OK

}


vrrp_instance VI_HAProxy {                      # 定义VRRP热备实例
        state MASTER                            # 热备状态，MASTER(主)；SLAVE(从)；BACKUP(备份);只能有一个MASTER
        interface eno1                          # 承载VIP地址的物理接口
        virtual_router_id 51                    # 虚拟路由器的ID号，组内保存一致
        priority 100                            # 优先级,数值越大优先级越高。
        advert_int 2                            # 主备之间的通告间隔秒数（心跳频率）
        nopreempt                               # 不抢占，只在优先级高的机器上设置便可，优先级低的机器不设置
        authentication {                        # 认证信息，每一个热备组保持一致
                auth_type PASS                  # 认证类型，主备切换时的验证
                auth_pass 1qaz                  # 密码字串
        }
        track_script {
                chk_health
        }
        virtual_ipaddress {                     # 指定漂移地址（VIP），能够有多个
                192.168.56.210/24 dev eno1 label eno1:1
                192.168.56.211/24 dev eno1 label eno1:2
                192.168.56.212/24 dev eno1 label eno1:3
        } 
        notify_master /etc/keepalived/scripts/lvs_notify_master.sh     # 切换成为主时的通知
        notify_backup /etc/keepalived/scripts/lvs_notify_backup.sh     # 切换成为备时的通知
        notify_fault  /etc/keepalived/scripts/lvs_notify_fault.sh      # 调用失败时的通知
}

/etc/keepalived/scripts/health_check.sh

#!/bin/bash
# 检查本身是否能够成为主 0:能够 大于0:不能够

IP_GATEWAY="192.168.56.1"
IP_LIST=$IP_GATEWAY" 192.168.56.201 192.168.56.202"

logfile="/var/log/keepalived_lvs_status.log"
log() {
	now=`date`
	content=$1
	echo  ${now}":"${content}  >>$logfile
}

ping_status() {
    IP=$1
    for ((i=1;i<=3;i++)); do
        if ping -c 1 -W 1 $IP >/dev/null; then
            #log "$IP Ping is successful."
            return 0
        else
            log "$IP Ping is failure " 
        fi
    done
    #三次都没有ping通，按失败处理。
    if [ "$IP" = "$IP_GATEWAY" ] ;then
        return 99
    else
        return 1
    fi
}

log '---------- health check start ...---------'

FAIL_COUNT=0
for IP in $IP_LIST; do
    log "ping $IP ..."
    ping_status $IP
    ret=$?
    FAIL_COUNT=$(($FAIL_COUNT+$ret))
done

log "fail ping num:$FAIL_COUNT"


if [ $FAIL_COUNT -gt 2 ]; then
    log "fail ping more than 2,exit 1"
    exit 1	
fi

status=$(systemctl status haproxy|grep Active|grep running|wc -l)
if [ "${status}" = "0" ]; then
    log 'haproxy is not running, restart it'
    service haproxy restart

    status2=$(systemctl status haproxy|grep Active|grep running|wc -l)
    if [ "${status2}" = "0"  ]; then
        log 'haproxy is not running, exit 1'
        exit 1
    fi
else
    log 'haproxy is running'
fi

log 'check ok,exit 0'
exit 0

/etc/keepalived/scripts/lvs_notify_master.sh

#!/bin/bash
# 调用公司内部的短信接口通知或者微信通知，邮件通知时效性较差

logfile="/var/log/keepalived_lvs_master.log"
log() {
        now=`date`
        content=$1
        echo  ${now}":"${content}  >>$logfile
}

log 'keepalived to master'

/etc/keepalived/scripts/lvs_notify_backup.sh

#!/bin/bash
# 调用公司内部的短信接口通知或者微信通知，邮件通知时效性较差

logfile="/var/log/keepalived_lvs_backup.log"
log() {
        now=`date`
        content=$1
        echo  ${now}":"${content}  >>$logfile
}

log 'keepalived to backup'

/etc/keepalived/scripts/lvs_notify_fault.sh

#!/bin/bash
# 调用公司内部的短信接口通知或者微信通知，邮件通知时效性较差

logfile="/var/log/keepalived_lvs_fault.log"
log() {
        now=`date`
        content=$1
        echo  ${now}":"${content}  >>$logfile
}

log 'keepalived to fault'

验证 keepalived

# systemctl start keepalived
# systemctl enable keepalived
# ps -ef|grep -i keep
root     16568     1  0 10:34 ?        00:00:00 /usr/sbin/keepalived
root     16569 16568  0 10:34 ?        00:00:00 /usr/sbin/keepalived
root     16570 16568  0 10:34 ?        00:00:00 /usr/sbin/keepalived

# ip a show eno1
2: eno1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
    link/ether 08:00:27:7d:f5:f0 brd ff:ff:ff:ff:ff:ff
    inet 192.168.56.101/24 brd 192.168.56.255 scope global eno1
       valid_lft forever preferred_lft forever
    inet 192.168.56.210/24 scope global secondary eno1:1
       valid_lft forever preferred_lft forever
    inet 192.168.56.211/24 scope global secondary eno1:2
       valid_lft forever preferred_lft forever
    inet 192.168.56.212/24 scope global secondary eno1:3
       valid_lft forever preferred_lft forever
    inet6 fe80::a00:27ff:fe7d:f5f0/64 scope link 
       valid_lft forever preferred_lft forever

chk_health功能验证

# systemctl stop haproxy

# tail -f /var/log/keepalived_lvs_status.log
Mon Oct 15 10:40:53 CST 2018:---------- health check start ...---------
Mon Oct 15 10:40:53 CST 2018:ping 192.168.56.1 ...
Mon Oct 15 10:40:53 CST 2018:ping 192.168.56.101 ...
Mon Oct 15 10:40:53 CST 2018:ping 192.168.56.102 ...
Mon Oct 15 10:40:53 CST 2018:fail ping num:0
Mon Oct 15 10:40:53 CST 2018:haproxy is not running, restart it
Mon Oct 15 10:40:53 CST 2018:check ok,exit 0

notify_master功能验证
haproxy1 节点上关闭 keepalived

# systemctl stop keepalived

haproxy2 节点上查看日志

# cat /var/log/keepalived_lvs_master.log 
Mon Oct 15 10:45:53 CST 2018:keepalived to master

notify_backup功能验证
haproxy1 节点再启动 keepalived,因为是不抢占模式，因此haproxy1起来后就是backup模式。

# systemctl start keepalived
# cat /var/log/keepalived_lvs_backup.log 
Mon Oct 15 11:15:46 CST 2018:keepalived to backup

此时，应用只须要链接 keepalived 的 vip 便可。

参考：
http://www.javashuo.com/article/p-esqjlspd-ds.html
http://www.keepalived.org/