keepalived+nginx的高可用

keepalived+nginx的高可用
#########
nginx+keepalived环境：
master:10.10.54.61(vip:10.10.54.69)
backup:10.10.54.64(vip:10.10.54.69)
realserver：10.10.54.63
realserver：10.10.54.67

本文不是作lvs，因此realserver不是配置在keepalived.conf
而是在nginx的配置文件中upstream

此架构需考虑的问题
1）Master没挂，则Master占有vip且nginx运行在Master上
2）Master挂了，则backup抢占vip且在backup上运行nginx服务
3）若是master服务器上的nginx服务挂了，则vip资源转移到backup服务器上
4）检测后端服务器的健康状态
Master和Backup两边都开启nginx服务，不管Master仍是Backup，当其中的一个keepalived服务中止后，vip都会漂移到keepalived服务还在的节点上，
若是要想使nginx服务挂了，vip也漂移到另外一个节点，则必须用脚本或者在配置文件里面用shell命令来控制。
1、安装keepalived+nginx
10.10.54.61/64
#############################
源码编译nginx
1.下载
[root@gyf  soft]#wget http://nginx.org/download/nginx-1.4.5.tar.gz
[root@gyf  soft]# tar xvf nginx-1.4.5.tar.gz
2.编译
[root@gyf  nginx-1.4.5]# ./configure --prefix=/usr/local/nginx --user=apache --group=apache --with-http_stub_status_module   --with-http_gzip_static_module --with-http_ssl_module
///
--with-http_stub_status_module     enable ngx_http_stub_status_module     ---支持监控
--with-http_gzip_static_module     enable ngx_http_gzip_static_module     ---支持压缩

3.安装
[root@gyf  nginx-1.4.5]# make && make install
4.启动
[root@gyf  conf]# /usr/local/nginx/sbin/nginx -c /usr/local/nginx/conf/nginx.conf
nginx: [emerg] getpwnam("apache") failed
[root@gyf  conf]# useradd apache

[root@gyf  conf]# /usr/local/nginx/sbin/nginx -c /usr/local/nginx/conf/nginx.conf
nginx: [emerg] bind() to 0.0.0.0:80 failed (98: Address already in use)
nginx: [emerg] bind() to 0.0.0.0:80 failed (98: Address already in use)
nginx: [emerg] bind() to 0.0.0.0:80 failed (98: Address already in use)
nginx: [emerg] bind() to 0.0.0.0:80 failed (98: Address already in use)
nginx: [emerg] bind() to 0.0.0.0:80 failed (98: Address already in use)
nginx: [emerg] still could not bind()
[root@gyf  conf]# netstat -ntlp|grep 80
tcp        0      0 0.0.0.0:80                  0.0.0.0:*                   LISTEN      1572/httpd          
[root@gyf  conf]# apachectl stop
[root@gyf  conf]# /usr/local/nginx/sbin/nginx -c /usr/local/nginx/conf/nginx.conf
5.关闭:给父进程发送一个TERM信号，试图杀死它和它的子进程。
[root@s01 logs]# cat /usr/local/nginx/logs/nginx.pid | xargs kill -TERM

6.重启
[root@s01 logs]# cat /usr/local/nginx/logs/nginx.pid | xargs kill -HUP

HUP      重启
TERM,INT 快速中止
USR1    从新打开日志文件，用于日志切割
USR2    平滑升级可执行程序
QUIT     从容关闭
WINCH    从容关闭工做进程

//测试配置文件
[root@s01 html]# /usr/local/nginx/sbin/nginx -t -c /usr/local/nginx/conf/nginx.conf

7.制做nginx自启动10.10.54.61/64
[root@gyf  init.d]# vim /etc/init.d/nginx
#!/bin/bash
#chkconfig: 2345 80 90
#description:  nginx
alter=$1
nginx=/usr/local/nginx/sbin/nginx
nginx_conf=/usr/local/nginx/conf/nginx.conf
nginx_pid=/usr/local/nginx/logs/nginx.pid
. /etc/rc.d/init.d/functions
function if_info
{
        if [ $2 == 0 ];then
                echo -n "nginx $1 is ok!" && success && echo
        else
                echo -n "nginx $1 is error!" && success && echo
        fi
}
case $alter in
  start)
         if [ -f $nginx_pid ];then

                echo "nginx is already start!"
        else
                $nginx -c $nginx_conf
                if_info start $?
        fi
        ;;
  stop)
       if [ ! -f $nginx_pid ];then
                echo "nginx is already stop!"       
         else
                kill -TERM `cat $nginx_pid`

                if_info stop $?
        fi
        ;;
  restart)
        if [ ! -f $nginx_pid ];then
                echo "nginx is stop,please start nginx!"
        else
                kill -HUP `cat $nginx_pid`
                if_info restart $?
        fi
        ;;
test)
        $nginx -t -c $nginx_conf
#       $nginx -t
        if_info test $?
        ;;
  status)
        if [ ! -f $nginx_pid ];then
                echo "nginx is stop"
        else
                echo "nginx is runing"   
        fi
        ;;
  *)
        echo "Usage: $0 {start|stop|status|restart|test}"
        ;;
esac

chmod  +x  /etc/init.d/nginx

chkconfig  --add  nginx
chkconfig    nginx  on

chkconfig   nginx  --list

 /etc/init.d/nginx start
[root@Cent64 keepalived]# ps -ef |grep nginx
                                 
#########################
编译ipvsadm10.10.54.61/64

//安装依赖包
yum -y install wget libnl* popt* gcc.x86_64 gcc-c++.x86_64 gcc-objc++.x86_64 kernel-devel.x86_64 make popt-static.x86_64

//编译ipvsadm
[root@Cent64 softs]# tar xvf ipvsadm-1.26.tar.gz
[root@tech2 lvs]# cd ipvsadm-1.26
[root@tech2 ipvsadm-1.26]# make && make install

//确认lvs模块
[root@tech2 ipvsadm-1.26]# modprobe -l|grep ipvs
kernel/net/netfilter/ipvs/ip_vs.ko
kernel/net/netfilter/ipvs/ip_vs_rr.ko
kernel/net/netfilter/ipvs/ip_vs_wrr.ko
kernel/net/netfilter/ipvs/ip_vs_lc.ko
kernel/net/netfilter/ipvs/ip_vs_wlc.ko
kernel/net/netfilter/ipvs/ip_vs_lblc.ko
kernel/net/netfilter/ipvs/ip_vs_lblcr.ko
kernel/net/netfilter/ipvs/ip_vs_dh.ko
kernel/net/netfilter/ipvs/ip_vs_sh.ko
kernel/net/netfilter/ipvs/ip_vs_sed.ko
kernel/net/netfilter/ipvs/ip_vs_nq.ko
kernel/net/netfilter/ipvs/ip_vs_ftp.ko

3.编译keepalived
[root@tech2 lvs]# tar xvf keepalived-1.2.9.tar.gz
[root@tech2 keepalived-1.2.9]# ls

//基础软件包
  In order to compile Keepalived needs the following libraries :

  * OpenSSL, <www.openssl.org>
  * popt

[root@tech2 keepalived-1.2.9]# yum install -y net-snmp.x86_64 net-snmp-devel.x86_64

[root@tech2 keepalived-1.2.9]# ./configure --prefix=/usr/local/keepalived --enable-snmp --sysconfdir=/etc

Keepalived configuration
------------------------
Keepalived version       : 1.2.9
Compiler                 : gcc
Compiler flags           : -g -O2
Extra Lib                : -Wl,-z,relro -Wl,-z,now -L/usr/lib64 -lnetsnmpagent -lnetsnmphelpers -lnetsnmpmibs -lnetsnmp -Wl,-E -Wl,-rpath,/usr/lib64/perl5/CORE -lssl -lcrypto -lcrypt  -lnl
Use IPVS Framework       : Yes
IPVS sync daemon support : Yes
IPVS use libnl           : Yes
Use VRRP Framework       : Yes
Use VRRP VMAC            : Yes
SNMP support             : Yes
SHA1 support             : No
Use Debug flags          : No

[root@tech2 keepalived-1.2.9]# make && make install

[root@tech2 sbin]# cp /usr/local/keepalived/sbin/keepalived  /sbin/
[root@tech2 bin]# cp /usr/local/keepalived/bin/genhash /bin/
[root@tech2 bin]# chkconfig --add keepalived
[root@centos61 ~]# /etc/init.d/keepalived start
二，修改keepalived配置文件
10.10.54.61
[root@centos61 ~]# vim /etc/keepalived/keepalived.conf

! Configuration File for keepalived

global_defs {
   notification_email {
     yangry@shiwei.com
   }
   notification_email_from yangry@shiwei.com
   smtp_server mail.shiwei.com
   smtp_connect_timeout 30
   router_id LVS_MASTER1  #表示运行keepalived服务器的一个标识，发邮件时显示在邮件主题中的信息
}
vrrp_script chk_http_port {
script "/usr/local/keepalived/nginx.sh" ####检测nginx状态的脚本连接
interval 2
weight 2
}
vrrp_instance VI_2 {   #vrrp实例
    state MASTER     #MASTER/BACKUP
    interface eth0    ####HA 监测网络接口
    virtual_router_id 51  #虚拟路由标识，是一个数字，同一个VRRP实例使用惟一的标识,master和backup要同样
    priority 100          #用于主从模式，优先级主高于100，从低于100
    advert_int 1           #主备之间的通告间隔秒数
    authentication {        #认证用于主从模式，mater和backup配置同样
        auth_type PASS          ###主备切换时的验证
        auth_pass 1111          #密码
    }
track_script {
chk_http_port ### 执行监控的服务
}
    virtual_ipaddress {
      
 10.10.54.69/24 dev eth0 label eth0:1   ###########虚拟ip
    }
}

[root@centos61 ~]#vim /usr/local/keepalived/nginx.sh
#!/bin/bash
if [ `ps -C nginx --no-header |wc -l` -eq 0 ];then

#if [ `ps -ef|grep nginx:mater process|wc -l` -eq 0 ]; then
killall keepalived
fi

三，修改keepalived配置文件
10.10.54.64
[root@centos64 ~]# vim /etc/keepalived/keepalived.conf

! Configuration File for keepalived

global_defs {
   notification_email {
     yangry@shiwei.com
   }
   notification_email_from yangry@shiwei.com
   smtp_server mail.shiwei.com
   smtp_connect_timeout 30
   router_id LVS_SLAVE  #表示运行keepalived服务器的一个标识，发邮件时显示在邮件主题中的信息
}
vrrp_script chk_http_port {
 script "/usr/local/keepalived/nginx.sh" ####检测nginx状态的脚本连接
interval 2    #脚本执行间隔
weight 2       #脚本结果致使的优先级变动
}
vrrp_instance VI_2 {   #vrrp实例
    state BACKUP     #MASTER/BACKUP
    interface eth0    ####HA 监测网络接口
    virtual_router_id 51  #虚拟路由标识，是一个数字，同一个VRRP实例使用惟一的标识,master和backup要同样
    priority 80          #用于主从模式，优先级主高于100，从低于100
    advert_int 1           #主备之间的通告间隔秒数
    authentication {        #认证用于主从模式，mater和backup配置同样
        auth_type PASS          ###主备切换时的验证
        auth_pass 1111          #密码验证要一致
    }
track_script {
chk_http_port ### 执行监控的服务
}
    virtual_ipaddress {
      
 10.10.54.69/24 dev eth0 label eth0:1   ###########虚拟ip
    }
}

[root@centos64 ~]#vim /usr/local/keepalived/nginx.sh
#!/bin/bash
if [ `ps -C nginx --no-header |wc -l` -eq 0 ];then

#if [ `ps -ef|grep nginx:mater process|wc -l` -eq 0 ];then

killall keepalived
fi
######以上作完测试vip是否能够飘移，nginx中止vip也能漂移
四.nginx实现后端realserver的负载均衡
10.10.54.61/64
1.配置代理文件
[root@gyf  htdocs]# cd /usr/local/nginx/conf/
[root@gyf  conf]#mkdir virtual
[root@gyf  conf]# vim virtual/bbs.ssr.com.conf
upstream bbs_ssr_com {
    server 10.10.54.63:80 max_fails=3 weight=1 fail_timeout=60s;
    server 10.10.54.67:80 max_fails=3 weight=3 fail_timeout=60s;
    }


server {
     listen      80;
     server_name bbs.ssr.com; #bbs.ssr.com 的dns能解析到10.10.54.69
     access_log  logs/www.access.log;
    error_log   logs/www.error.log;
    location / {
        proxy_pass http://bbs_ssr_com;
        proxy_set_header HOST                               $host;
        proxy_set_header X-Real-IP                          $remote_addr;
        proxy_set_header X-Forwarded-For             $proxy_add_x_forwarded_for;
                }    
}

.主配置文件配置

[root@gyf  ~]# vi /usr/local/nginx/conf/nginx.conf
user  nginx nginx;
worker_processes  2;
error_log  logs/error.log info;
pid      logs/nginx.pid;
events {
    worker_connections  1024;
}
http {
    include      mime.types;
    default_type  application/octet-stream;
    #日志格式定义
    log_format main  ‘$remote_addr – $remote_user[$time_local] “$request” ‘
                      ‘$status $body_bytes_sent”$http_referer” ‘
                    ‘”$http_user_agent” “$http_x_forwarded_for”‘;
    access_log logs/access.log  main;
    sendfile        on;
keepalive_timeout  65;
#gzip压缩功能设置
    gzip on;
    gzip_min_length 1k;
    gzip_buffers    4 16k;
    gzip_http_version 1.0;
    gzip_comp_level 2;
    gzip_types text/plain application/x-javascripttext/css application/xml;
    gzip_vary on;

include virtual/bbs.ssr.com.conf;

}

五.在10.10.54.63/67上安装apache 制做网站
10.10.54.63/67
yum install -y httpd.x86_64 httpd-devel.x86_64 httpd-tools.x86_64

六.重启各类服务