搭建K8S多节点LB负载均衡和keepalived

时间 2020-05-08

原文原文链接

搭建LB负载均衡和keepalived

1、环境优化
LB1html

[root@localhost ~]# hostnamectl set-hostname lb1
[root@localhost ~]# su   //修改主机名
[root@lb1 ~]# systemctl stop NetworkManager
//关闭NetworkManage服务
[root@lb1 ~]# systemctl disable NetworkManager
Removed symlink /etc/systemd/system/multi-user.target.wants/NetworkManager.service.
Removed symlink /etc/systemd/system/dbus-org.freedesktop.nm-dispatcher.service.
Removed symlink /etc/systemd/system/network-online.target.wants/NetworkManager-wait-online.service.
[root@lb1 ~]# setenforce //关闭加强型安全功能
[root@lb1 ~]# sed -i 's/^SELINUX=.*/SELINUX=disabled/' /etc/selinux/config
[root@lb1 ~]# iptables -F //清空防火墙策略

LB2node

[root@localhost ~]# hostnamectl set-hostname lb2
[root@localhost ~]# su
[root@lb2 ~]# systemctl stop NetworkManager
[root@lb2 ~]# systemctl disable NetworkManager
Removed symlink /etc/systemd/system/multi-user.target.wants/NetworkManager.service.
Removed symlink /etc/systemd/system/dbus-org.freedesktop.nm-dispatcher.service.
Removed symlink /etc/systemd/system/network-online.target.wants/NetworkManager-wait-online.service.
[root@lb2 ~]# setenforce 0
[root@lb2 ~]# sed -i 's/^SELINUX=.*/SELINUX=disabled/' /etc/selinux/config
[root@lb2 ~]# iptables -F

2.在两个LB服务器上安装nginxlinux

[root@lb1 ~]# echo -e '[nginx]\nname=nginx.repo\nbaseurl=http://nginx.org/packages/centos/7/$basearch/\ngpgcheck=0' > /etc/yum.repos.d/nginx.repo
[root@lb1 ~]# yum makecache
[root@lb1 ~]# yum install nginx -y

[root@lb2 ~]# echo -e '[nginx]\nname=nginx.repo\nbaseurl=http://nginx.org/packages/centos/7/$basearch/\ngpgcheck=0' > /etc/yum.repos.d/nginx.repo
[root@lb2 ~]# yum makecache
[root@lb2 ~]# yum install nginx -y

3.在两个LB服务器添加四层转发upstream
以LB1为例nginx

[root@lb1 ~]# vim /etc/nginx/nginx.conf 
events {
    worker_connections  1024;
}
stream {
    log_format  main    '$remote_addr $upstream_addr - [$time_local] $status $upstream_bytes_sent';
    access_log /var/log/nginx/k8s-access.log main;

    upstream k8s-apiserver {
        server 192.168.191.134:6443;
        server 192.168.191.133:6443;
        #两个master地址，apiserver端口号6443
    }
    server {
        listen 6443;
        proxy_pass k8s-apiserver;
    }
}

http {
[root@lb1 ~]# nginx -t
nginx: the configuration file /etc/nginx/nginx.conf syntax is ok
nginx: configuration file /etc/nginx/nginx.conf test is successful

4.开启nginx服务（以LB1为例）web

[root@lb1 ~]# systemctl start nginx
[root@lb1 ~]# systemctl status nginx
● nginx.service - nginx - high performance web server
   Loaded: loaded (/usr/lib/systemd/system/nginx.service; disabled; vendor preset: disabled)
   Active: active (running) since Sat 2020-05-03 13:02:50 CST; 5s ago
     Docs: http://nginx.org/en/docs/
  Process: 29485 ExecStart=/usr/sbin/nginx -c /etc/nginx/nginx.conf (code=exited, status=0/SUCCESS)
 Main PID: 29488 (nginx)
    Tasks: 2
   CGroup: /system.slice/nginx.service
           ├─29488 nginx: master process /usr/sbin/nginx -c /etc/nginx/nginx.conf
           └─29489 nginx: worker process

May 03 13:02:50 lb1 systemd[1]: Starting nginx - high performance web server...
May 03 13:02:50 lb1 systemd[1]: Started nginx - high performance web server.
[root@lb1 ~]# systemctl enable nginx
Created symlink from /etc/systemd/system/multi-user.target.wants/nginx.service to /usr/lib/systemd/system/nginx.service.

5.本地验证

2、部署Keepalived
以LB1为例
1.安装Keepaliveddocker

[root@lb1 ~]# yum install keepalived -y

2.修改Keepalived配置文件.shell

[root@lb1 ~]# mkdir /abc
[root@lb1 ~]# mount.cifs //192.168.0.88/linuxs /abc
Password for root@//192.168.0.88/linuxs:  
[root@lb1 ~]# cp /abc/k8s/keepalived.conf /etc/keepalived/keepalived.conf 
cp: overwrite ‘/etc/keepalived/keepalived.conf’? y

[root@lb1 ~]# vim /etc/keepalived/keepalived.conf 
! Configuration File for keepalived

global_defs {
   # 接收邮件地址
   notification_email {
     acassen@firewall.loc
     failover@firewall.loc
     sysadmin@firewall.loc
   }
   # 邮件发送地址
   notification_email_from Alexandre.Cassen@firewall.loc
   smtp_server 127.0.0.1
   smtp_connect_timeout 30
   router_id NGINX_MASTER
}

vrrp_script check_nginx {
    script "/etc/check_nginx.sh"        #这个配置文件后面会编辑
}   

vrrp_instance VI_1 {
    state MASTER
    interface ens32     #指定物理网口
    virtual_router_id 51 # VRRP 路由 ID实例，每一个实例是惟一的
    priority 100    # 优先级，备服务器设置 90
    advert_int 1    # 指定VRRP 心跳包通告间隔时间，默认1秒
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        192.168.191.135/24  #指定虚拟IP
    }
    track_script {          #监控脚本
        check_nginx
    }
}

LB2虚拟路由IP不要一致，state为BACKUP，其余同样bootstrap

vrrp_instance VI_1 {
    state BACKUP
    interface ens32
    virtual_router_id 52
    priority 90

3.编辑nginx脚本vim

[root@lb1 ~]# vim /etc/nginx/check_nginx.sh

count=$(ps -ef |grep nginx |egrep -cv "grep|$$")
#变量是建厂nginx是否开启，若是没有开启，那么就关闭keepalived
if [ "$count" -eq 0 ];then
    /etc/init.d/keepalived stop
fi
[root@lb1 ~]# chmod +x /etc/nginx/check_nginx.sh

4.开启LB1的Keepalived服务，LB2backup状态centos

[root@lb1 ~]# systemctl start keepalived.service

5.使用ip a命令能够查看到虚拟IP

[root@lb1 ~]# ip a
2: ens32: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
    link/ether 00:0c:29:ef:81:b6 brd ff:ff:ff:ff:ff:ff
    inet 192.168.191.130/24 brd 192.168.191.255 scope global noprefixroute dynamic ens32
       valid_lft 5355394sec preferred_lft 5355394sec
    inet 192.168.191.135/24 scope global secondary ens32

3、将k8s中的node节点关于apiserver地址指向为vip
1.node节点经过master找vip

[root@node01 ~]# cd /k8s/cfg/
[root@node01 cfg]# ls
bootstrap.kubeconfig  kubelet.config      kube-proxy
kubelet               kubelet.kubeconfig  kube-proxy.kubeconfig
[root@node01 cfg]# vim bootstrap.kubeconfig 
    server: https://192.168.191.133:6443
[root@node01 cfg]# vim kubelet.kubeconfig 
    server: https://192.168.191.133:6443
[root@node01 cfg]# vim kube-proxy.kubeconfig 
    server: https://192.168.191.133:6443

2.重启kubelet、proxy服务

[root@node01 cfg]# systemctl restart kubelet.service 
[root@node01 cfg]# systemctl restart kube-proxy.service

3.替换完成自检

[root@node01 cfg]# grep 100 *
bootstrap.kubeconfig:    server: https://192.168.191.133:6443
kubelet.kubeconfig:    server: https://192.168.191.133:6443
kube-proxy.kubeconfig:    server: https://192.168.191.133:6443

4.验证apiserver飘逸地址
先在lb1节点关掉nginx，再在lb2查看虚拟IP是否生效，如果检测到nginx发现关闭，keepalived会自动关闭

[root@lb1 ~]# pkill nginx
[root@lb1 ~]# ps -ef |grep nginx |egrep -cv "grep|$$"
0
[root@lb1 ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: ens32: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
    link/ether 00:0c:29:ef:81:b6 brd ff:ff:ff:ff:ff:ff
    inet 192.168.191.130/24 brd 192.168.191.255 scope global noprefixroute dynamic ens32
       valid_lft 5354179sec preferred_lft 5354179sec
    inet6 fe80::d8f:d3dc:3ef7:446/64 scope link noprefixroute 
       valid_lft forever preferred_lft forever

此时vip.不在LB1上，再查看LB2

[root@lb2 ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: ens32: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
    link/ether 00:0c:29:df:af:4e brd ff:ff:ff:ff:ff:ff
    inet 192.168.191.132/24 brd 192.168.247.255 scope global noprefixroute dynamic ens32
       valid_lft 5354144sec preferred_lft 5354144sec
    inet 192.168.191.133/24 scope global secondary ens33

3.重启LB1上的nginx 在查看在线vip回到了LB1上

[root@lb1 ~]# systemctl restart nginx
[root@lb1 ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: ens32: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
    link/ether 00:0c:29:ef:81:b6 brd ff:ff:ff:ff:ff:ff
    inet 192.168.191.132/24 brd 192.168.191.255 scope global noprefixroute dynamic ens32
       valid_lft 5354038sec preferred_lft 5354038sec
    inet 192.168.247.100/24 scope global secondary ens33

4、建立pod测试一下
1.此时node节点docker状态为
node1

[root@node01 cfg]# docker ps -a
CONTAINER ID        IMAGE               COMMAND             CREATED             STATUS              PORTS               NAMES
39f034a2f24e        centos:7            "/bin/bash"         3 days ago          Up 3 days                               beautiful_jennings
[root@node01 cfg]# docker images
REPOSITORY          TAG                 IMAGE ID            CREATED             SIZE
centos              7                   5e35e350aded        5 months ago        203MB

node2

[root@node02 cfg]# docker images
REPOSITORY          TAG                 IMAGE ID            CREATED             SIZE
centos              7                   5e35e350aded        5 months ago        203MB
[root@node02 cfg]# docker ps -a
CONTAINER ID        IMAGE               COMMAND             CREATED             STATUS              PORTS               NAMES
fea29d0ff39b        centos:7            "/bin/bash"         3 days ago

2.使用kublet建立pod
在集群中运行一个指定的镜像

[root@master1 cfg]# kubectl run nginx --image=nginx
kubectl run --generator=deployment/apps.v1beta1 is DEPRECATED and will be removed in a future version. Use kubectl create instead.
deployment.apps/nginx created
[root@master1 cfg]# kubectl get pods
NAME                    READY   STATUS    RESTARTS   AGE
nginx-dbddb74b8-sx4m6   1/1     Running   0          49s

pod在run运行状态前，还有一个containercreating建立状态

kubectl controls the Kubernetes cluster manager. 

Find more information at: https://kubernetes.io/docs/reference/kubectl/overview/

Basic Commands (Beginner):
  create         从文件或stdin建立资源。
  expose         使用 replication controller, service, deployment 或者 pod 并暴露它做为一个 新的 Kubernetes Service
  run            在集群中运行一个指定的镜像
  set            为 objects 设置一个指定的特征

Basic Commands (Intermediate):
  explain        查看资源的文档
  get            显示一个或更多 resources
  edit           在服务器上编辑一个资源
  delete         按文件名、stdin、资源和名称删除资源，或按资源和标签选择器删除资源

Deploy Commands:
  rollout        管理资源的推出
  scale          为 Deployment, ReplicaSet, Replication Controller 或者 Job 设置一个新的副本数量
  autoscale      自动调整一个 Deployment, ReplicaSet, 或者 ReplicationController 的副本数量

Cluster Management Commands:
  certificate    修改 certificate 资源.
  cluster-info   显示集群信息
  top            Display Resource (CPU/Memory/Storage) usage.
  cordon         标记 node 为 unschedulable
  uncordon       标记 node 为 schedulable
  drain          Drain node in preparation for maintenance
  taint          更新一个或者多个 node 上的 taints

Troubleshooting and Debugging Commands:
  describe       显示一个指定 resource 或者 group 的 resources 详情
  logs           输出容器在 pod 中的日志
  attach         Attach 到一个运行中的 container
  exec           在一个 container 中执行一个命令
  port-forward   Forward one or more local ports to a pod
  proxy          运行一个 proxy 到 Kubernetes API server
  cp             复制 files 和 directories 到 containers 和从容器中复制 files 和 directories.
  auth           Inspect authorization

Advanced Commands:
  apply          经过文件名或标准输入流(stdin)对资源进行配置
  patch          使用 strategic merge patch 更新一个资源的 field(s)
  replace        经过 filename 或者 stdin替换一个资源
  wait           Experimental: Wait for a specific condition on one or many resources.
  convert        在不一样的 API versions 转换配置文件

Settings Commands:
  label          更新在这个资源上的 labels
  annotate       更新一个资源的注解
  completion     Output shell completion code for the specified shell (bash or zsh)

Other Commands:
  alpha          Commands for features in alpha
  api-resources  Print the supported API resources on the server
  api-versions   Print the supported API versions on the server, in the form of "group/version"
  config         修改 kubeconfig 文件
  plugin         Provides utilities for interacting with plugins.
  version        输出 client 和 server 的版本信息

Usage:
  kubectl [flags] [options]

Use "kubectl <command> --help" for more information about a given command.
Use "kubectl options" for a list of global command-line options (applies to all commands).

3.查看pod网络，这也能够查看出此pod被部署到哪一个node上

[root@master1 cfg]# kubectl get pods -o wide
NAME                    READY   STATUS    RESTARTS   AGE   IP            NODE              NOMINATED NODE
nginx-dbddb74b8-sx4m6   1/1     Running   0          16m   172.17.42.3   192.168.191.131   <none>

4.此时在node2节点上有三个容器，一个刚刚建立的，一个是容器仓库，还有一个是以前测试flannel

[root@node02 cfg]# docker ps -a
CONTAINER ID        IMAGE                                                                 COMMAND                  CREATED             STATUS              PORTS               NAMES
6eff0af2c578        nginx                                                                 "nginx -g 'daemon of…"   16 minutes ago      Up 16 minutes                           k8s_nginx_nginx-dbddb74b8-sx4m6_default_cd5a2ea4-8c68-11ea-a668-000c29db840b_0
c4ca11690aa1        registry.cn-hangzhou.aliyuncs.com/google-containers/pause-amd64:3.0   "/pause"                 16 minutes ago      Up 16 minutes                           k8s_POD_nginx-dbddb74b8-sx4m6_default_cd5a2ea4-8c68-11ea-a668-000c29db840b_0
fea29d0ff39b        centos:7                                                              "/bin/bash"              3 days ago          Up 3 days                               kind_burnell
[root@node02 cfg]# docker images
REPOSITORY                                                        TAG                 IMAGE ID            CREATED             SIZE
nginx                                                             latest              602e111c06b6        8 days ago          127MB
centos                                                            7                   5e35e350aded        5 months ago        203MB
registry.cn-hangzhou.aliyuncs.com/google-containers/pause-amd64   3.0                 99e59f495ffa        3 years ago         747kB

在node2节点上能够直接访问nginx

[root@node02 cfg]# curl 172.17.42.3
<!DOCTYPE html>
<html>
<head>
<title>Welcome to nginx!</title>
<style>
    body {
        width: 35em;
        margin: 0 auto;
        font-family: Tahoma, Verdana, Arial, sans-serif;
    }
</style>

此时再次查看容器的日志

[root@master1 cfg]# kubectl logs nginx-dbddb74b8-sx4m6
172.17.42.1 - - [02/May/2020:11:52:45 +0000] "GET / HTTP/1.1" 200 612 "-" "curl/7.29.0" "-"