高可用，多路冗余GFS2集群文件系统搭建详解

时间 2019-11-11

原文原文链接

2014.06

实验拓扑图：html

实验原理：node

实验目的：经过RHCS集群套件搭建GFS2集群文件系统，保证不一样节点可以同时对GFS2集群文件系统进行读取和写入，其次经过multipath实现node和FC，FC和Share Storage之间的多路冗余，最后实现存储的mirror复制达到高可用。python

GFS2：全局文件系统第二版，GFS2是应用最普遍的集群文件系统。它是由红帽公司开发出来的，容许全部集群节点并行访问。元数据一般会保存在共享存储设备或复制存储设备的一个分区里或逻辑卷中。web

实验环境：vim

 
         [root@storage1 ~] 
         # uname -r 
        
         2.6.32-279.el6.x86_64 
        
         [root@storage1 ~] 
         # cat /etc/redhat-release 
        
         Red Hat Enterprise Linux Server release 6.3 (Santiago) 
        
         [root@storage1 ~] 
         # /etc/rc.d/init.d/iptables status 
        
         iptables: Firewall is not running. 
        
         [root@storage1 ~] 
         # getenforce 
        
         Disabled

实验步骤：bash

一、前期准备工做服务器

0）、设置一台管理端（192.168.100.102manager.rsyslog.org）配置ssh 私钥、公钥，将公钥传递到全部节点上session

 
         [root@manager ~] 
         # ssh-keygen  \\生成公钥和私钥 
        
         Generating public 
         /private  
         rsa key pair. 
        
         Enter  
         file  
         in  
         which  
         to save the key ( 
         /root/ 
         . 
         ssh 
         /id_rsa 
         ): 
        
         Enter passphrase (empty  
         for  
         no passphrase): 
        
         …… 
        
         [root@manager ~] 
         # for i in {1..6}; do ssh-copy-id -i 192.168.100.17$i; done \\将公钥传输到各节点/root/.ssh/目录下 
        
         root@192.168.100.171's password: 
        
         Now try logging into the machine, with  
         "ssh '192.168.100.171'" 
         , and check  
         in 
         : 
        
         . 
         ssh 
         /authorized_keys 
        
         to  
         make  
         sure we haven 
         't added extra keys that you weren' 
         t expecting 
        
         ..…… 
        
         [root@manager ~] 
         # ssh node1  \\测试登陆 
        
         Last login: Sat Jun  8 17:58:51 2013 from 192.168.100.31 
        
         [root@node1 ~] 
         #

1）、配置双网卡IP，全部节点参考拓扑图配置双网卡，并配置相应IP便可app

 
         [root@storage1 ~] 
         # ifconfig eth0 | grep "inet addr" | awk -F[:" "]+ '{ print $4 }' 
        
         192.168.100.171 
        
         [root@storage1 ~] 
         # ifconfig eth1 | grep "inet addr" | awk -F[:" "]+ '{ print $4 }' 
        
         192.168.200.171 
        
         ……

2）、配置hosts文件并同步到全部节点去（也能够配置DNS，不过DNS解析绝对不会有hosts解析快，其次DNS服务器出问题会直接致使节点和节点以及和存储直接不可以解析而崩溃）less

 
         [root@manager ~] 
         # cat /etc/hosts 
        
         127.0.0.1 localhost localhost.rsyslog.org 
        
         192.168.100.102 manager  manager.rsyslog.org 
        
         192.168.100.171 storage1 storage1.rsyslog.org 
        
         192.168.200.171 storage1 storage1.rsyslog.org 
        
         192.168.100.172 storage2 storage2.rsyslog.org 
        
         192.168.200.172 storage2 storage2.rsyslog.org 
        
         192.168.100.173 node1 node1.rsyslog.org 
        
         192.168.200.173 node1 node1.rsyslog.org 
        
         192.168.100.174 node2 node2.rsyslog.org 
        
         192.168.200.174 node2 node2.rsyslog.org 
        
         192.168.100.175 node3 node3.rsyslog.org 
        
         192.168.200.175 node3 node3.rsyslog.org 
        
         192.168.100.176 node4 node4.rsyslog.org 
        
         192.168.200.176 node4 node4.rsyslog.org 
        
         [root@manager ~] 
         # for i in {1..6}; do scp /etc/hosts 192.168.100.17$i:/etc/ ; done 
        
         hosts                                                                           100%  591     0.6KB 
         /s    
         00:00 
        
         hosts                                                                           100%  591     0.6KB 
         /s    
         00:00 
        
         hosts                                                                           100%  591     0.6KB 
         /s    
         00:00 
        
         hosts                                                                           100%  591     0.6KB 
         /s    
         00:00 
        
         hosts                                                                           100%  591     0.6KB 
         /s    
         00:00 
        
         hosts                                                                           100%  591     0.6KB 
         /s    
         00:00

3）、配置yum源（将全部节点光盘挂接到/media/cdrom，若是不方便，也能够作NFS，将镜像挂载到NFS里面，而后节点挂载到NFS共享目录中便可，注意：不一样版本的系统，RHCS集群套件存放位置会有所不一样，因此yum源的指向位置也会有所不一样）

 
         [root@manager ~] 
         # cat /etc/yum.repos.d/rhel-gfs2.repo 
        
         [rhel-cdrom] 
        
         name=RHEL6U3-cdrom 
        
         baseurl= 
         file 
         : 
         ///media/cdrom 
        
         enabled=1 
        
         gpgcheck=0 
        
         [rhel-cdrom-HighAvailability] 
        
         name=RHEL6U3-HighAvailability 
        
         baseurl= 
         file 
         : 
         ///media/cdrom/HighAvailability 
        
         enabled=1 
        
         gpgcheck=0 
        
         [rhel-cdrom-ResilientStorage] 
        
         name=RHEL6U3-ResilientStorage 
        
         baseurl= 
         file 
         : 
         ///media/cdrom/ResilientStorage 
        
         enabled=1 
        
         gpgcheck=0 
        
         [rhel-cdrom-LoadBalancer] 
        
         name=RHEL6U3-LoadBalancer 
        
         baseurl= 
         file 
         : 
         ///media/cdrom/LoadBalancer 
        
         enabled=1 
        
         gpgcheck=0 
        
         [rhel-cdrom-ScalableFileSystem] 
        
         name=RHEL6U3-ScalableFileSystem 
        
         baseurl= 
         file 
         : 
         ///media/cdrom/ScalableFileSystem 
        
         enabled=1 
        
         gpgcheck=0 
        
         [root@manager ~] 
         # for i in {1..6}; do scp /etc/yum.repos.d/rhel-gfs2.repo  192.168.100.17$i:/etc/yum.repos.d ; done 
        
         rhel-gfs2.repo                                                                  100%  588     0.6KB 
         /s    
         00:00 
        
         rhel-gfs2.repo                                                                  100%  588     0.6KB 
         /s    
         00:00 
        
         rhel-gfs2.repo                                                                  100%  588     0.6KB 
         /s    
         00:00 
        
         rhel-gfs2.repo                                                                  100%  588     0.6KB 
         /s    
         00:00 
        
         rhel-gfs2.repo                                                                  100%  588     0.6KB 
         /s    
         00:00 
        
         rhel-gfs2.repo                                                                  100%  588     0.6KB 
         /s    
         00:00 
        
         [root@manager ~] 
         # for i in {1..6}; do ssh 192.168.100.17$i "yum clean all && yum makecache"; done 
        
         Loaded plugins: product- 
         id 
         , security, subscription-manager 
        
         Updating certificate-based repositories. 
        
         Unable to  
         read  
         consumer identity 
        
         ……

4）、时间要同步，能够考虑配置NTP时间服务器，若是联网能够考虑同步互联网时间，固然也能够经过date命令设置相同时间。

二、安装luci和ricci（管理端安装luci，节点安装ricci）

Luci是运行WEB样式的Conga服务器端，它能够经过web界面很容易管理整个RHCS集群，每一步操做都会在/etc/cluster/cluster.conf生成相应的配置信息

 
         [root@manager ~] 
         # yum install luci –y 
        
         [root@manager ~] 
         # /etc/rc.d/init.d/luci start \\生成如下信息，说明配置成功，注意：安装luci会安装不少python包，python包尽可能采用光盘自带的包，不然启动luci会出现报错现象。 
        
         Adding following auto-detected host IDs (IP addresses 
         /domain  
         names), corresponding to `manager.rsyslog.org 
         ' address, to the configuration of self-managed certificate `/var/lib/luci/etc/cacert.config'  
         (you can change them by editing ` 
         /var/lib/luci/etc/cacert 
         .config 
         ', removing the generated certificate `/var/lib/luci/certs/host.pem'  
         and restarting luci): 
        
         (none suitable found, you can still  
         do  
         it manually as mentioned above) 
        
         Generating a 2048 bit RSA private key 
        
         writing new private key to  
         '/var/lib/luci/certs/host.pem' 
        
         正在启动 saslauthd：                                       [肯定] 
        
         Start luci...                                              [肯定] 
        
         Point your web browser to https: 
         //manager 
         .rsyslog.org:8084 (or equivalent) to access luci 
        
         [root@manager ~] 
         # for i in {1..4}; do ssh node$i "yum install ricci -y"; done 
        
         [root@manager ~] 
         # for i in {1..4}; do ssh node$i "chkconfig ricci on && /etc/rc.d/init.d/ricci start"; done 
        
         [root@manager ~] 
         # for i in {1..4}; do ssh node$i "echo '123.com' | passwd ricci --stdin"; done  \\ricci设置密码，在Conga web页面添加节点的时候须要输入ricci密码。 
        
         更改用户 ricci 的密码 。 
        
         passwd 
         ： 全部的身份验证令牌已经成功更新。 
        
         ……

三、经过luci web管理界面安装RHCS集群套件

https://manager.rsyslog.org:8084或者https://192.168.100.102:8084

添加节点node1-node3，先设置3个，后期在增长一个节点，password为各节点ricci的密码，而后勾选“Download Packages”（在各节点yum配置好的基础上，自动安装cman和rgmanager及相关的依赖包），勾选“Enable Shared Storage Support”，安装存储相关的包，并支持gfs2文件系统、DLM锁、clvm逻辑卷等。

安装过程以下：

如下为安装完成以后，全部节点状态

点开一个节点，能够看到这个节点上全部相关服务都处于运行状态。

登陆任意一个节点查看各服务的开机启动状况，为2-5级别自动启动。

 
         [root@manager ~] 
         # ssh node1 "chkconfig --list | grep cman" 
        
         cman            0:关闭    1:关闭    2:启用    3:启用    4:启用    5:启用    6:关闭 
        
         [root@manager ~] 
         # ssh node1 "chkconfig --list | grep rgmanager" 
        
         rgmanager       0:关闭    1:关闭    2:启用    3:启用    4:启用    5:启用    6:关闭 
        
         [root@manager ~] 
         # ssh node1 "chkconfig --list | grep clvmd" 
        
         clvmd           0:关闭    1:关闭    2:启用    3:启用    4:启用    5:启用    6:关闭 
        
         [root@node2 ~] 
         # cat /etc/cluster/cluster.conf   \\查看各节点集群配置信息，各节点这部分必须同样。 
        
         <?xml version= 
         "1.0" 
         ?> 
        
         <cluster config_version= 
         "1"  
         name= 
         "rsyslog" 
         > 
        
         <clusternodes> 
        
         <clusternode name= 
         "node1"  
         nodeid= 
         "1" 
         /> 
        
         <clusternode name= 
         "node2"  
         nodeid= 
         "2" 
         /> 
        
         < 
         /clusternodes 
         > 
        
         <cman expected_votes= 
         "1"  
         two_node= 
         "1" 
         /> 
        
         <fencedevices/> 
        
         < 
         rm 
         /> 
        
         < 
         /cluster 
         > 
        
         [root@node2 ~] 
         # clustat  \\查看集群节点状态(能够经过 cluster -i 1 动态查看变化状态) 
        
         Cluster Status  
         for  
         rsyslog @ Sat Jun  8 00:03:40 2013 
        
         Member Status: Quorate 
        
         Member Name                                              ID   Status 
        
         ------ ----                                              ---- ------ 
        
         node1                                                        1 Online 
        
         node2                                                        2 Online, Local 
        
         node3                                                        3 Online

四、安装存储管理管理软件，并导出磁

 
         [root@storage1 ~] 
         # fdisk /dev/sda  \\建立一个大小为2G的逻辑分区并导出 
        
         Command (m  
         for  
         help): n 
        
         Command action 
        
         e   extended 
        
         p   primary partition (1-4) 
        
         e 
        
         Selected partition 4 
        
         First cylinder (1562-2610, default 1562): 
        
         Using default value 1562 
        
         Last cylinder, +cylinders or +size{K,M,G} (1562-2610, default 2610): +4G 
        
         Command (m  
         for  
         help): n 
        
         First cylinder (1562-2084, default 1562): 
        
         Using default value 1562 
        
         Last cylinder, +cylinders or +size{K,M,G} (1562-2084, default 2084): +2G 
        
         Command (m  
         for  
         help): w 
        
         …… 
        
         [root@storage1 ~] 
         # partx -a /dev/sda 
        
         [root@storage1 ~] 
         # ll /dev/sda 
        
         sda   sda1  sda2  sda3  sda4  sda5 
        
         [root@storage1 ~] 
         # yum install scsi-target-utils –y  \\安装target管理端 
        
         [root@storage1 ~] 
         # vim /etc/tgt/targets.conf \\配置导出磁盘的信息 
        
         <target iqn.2013.05.org.rsyslog:storage1.sda5> 
        
         <backing-store  
         /dev/sda5 
         > 
        
         scsi_id storage1_id 
        
         scsi_sn storage1_sn 
        
         < 
         /backing-store 
         > 
        
         incominguser xiaonuo 081ac67e74a6bb13b7a22b8a89e7177b \\设置用户名及密码访问 
        
         initiator-address 192.168.100.173  \\设置容许的IP地址 
        
         initiator-address 192.168.100.174 
        
         initiator-address 192.168.100.175 
        
         initiator-address 192.168.100.176 
        
         initiator-address 192.168.200.173 
        
         initiator-address 192.168.200.174 
        
         initiator-address 192.168.200.175 
        
         initiator-address 192.168.200.176 
        
         < 
         /target 
         > 
        
         [root@storage1 ~] 
         # /etc/rc.d/init.d/tgtd start  && chkconfig tgtd on 
        
         [root@storage1 ~] 
         # tgtadm --lld iscsi --mode target --op show  \\查看是否导出成功 
        
         Target 1: iqn.2013.05.org.rsyslog:storage1.sda5 
        
         …… 
        
         LUN: 1 
        
         Type: disk 
        
         SCSI ID: storage1_id 
        
         SCSI SN: storage1_sn 
        
         Size: 2151 MB, Block size: 512 
        
         Online: Yes 
        
         Removable media: No 
        
         Prevent removal: No 
        
         Readonly: No 
        
         Backing store  
         type 
         : rdwr 
        
         Backing store path:  
         /dev/sda5 
        
         Backing store flags: 
        
         Account information: 
        
         xiaonuo 
        
         ACL information: 
        
         192.168.100.173 
        
         192.168.100.174 
        
         192.168.100.175 
        
         192.168.100.176 
        
         192.168.200.173 
        
         192.168.200.174 
        
         192.168.200.175 
        
         192.168.200.176 
        
         [root@manager ~] 
         # for i in {1..3}; do ssh node$i "yum -y install iscsi-initiator-utils"; done \\节点安装iscsi客户端软件 
        
         [root@node1 ~] 
         # vim /etc/iscsi/iscsid.conf  \\全部节点配置文件加上如下3行，设置帐户密码 
        
         node.session.auth.authmethod = CHAP 
        
         node.session.auth.username = xiaonuo 
        
         node.session.auth.password = 081ac67e74a6bb13b7a22b8a89e7177b 
        
         [root@manager ~] 
         # for i in {1..3}; do ssh node$i "iscsiadm -m discovery -t st -p 192.168.100.171"; done \\发现共享设备 
        
         192.168.100.171:3260,1 iqn.2013.05.org.rsyslog:storage1.sda5 
        
         192.168.100.171:3260,1 iqn.2013.05.org.rsyslog:storage1.sda5 
        
         192.168.100.171:3260,1 iqn.2013.05.org.rsyslog:storage1.sda5 
        
         [root@manager ~] 
         # for i in {1..3}; do ssh node$i "iscsiadm -m discovery -t st -p 192.168.200.171"; done 
        
         192.168.200.171:3260,1 iqn.2013.05.org.rsyslog:storage1.sda5 
        
         192.168.200.171:3260,1 iqn.2013.05.org.rsyslog:storage1.sda5 
        
         192.168.200.171:3260,1 iqn.2013.05.org.rsyslog:storage1.sda5 
        
         [root@manager ~] 
         # for i in {1..3}; do ssh node$i "iscsiadm -m node -l"; done \\注册iscsi共享设备 
        
         Logging  
         in  
         to [iface: default, target: iqn.2013.05.org.rsyslog:storage1.sda5, portal: 192.168.200.171,3260] (multiple) 
        
         Logging  
         in  
         to [iface: default, target: iqn.2013.05.org.rsyslog:storage1.sda5, portal: 192.168.100.171,3260] (multiple) 
        
         Login to [iface: default, target: iqn.2013.05.org.rsyslog:storage1.sda5, portal: 192.168.200.171,3260] successful. 
        
         Login to [iface: default, target: iqn.2013.05.org.rsyslog:storage1.sda5, portal: 192.168.100.171,3260] successful. 
        
         …… 
        
         [root@storage1 ~] 
         # tgtadm --lld  iscsi --op show --mode conn --tid 1 \\iscsi服务器端查看共享状况 
        
         Session: 12 
        
         Connection: 0 
        
         Initiator: iqn.1994-05.com.redhat:a12e282371a1 
        
         IP Address: 192.168.200.175 
        
         Session: 11 
        
         Connection: 0 
        
         Initiator: iqn.1994-05.com.redhat:a12e282371a1 
        
         IP Address: 192.168.100.175 
        
         ……. 
        
         [root@node1 ~] 
         # netstat -nlatp | grep 3260 
        
         tcp        0      0 192.168.200.173:37946       192.168.200.171:3260        ESTABLISHED 37565 
         /iscsid 
        
         tcp        0      0 192.168.100.173:54306       192.168.100.171:3260        ESTABLISHED 37565 
         /iscsid 
        
         [root@node1 ~] 
         # ll /dev/sd   \\在各个节点上面都会多出两个iscsi设备 
        
         sda   sda1  sda2  sda3  sdb   sdc

五、安装配置multipath多路冗余实现线路冗余

 
         [root@manager ~] 
         # for i in {1..3}; do ssh node$i "yum -y install device-mapper-*"; done 
        
         [root@manager ~] 
         # for i in {1..3}; do ssh node$i "mpathconf --enable"; done \\生成配置文件 
        
         [root@node1 ~] 
         # /sbin/scsi_id -g -u /dev/sdb \\查看导入设备的WWID 
        
         1storage1_id 
        
         [root@node1 ~] 
         # /sbin/scsi_id -g -u /dev/sdc 
        
         1storage1_id 
        
         [root@node1 ~] 
         # vim /etc/multipath.conf 
        
         multipaths { 
        
         multipath { 
        
         wwid                    1storage1_id  \\设置导出设备的WWID 
        
         alias                    
         iscsi1 \\设置别名 
        
         path_grouping_policy    multibus 
        
         path_selector            
         "round-robin 0" 
        
         failback                manual 
        
         rr_weight               priorities 
        
         no_path_retry           5 
        
         } 
        
         } 
        
         [root@node1 ~] 
         # /etc/rc.d/init.d/multipathd start 
        
         Starting multipathd daemon:                                [  OK  ] 
        
         [root@node1 ~] 
         # ll /dev/mapper/iscsi1 
        
         lrwxrwxrwx 1 root root 7 Jun  7 23:58  
         /dev/mapper/iscsi1  
         -> .. 
         /dm-0 
        
         [root@node1 ~] 
         # multipath –ll  \\查看绑定是否成功 
        
         iscsi1 (1storage1_id) dm-0 IET,VIRTUAL-DISK 
        
         size=2.0G features= 
         '1 queue_if_no_path'  
         hwhandler= 
         '0'  
         wp=rw 
        
         `-+- policy= 
         'round-robin 0'  
         prio=1 status=active 
        
         |- 20:0:0:1 sdb 8:16 active ready running 
        
         `- 19:0:0:1 sdc 8:32 active ready running 
        
         ……\\其余两个节点同上

六、在节点上建立clvm逻辑卷并建立gfs2集群文件系统

 
         [root@node1 ~] 
         # pvcreate /dev/mapper/iscsi1  \\将多路冗余设备建立成pv 
        
         Writing physical volume data to disk  
         "/dev/mapper/iscsi1" 
        
         Physical volume  
         "/dev/mapper/iscsi1"  
         successfully created 
        
         [root@node1 ~] 
         # vgcreate cvg0 /dev/mapper/iscsi1 \\建立vg 
        
         Clustered volume group  
         "cvg0"  
         successfully created 
        
         [root@node1 ~] 
         # lvcreate -L +1G cvg0 -n clv0 \\建立大小为1G的lv 
        
         Logical volume  
         "clv0"  
         created 
        
         [root@node1 ~] 
         # lvs  \\从node1查看lv状况 
        
         LV   VG   Attr     LSize Pool Origin Data%  Move Log Copy%  Convert 
        
         clv0 cvg0 -wi-a--- 1.00g 
        
         [root@node2 ~] 
         # lvs  \\从node2查看lv状况 
        
         LV   VG   Attr     LSize Pool Origin Data%  Move Log Copy%  Convert 
        
         clv0 cvg0 -wi-a--- 1.00g 
        
         [root@manager ~] 
         # for i in {1..3}; do ssh node$i "lvmconf --enable-cluster"; done \\打开DLM锁机制，在web配置时候，若是勾选了“Enable Shared Storage Support”，则默认就打开了。 
        
         [root@node2 ~] 
         # mkfs.gfs2 -j 3 -p lock_dlm -t rsyslog:web /dev/cvg0/clv0  \\建立gfs2集群文件系统，并设置节点为3个，锁协议为lock_dlm 
        
         This will destroy any data on  
         /dev/cvg0/clv0 
         . 
        
         It appears to contain: symbolic link to `.. 
         /dm-1 
         ' 
        
         Are you sure you want to proceed? [y 
         /n 
         ] y 
        
         Device:                     
         /dev/cvg0/clv0 
        
         Blocksize:                 4096 
        
         Device Size                1.00 GB (262144 blocks) 
        
         Filesystem Size:           1.00 GB (262142 blocks) 
        
         Journals:                  3 
        
         Resource Groups:           4 
        
         Locking Protocol:           
         "lock_dlm" 
        
         Lock Table:                 
         "rsyslog:web" 
        
         UUID:                      7c293387-b59a-1105-cb26-4ffc41b5ae3b

七、在storage2上建立storage1的mirror，实现备份及高可用

1)、建立跟storage1同样大的iscs空间2G，并配置targets.conf

 
         [root@storage2 ~] 
         # vim /etc/tgt/targets.conf 
        
         <target iqn.2013.05.org.rsyslog:storage2.sda5> 
        
         <backing-store  
         /dev/sda5 
         > 
        
         scsi_id storage2_id 
        
         scsi_sn storage2_sn 
        
         < 
         /backing-store 
         > 
        
         incominguser xiaonuo 081ac67e74a6bb13b7a22b8a89e7177b 
        
         initiator-address 192.168.100.173 
        
         initiator-address 192.168.100.174 
        
         initiator-address 192.168.100.175 
        
         initiator-address 192.168.100.176 
        
         initiator-address 192.168.200.173 
        
         initiator-address 192.168.200.174 
        
         initiator-address 192.168.200.175 
        
         initiator-address 192.168.200.176 
        
         < 
         /target 
         >

2）、各节点导入storage1设备

 
         [root@manager ~] 
         # for i in {1..3}; do ssh node$i "iscsiadm -m discovery -t st -p 192.168.100.172"; done 
        
         192.168.100.172:3260,1 iqn.2013.05.org.rsyslog:storage2.sda5 
        
         192.168.100.172:3260,1 iqn.2013.05.org.rsyslog:storage2.sda5 
        
         192.168.100.172:3260,1 iqn.2013.05.org.rsyslog:storage2.sda5 
        
         [root@manager ~] 
         # for i in {1..3}; do ssh node$i "iscsiadm -m discovery -t st -p 192.168.200.172"; done 
        
         192.168.200.172:3260,1 iqn.2013.05.org.rsyslog:storage2.sda5 
        
         192.168.200.172:3260,1 iqn.2013.05.org.rsyslog:storage2.sda5 
        
         192.168.200.172:3260,1 iqn.2013.05.org.rsyslog:storage2.sda5 
        
         [root@manager ~] 
         # for i in {1..3}; do ssh node$i "iscsiadm -m node -l"; done 
        
         Logging  
         in  
         to [iface: default, target: iqn.2013.05.org.rsyslog:storage2.sda5, portal: 192.168.100.172,3260] (multiple) 
        
         Logging  
         in  
         to [iface: default, target: iqn.2013.05.org.rsyslog:storage2.sda5, portal: 192.168.200.172,3260] (multiple) 
        
         Login to [iface: default, target: iqn.2013.05.org.rsyslog:storage2.sda5, portal: 192.168.100.172,3260] successful. 
        
         Login to [iface: default, target: iqn.2013.05.org.rsyslog:storage2.sda5, portal: 192.168.200.172,3260] successful. 
        
         ……

3）、设置multipath

 
         [root@node1 ~] 
         # ll /dev/sd 
        
         sda   sda1  sda2  sda3  sdb   sdc   sdd   sde 
        
         [root@node1 ~] 
         # /sbin/scsi_id -g -u /dev/sdd 
        
         1storage2_id 
        
         [root@node1 ~] 
         # /sbin/scsi_id -g -u /dev/sde 
        
         1storage2_id 
        
         [root@node1 ~] 
         # vim /etc/multipath.conf   \\其它两个节点配置类同 
        
         multipaths { 
        
         multipath { 
        
         wwid                    1storage1_id 
        
         alias                    
         iscsi1 
        
         path_grouping_policy    multibus 
        
         path_selector            
         "round-robin 0" 
        
         failback                manual 
        
         rr_weight               priorities 
        
         no_path_retry           5 
        
         } 
        
         multipath { 
        
         wwid                    1storage2_id 
        
         alias                    
         iscsi2 
        
         path_grouping_policy    multibus 
        
         path_selector            
         "round-robin 0" 
        
         failback                manual 
        
         rr_weight               priorities 
        
         no_path_retry           5 
        
         } 
        
         } 
        
         [root@node1 ~] 
         # /etc/rc.d/init.d/multipathd reload 
        
         Reloading multipathd:                                      [  OK  ] 
        
         [root@node1 ~] 
         # multipath -ll 
        
         iscsi2 (1storage2_id) dm-2 IET,VIRTUAL-DISK 
        
         size=2.0G features= 
         '1 queue_if_no_path'  
         hwhandler= 
         '0'  
         wp=rw 
        
         |-+- policy= 
         'round-robin 0'  
         prio=1 status=active 
        
         | `- 21:0:0:1 sde 8:64 active ready running 
        
         `-+- policy= 
         'round-robin 0'  
         prio=1 status=enabled 
        
         `- 22:0:0:1 sdd 8:48 active ready running 
        
         iscsi1 (1storage1_id) dm-0 IET,VIRTUAL-DISK 
        
         size=2.0G features= 
         '1 queue_if_no_path'  
         hwhandler= 
         '0'  
         wp=rw 
        
         `-+- policy= 
         'round-robin 0'  
         prio=1 status=active 
        
         |- 20:0:0:1 sdb 8:16 active ready running 
        
         `- 19:0:0:1 sdc 8:32 active ready running 
        
         4）、将新的iscsi设备加入卷组cvg0。 
        
         [root@node3 ~] 
         # pvcreate /dev/mapper/iscsi2 
        
         Writing physical volume data to disk  
         "/dev/mapper/iscsi2" 
        
         Physical volume  
         "/dev/mapper/iscsi2"  
         successfully created 
        
         [root@node3 ~] 
         # vgextend cvg0 /dev/mapper/iscsi2 
        
         Volume group  
         "cvg0"  
         successfully extended 
        
         [root@node3 ~] 
         # vgs 
        
         VG    
         #PV #LV #SN Attr   VSize VFree 
        
         cvg0   2   1   0 wz--nc 4.00g 3.00g

5）、安装cmirror，并在节点建立stoarge1的mirror为stoarage2

 
         [root@manager ~] 
         # for i in {1..3}; do ssh node$i "yum install cmirror -y"; done 
        
         [root@manager ~] 
         # for i in {1..3}; do ssh node$i "/etc/rc.d/init.d/cmirrord start && chkconfig cmirrord on"; done 
        
         [root@node3 ~] 
         # dmsetup ls –tree  \\没有建立mirror以前的情况 
        
         iscsi2 (253:2) 
        
         ├─ (8:48) 
        
         └─ (8:64) 
        
         cvg0-clv0 (253:1) 
        
         └─iscsi1 (253:0) 
        
         ├─ (8:32) 
        
         └─ (8:16) 
        
         [root@node3 ~] 
         # lvs  \\没有建立mirror以前的情况 
        
         LV   VG   Attr     LSize Pool Origin Data%  Move Log Copy%  Convert 
        
         clv0 cvg0 -wi-a--- 1.00g 
        
         [root@node3 ~] 
         # lvconvert -m 1 /dev/cvg0/clv0 /dev/mapper/iscsi1 /dev/mapper/iscsi2 \\建立先有lv的mirror，如下能够看到数据在复制 
        
         cvg0 
         /clv0 
         : Converted: 0.4% 
        
         cvg0 
         /clv0 
         : Converted: 10.9% 
        
         cvg0 
         /clv0 
         : Converted: 18.4% 
        
         cvg0 
         /clv0 
         : Converted: 28.1% 
        
         cvg0 
         /clv0 
         : Converted: 42.6% 
        
         cvg0 
         /clv0 
         : Converted: 56.6% 
        
         cvg0 
         /clv0 
         : Converted: 70.3% 
        
         cvg0 
         /clv0 
         : Converted: 85.9% 
        
         cvg0 
         /clv0 
         : Converted: 100.0% 
        
         [root@node2 ~] 
         # lvs \\建立过程当中，storage1中的clv0正在想storage2复制相同内容 
        
         LV   VG   Attr     LSize Pool Origin Data%  Move Log       Copy%  Convert 
        
         clv0 cvg0 mwi-a-m- 1.00g                         clv0_mlog   6.64 
        
         [root@node2 ~] 
         # lvs 
        
         LV   VG   Attr     LSize Pool Origin Data%  Move Log       Copy%  Convert 
        
         clv0 cvg0 mwi-a-m- 1.00g                         clv0_mlog 100.00 
        
         [root@node3 ~] 
         # dmsetup ls –tree \\查看现有iscsi导出设备的状态为mirror型 
        
         cvg0-clv0 (253:1) 
        
         ├─cvg0-clv0_mimage_1 (253:5) 
        
         │  └─iscsi2 (253:2) 
        
         │     ├─ (8:48) 
        
         │     └─ (8:64) 
        
         ├─cvg0-clv0_mimage_0 (253:4) 
        
         │  └─iscsi1 (253:0) 
        
         │     ├─ (8:32) 
        
         │     └─ (8:16) 
        
         └─cvg0-clv0_mlog (253:3) 
        
         └─iscsi2 (253:2) 
        
         ├─ (8:48) 
        
         └─ (8:64)

八、集群管理

1）、当基于clvm的gfs2文件系统不够用时，如何增长

 
         [root@node3 ~] 
         # lvextend -L +200M /dev/cvg0/clv0 
        
         Extending 2 mirror images. 
        
         Extending logical volume clv0 to 1.20 GiB 
        
         Logical volume clv0 successfully resized 
        
         [root@node3 ~] 
         # gfs2_grow /opt \\同步文件系统 
        
         Error: The device has grown by  
         less  
         than one Resource Group (RG). 
        
         The device grew by 200MB.  One RG is 255MB  
         for  
         this  
         file  
         system. 
        
         gfs2_grow complete. 
        
         [root@node3 ~] 
         # lvs 
        
         LV   VG   Attr     LSize Pool Origin Data%  Move Log       Copy%  Convert 
        
         clv0 cvg0 mwi-aom- 1.20g                         clv0_mlog  96.08

2）、当节点不够用时，若是添加一个新的节点加入集群

步骤以下：

1>、安装ricci

 
         [root@node4 ~] 
         # yum install ricci -y

2>、登陆luci web，添加ricci

3>、导入共享存储设备

 
         [root@node4 ~] 
         # iscsiadm -m discovery -t st -p 192.168.100.171 
        
         [root@node4 ~] 
         # iscsiadm -m discovery -t st -p 192.168.100.172 
        
         [root@node4 ~] 
         # iscsiadm -m discovery -t st -p 192.168.200.172 
        
         [root@node4 ~] 
         # iscsiadm -m discovery -t st -p 192.168.200.171 
        
         [root@node2 ~] 
         # scp /etc/iscsi/iscsid.conf node4:/etc/iscsi/ 
        
         [root@node4 ~] 
         # iscsiadm -m node –l

4>、设置multipath

 
         [root@node4 ~] 
         # yum -y install device-mapper-* 
        
         [root@node4 ~] 
         # mpathconf –enable  \\生成配置文件 
        
         [root@node4 ~] 
         # scp node1:/etc/multipath.conf /etc/ \\也能够直接从其余节点复制过来直接使用 
        
         [root@node4 ~] 
         # /etc/rc.d/init.d/multipathd start 
        
         [root@node4 ~] 
         # lvs 
        
         LV   VG   Attr     LSize Pool Origin Data%  Move Log       Copy%  Convert 
        
         clv0 cvg0 mwi---m- 1.20g                         clv0_mlog 
        
         [root@node4 ~] 
         # multipath -ll 
        
         iscsi2 (1storage2_id) dm-0 IET,VIRTUAL-DISK 
        
         size=2.0G features= 
         '1 queue_if_no_path'  
         hwhandler= 
         '0'  
         wp=rw 
        
         `-+- policy= 
         'round-robin 0'  
         prio=1 status=active 
        
         |- 11:0:0:1 sdb 8:16 active ready running 
        
         `- 12:0:0:1 sdd 8:48 active ready running 
        
         iscsi1 (1storage1_id) dm-1 IET,VIRTUAL-DISK 
        
         size=2.0G features= 
         '1 queue_if_no_path'  
         hwhandler= 
         '0'  
         wp=rw 
        
         `-+- policy= 
         'round-robin 0'  
         prio=1 status=active 
        
         |- 13:0:0:1 sde 8:64 active ready running 
        
         `- 14:0:0:1 sdc 8:32 active ready running 
        
         5>、安装cmirror，支持mirror 
        
         [root@node4 ~] 
         # yum install cmirror –y 
        
         [root@node4 ~] 
         # /etc/rc.d/init.d/cmirrord start && chkconfig cmirrord on

6>、在已成功挂载的节点上增长节点数，并实现挂载使用（注意：若是系统看不到/dev/cvg0/clv0，或者经过lvs看不到mirror的copy或者经过dmsetup ls –tree查看不到mirror结构，则从新启动节点系统便可生效）

 
         [root@node4 ~] 
         # mount /dev/cvg0/clv0 /opt/  \\节点数不够 
        
         Too many nodes mounting filesystem, no  
         free  
         journals 
        
         [root@node2 ~] 
         # gfs2_jadd -j 1 /opt  \\增长一个节点数 
        
         Filesystem:             
         /opt 
        
         Old Journals           3 
        
         New Journals           4 
        
         [root@node4 ~] 
         # mount /dev/cvg0/clv0 /opt/ 
        
         [root@node4 ~] 
         # ll /opt/ 
        
         total 4 
        
         -rw-r--r-- 1 root root 210 Jun  8 00:42  
         test 
         .txt 
        
         [root@node4 ~] 
         #

一、总体测试

1）、测试多路冗余是否OK

 
         [root@node2 ~] 
         # ifdown eth1 \\关闭某一个网卡，模拟单线路故障 
        
         [root@node2 ~] 
         # multipath -ll 
        
         iscsi2 (1storage2_id) dm-1 IET,VIRTUAL-DISK 
        
         size=2.0G features= 
         '1 queue_if_no_path'  
         hwhandler= 
         '0'  
         wp=rw 
        
         `-+- policy= 
         'round-robin 0'  
         prio=1 status=active 
        
         |- 4:0:0:1 sde 8:64 failed faulty running  \\导出设备故障 
        
         `- 3:0:0:1 sdd 8:48 active ready  running 
        
         iscsi1 (1storage1_id) dm-0 IET,VIRTUAL-DISK 
        
         size=2.0G features= 
         '1 queue_if_no_path'  
         hwhandler= 
         '0'  
         wp=rw 
        
         `-+- policy= 
         'round-robin 0'  
         prio=1 status=active 
        
         |- 6:0:0:1 sdc 8:32 active ready  running 
        
         `- 5:0:0:1 sdb 8:16 failed faulty running \\导出设备故障 
        
         [root@node2 opt] 
         # mount | grep opt 
        
         /dev/mapper/cvg0-clv0  
         on  
         /opt  
         type  
         gfs2 (rw,relatime,hostdata=jid=0) 
        
         [root@node2 opt] 
         # touch test  \\单线路故障并不影响集群文件系统正常使用 
        
         [root@node2 ~] 
         # ifup eth1 \\恢复网卡 
        
         [root@node2 opt] 
         # multipath –ll \\查看多路冗余是否恢复 
        
         iscsi2 (1storage2_id) dm-1 IET,VIRTUAL-DISK 
        
         size=2.0G features= 
         '1 queue_if_no_path'  
         hwhandler= 
         '0'  
         wp=rw 
        
         `-+- policy= 
         'round-robin 0'  
         prio=1 status=active 
        
         |- 4:0:0:1 sde 8:64 active ready running 
        
         `- 3:0:0:1 sdd 8:48 active ready running 
        
         iscsi1 (1storage1_id) dm-0 IET,VIRTUAL-DISK 
        
         size=2.0G features= 
         '1 queue_if_no_path'  
         hwhandler= 
         '0'  
         wp=rw 
        
         `-+- policy= 
         'round-robin 0'  
         prio=1 status=active 
        
         |- 6:0:0:1 sdc 8:32 active ready running 
        
         `- 5:0:0:1 sdb 8:16 active ready running

2）、测试基于gfs2文件系统的集群节点是否支持同时读写操做

 
         [root@manager ~] 
         # for i in {1..3}; do ssh node$i "mount /dev/cvg0/clv0 /opt"; done 
        
         [root@node1 ~] 
         # while :; do echo node1 >>/opt/test.txt;sleep 1; done \\节点1模拟向test.txt文件写入node1 
        
         [root@node2 ~] 
         # while :; do echo node2 >>/opt/test.txt;sleep 1; done \\节点2模拟向test.txt文件写入node1 
        
         [root@node3 ~] 
         # tail -f /opt/test.txt  \\节点3模拟读出节点1和节点2同时写入的数据 
        
         node1 
        
         node2 
        
         node1 
        
         node2 
        
         ………

3）、测试Storage损坏一个是否可以正常工做

 
         [root@node1 ~] 
         # lvs \\mirror正常状况下的lv 
        
         LV   VG   Attr     LSize Pool Origin Data%  Move Log       Copy%  Convert 
        
         clv0 cvg0 mwi-a-m- 1.20g                         clv0_mlog 100.00 
        
         [root@storage1 ~] 
         # ifdown eth1 && ifdown eth0 \\关闭storage1的两块网卡，至关于storage1宕机 
        
         [root@node2 opt] 
         # lvs  \\ \mirror在storage1宕机正常状况下的lv 
        
         /dev/mapper/iscsi1 
         :  
         read  
         failed after 0 of 4096 at 2150563840: Input 
         /output  
         error 
        
         /dev/mapper/iscsi1 
         :  
         read  
         failed after 0 of 4096 at 2150637568: Input 
         /output  
         error 
        
         /dev/mapper/iscsi1 
         :  
         read  
         failed after 0 of 4096 at 0: Input 
         /output  
         error 
        
         /dev/mapper/iscsi1 
         :  
         read  
         failed after 0 of 4096 at 4096: Input 
         /output  
         error 
        
         /dev/sdb 
         :  
         read  
         failed after 0 of 4096 at 0: Input 
         /output  
         error 
        
         /dev/sdb 
         :  
         read  
         failed after 0 of 4096 at 2150563840: Input 
         /output  
         error 
        
         /dev/sdb 
         :  
         read  
         failed after 0 of 4096 at 2150637568: Input 
         /output  
         error 
        
         /dev/sdb 
         :  
         read  
         failed after 0 of 4096 at 4096: Input 
         /output  
         error 
        
         /dev/sdc 
         :  
         read  
         failed after 0 of 4096 at 0: Input 
         /output  
         error 
        
         /dev/sdc 
         :  
         read  
         failed after 0 of 4096 at 2150563840: Input 
         /output  
         error 
        
         /dev/sdc 
         :  
         read  
         failed after 0 of 4096 at 2150637568: Input 
         /output  
         error 
        
         /dev/sdc 
         :  
         read  
         failed after 0 of 4096 at 4096: Input 
         /output  
         error 
        
         Couldn't  
         find  
         device with uuid ziwJmg-Si56-l742-R3Nx-h0rK-KggJ-NdCigs. 
        
         LV   VG   Attr     LSize Pool Origin Data%  Move Log Copy%  Convert 
        
         clv0 cvg0 -wi-ao-- 1.20g 
        
         [root@node2 opt] 
         # cp /var/log/messages .\\copy数据到挂载的目录，发现存储宕机一个并不影响读取和写入。 
        
         [root@node2 opt] 
         # ll messages 
        
         -rw------- 1 root root 1988955 Jun  8 18:08 messages 
        
         [root@node2 opt] 
         # dmsetup ls --tree 
        
         cvg0-clv0 (253:5) 
        
         └─iscsi2 (253:1) 
        
         ├─ (8:48) 
        
         └─ (8:64) 
        
         iscsi1 (253:0) 
        
         ├─ (8:16) 
        
         └─ (8:32) 
        
         [root@node2 opt] 
         # vgs \\查看vgs状况 
        
         WARNING: Inconsistent metadata found  
         for  
         VG cvg0 - updating to use version 11 
        
         Missing device  
         /dev/mapper/iscsi1  
         reappeared, updating metadata  
         for  
         VG cvg0 to version 11. 
        
         VG    
         #PV #LV #SN Attr   VSize VFree 
        
         cvg0   2   1   0 wz--nc 4.00g 2.80g 
        
         [root@node2 opt] 
         # lvconvert -m 1 /dev/cvg0/clv0 /dev/mapper/iscsi1 \\恢复mirror 
        
         cvg0 
         /clv0 
         : Converted: 0.0% 
        
         cvg0 
         /clv0 
         : Converted: 8.5% 
        
         [root@node1 ~] 
         # lvs 
        
         LV   VG   Attr     LSize Pool Origin Data%  Move Log       Copy%  Convert 
        
         clv0 cvg0 mwi-a-m- 1.20g                         clv0_mlog  77.45 
        
         [root@node1 ~] 
         # lvs 
        
         LV   VG   Attr     LSize Pool Origin Data%  Move Log       Copy%  Convert 
        
         clv0 cvg0 mwi-a-m- 1.20g                         clv0_mlog  82.35 
        
         [root@node1 ~] 
         # dmsetup ls --tree 
        
         cvg0-clv0 (253:5) 
        
         ├─cvg0-clv0_mimage_1 (253:4) 
        
         │  └─iscsi1 (253:1) 
        
         │     ├─ (8:64) 
        
         │     └─ (8:48) 
        
         ├─cvg0-clv0_mimage_0 (253:3) 
        
         │  └─iscsi2 (253:0) 
        
         │     ├─ (8:16) 
        
         │     └─ (8:32) 
        
         └─cvg0-clv0_mlog (253:2) 
        
         └─iscsi1 (253:1) 
        
         ├─ (8:64) 
        
         └─ (8:48) 
        
         [root@node1 ~] 
         # ll /opt/messages  \\能够看到数据还在 
        
         -rw------- 1 root root 1988955 Jun  8 18:08  
         /opt/messages