第九章·Logstash深刻-Logstash配合rsyslog收集haproxy日志

rsyslog介绍及安装配置

在centos 6及以前的版本叫作syslog，centos 7开始叫作rsyslog，根据官方的介绍，rsyslog(2013年版本)能够达到每秒转发百万条日志的级别，官方网址：http://www.rsyslog.com/

---

安装配置rsyslog

#安装rsyslog [root@elkstack03 ~]# yum install -y rsyslog #编辑rsyslog配置文件 [root@elkstack03 ~]# vim /etc/rsyslog.conf $ModLoad imudp $UDPServerRun 514 $ModLoad imtcp $InputTCPServerRun 514 #最后面一行添加，local6对应haproxy配置文件定义的local级别,端口为Logstash的端口 local6.* @@10.0.0.53:2222 

---

安装配置haproxy

#安装haproxy [root@elkstack03 ~]# yum install -y haproxy #编辑haproxy配置文件 [root@elkstack03 ~]# vim /etc/haproxy/haproxy.cfg global maxconn 100000 chroot /var/lib/haproxy uid 99 gid 99 daemon nbproc 1 pidfile /var/run/haproxy.pid log 127.0.0.1 local6 info defaults option http-keep-alive option forwardfor maxconn 100000 mode http timeout connect 300000ms timeout client 300000ms timeout server 300000ms listen stats mode http bind 0.0.0.0:9999 stats enable log global stats uri /haproxy-status stats auth haadmin:123456 #frontend web_port frontend web_port bind 0.0.0.0:80 mode http option httplog log global option forwardfor ###################ACL Setting########################## acl pc hdr_dom(host) -i www.elk.com acl mobile hdr_dom(host) -i m.elk.com ###################USE ACL############################## use_backend pc_host if pc use_backend mobile_host if mobile ######################################################## backend pc_host mode http option httplog balance source server web1 10.0.0.53:8081 check inter 2000 rise 3 fall 2 weight 1 backend mobile_host mode http option httplog balance source server web1 10.0.0.53:8080 check inter 2000 rise 3 fall 2 weight 1 #启动haproxy [root@elkstack03 ~]# /etc/init.d/haproxy start 正在启动 haproxy： [肯定] #启动rsyslog [root@elkstack03 ~]# /etc/init.d/rsyslog start 启动系统日志记录器： #验证端口 [root@elkstack03 ~]# netstat -lntup tcp 0 0 0.0.0.0:9999 0.0.0.0:* LISTEN 9082/haproxy tcp 0 0 0.0.0.0:80 0.0.0.0:* LISTEN 9631/haproxy #验证进程 [root@elkstack03 ~]# ps -ef|grep haproxy nobody 9082 1 0 14:04 ? 00:00:00 /usr/sbin/haproxy -D -f /etc/haproxy/haproxy.cfg -p /var/run/haproxy.pid #修改nginx配置文件，将端口改成8081 [root@elkstack03 ~]# vim /usr/local/nginx/conf/nginx.conf worker_processes 1; events { worker_connections 1024; } http { include mime.types; default_type application/octet-stream; sendfile on; keepalive_timeout 65; log_format main '$remote_addr - $remote_user [$time_local] "$request" ' '$status $body_bytes_sent "$http_referer" ' '"$http_user_agent" "$http_x_forwarded_for"'; access_log logs/access.log main; log_format access_json '{"@timestamp":"$time_iso8601",' '"host":"$server_addr",' '"clientip":"$remote_addr",' '"size":$body_bytes_sent,' '"responsetime":$request_time,' '"upstreamtime":"$upstream_response_time",' '"upstreamhost":"$upstream_addr",' '"http_host":"$host",' '"url":"$uri",' '"domain":"$host",' '"xff":"$http_x_forwarded_for",' '"referer":"$http_referer",' '"status":"$status"}'; access_log logs/access_json.log access_json; server { listen 8081; server_name 10.0.0.53; location / { root /code/html; index index.html index.htm; } } } #修改tomcat配置文件，将默认站点目录改为/webapps/webdir [root@elkstack03 ~]# vim /usr/local/tomcat/conf/server.xml <Host name="localhost" appBase="webapps" unpackWARs="true" autoDeploy="true"> <Context path="" docBase="/usr/local/tomcat/webapps/webdir" debug="0" reloadable="false" crossContext="true"/> #重启nginx [root@elkstack03 ~]# /usr/local/nginx/sbin/nginx -t nginx: the configuration file /usr/local/nginx-1.10.3/conf/nginx.conf syntax is ok nginx: configuration file /usr/local/nginx-1.10.3/conf/nginx.conf test is successful [root@elkstack03 ~]# /usr/local/nginx/sbin/nginx -s reload #重启tomcat [root@elkstack03 ~]# cd /usr/local/tomcat/bin/ [root@elkstack03 bin]# ./catalina.sh stop [root@elkstack03 bin]# ./catalina.sh start #修改本地hosts文件 10.0.0.53 www.elk.com 10.0.0.53 m.elk.com 

测试域名访问


测试haproxy，打开浏览器，访问：http://www.elk.com/


测试haproxy，打开浏览器，访问：http://m.elk.com/


---

配置Logstash

#编辑Logstash配置文件 [root@elkstack03 conf.d]# vim haproxy.cof input{ syslog { type => "rsyslog_haproxy" port => "2222" }} output{ stdout{ codec => rubydebug }} #启动Logstash [root@elkstack03 conf.d]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/haproxy.conf #检查Logstash端口 [root@elkstack03 ~]# netstat -lntup|grep 2222 tcp 0 0 :::2222 :::* LISTEN 9867/java udp 0 0 :::2222 :::* 9867/java 

访问haproxy管理页面测试数据

打开浏览器，访问：http://10.0.0.53:9999/haproxy-status

输入haproxy配置文件中的用户名和密码
用户名：haadmin
密码：123456







---

将输出改为ES

#进入Logstash配置文件目录 [root@elkstack03 ~]# cd /etc/logstash/conf.d #编辑配置文件 [root@elkstack03 conf.d]# vim haproxy.conf input{ syslog { type => "rsyslog_haproxy" port => "2222" } } output{ elasticsearch { hosts => ["10.0.0.51:9200"] index => "logstash_rsyslog-%{+YYYY.MM.dd}" } } #启动Logstash [root@elkstack03 conf.d]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/haproxy.conf & 

打开浏览器，访问：http://10.0.0.51:9100/



---

将ES索引添加到Kibana中

打开浏览器，访问：http://10.0.0.54:5601/