ELK+Filebeat+Kafka+ZooKeeper构建大数据日志分析平台

ELK+Filebeat+Kafka+ZooKeeper构建大数据日志分析平台安装包

主机规划

设置主机名

[root@localhost ~]# hostnamectl set-hostname filebeat
[root@localhost ~]# exit
登出

Connection closed by foreign host.

Disconnected from remote host(测试机 - 203) at 09:49:26.

安装JDK环境

# 除filebeat服务器外均需安装
[root@ek01 ~]# tar zxvf jdk-8u161-linux-x64.tar.gz -C /usr/local/
[root@ek01 ~]# cd /usr/local/
[root@ek01 local]# mv jdk1.8.0_161 jdk
[root@ek01 local]# vim /etc/profile
export JAVA_HOME=/usr/local/jdk
export PATH=$PATH:$JAVA_HOME/bin
[root@ek01 local]# source /etc/profile
[root@ek01 local]# java -version
java version "1.8.0_161"
Java(TM) SE Runtime Environment (build 1.8.0_161-b12)
Java HotSpot(TM) 64-Bit Server VM (build 25.161-b12, mixed mode)

Elasticsearch

解压Elasticsearch安装包

[root@ek01 ~]# ls
anaconda-ks.cfg  elasticsearch-6.7.2.tar.gz  jdk-8u161-linux-x64.tar.gz  kibana-6.7.2-linux-x86_64.tar.gz  sysconfigure.sh
[root@ek01 ~]# tar zxvf elasticsearch-6.7.2.tar.gz -C /usr/local/
[root@ek01 ~]# cd /usr/local/
[root@ek01 local]# mv elasticsearch-6.7.2 elasticsearch

创建Elasticsearch专用用户

[root@ek01 local]# cd /usr/local/elasticsearch/
[root@ek01 elasticsearch]# useradd elasticsearch
[root@ek01 elasticsearch]# chown -R elasticsearch:elasticsearch /usr/local/elasticsearch
[root@ek01 elasticsearch]# ll
总用量 460
drwxr-xr-x  3 elasticsearch elasticsearch   4096 12月 27 16:31 bin
drwxr-xr-x  2 elasticsearch elasticsearch    148 4月  29 2019 config
drwxr-xr-x  3 elasticsearch elasticsearch   4096 4月  29 2019 lib
-rw-r--r--  1 elasticsearch elasticsearch  13675 4月  29 2019 LICENSE.txt
drwxr-xr-x  2 elasticsearch elasticsearch      6 4月  29 2019 logs
drwxr-xr-x 31 elasticsearch elasticsearch   4096 4月  29 2019 modules
-rw-r--r--  1 elasticsearch elasticsearch 427502 4月  29 2019 NOTICE.txt
drwxr-xr-x  2 elasticsearch elasticsearch      6 4月  29 2019 plugins
-rw-r--r--  1 elasticsearch elasticsearch   8519 4月  29 2019 README.textile

操作系统调优

[root@ek01 elasticsearch]# vim /etc/sysctl.conf
fs.file-max=655360
vm.max_map_count=262144

[root@ek01 elasticsearch]# vim /etc/security/limits.conf
* soft nofile 655350   #软限制
* hard nofile 655350   #硬限制
* soft nproc 204800
* hard nproc 204800
* soft memlock unlimited
* hard memlock unlimited

[root@ek01 elasticsearch]# vim /etc/security/limits.d/20-nproc.conf
*          soft    nproc     40960
root       soft    nproc     unlimited

[root@ek01 elasticsearch]# sysctl -p
net.ipv4.tcp_syncookies = 1             #防范SYN洪水攻击，0为关闭
net.ipv4.tcp_max_tw_buckets = 20480     #此项参数可以控制TIME_WAIT套接字的最大数量，避免Squid服务器被大量的TIME_WAIT套接字拖死
net.ipv4.tcp_max_syn_backlog = 20480    #表示SYN队列的长度，默认为1024，加大队列长度为8192，可以容纳更多等待连接的网络连接数
net.core.netdev_max_backlog = 262144    #每个网络接口 接受数据包的速率比内核处理这些包的速率快时，允许发送到队列的数据包的最大数目
net.ipv4.tcp_fin_timeout = 20           #FIN-WAIT-2状态的超时时间，避免内核崩溃
fs.file-max = 655360
vm.max_map_count = 262144

[root@ek01 elasticsearch]# ulimit -a
core file size          (blocks, -c) 0
data seg size           (kbytes, -d) unlimited
scheduling priority             (-e) 0
file size               (blocks, -f) unlimited
pending signals                 (-i) 15633
max locked memory       (kbytes, -l) 64
max memory size         (kbytes, -m) unlimited
open files                      (-n) 65535
pipe size            (512 bytes, -p) 8
POSIX message queues     (bytes, -q) 819200
real-time priority              (-r) 0
stack size              (kbytes, -s) 8192
cpu time               (seconds, -t) unlimited
max user processes              (-u) 15633
virtual memory          (kbytes, -v) unlimited
file locks                      (-x) unlimited

[root@ek01 elasticsearch]# exit
登出

[root@ek01 ~]# ulimit -a
core file size          (blocks, -c) 0
data seg size           (kbytes, -d) unlimited
scheduling priority             (-e) 0
file size               (blocks, -f) unlimited
pending signals                 (-i) 15633
max locked memory       (kbytes, -l) unlimited
max memory size         (kbytes, -m) unlimited
open files                      (-n) 655350
pipe size            (512 bytes, -p) 8
POSIX message queues     (bytes, -q) 819200
real-time priority              (-r) 0
stack size              (kbytes, -s) 8192
cpu time               (seconds, -t) unlimited
max user processes              (-u) 204800
virtual memory          (kbytes, -v) unlimited
file locks                      (-x) unlimited

JVM调优

# 当前服务器仅作为elasticsearch使用时，内存参数最优设置为服务器内存的一半
[root@ek01 ~]# cd /usr/local/elasticsearch/config/
[root@ek01 config]# vim jvm.options
-Xms2g
-Xmx2g

配置elasticsearch集群，除此处配置外，其他Elasticsearch节点均相同

[root@ek01 config]# vim elasticsearch.yml
# 集群名称
cluster.name: elkbigdata
# 当前节点名称
node.name: server1
# 是否可被选择为master
node.master: true
# 是否为数据节点
node.data: true
# 数据存储位置，可指定多个存储位置
path.data: /data1/elasticsearch,/data2/elasticsearch
# 日志位置
path.logs: /usr/local/elasticsearch/logs
# 使用物理内存，不使用swap内存
bootstrap.memory_lock: true
network.host: 0.0.0.0
# http服务端口
http.port: 9200
# 最小主节点数
discovery.zen.minimum_master_nodes: 1
# master节点列表
discovery.zen.ping.unicast.hosts: ["192.168.10.105:9300","192.168.10.106:9300"]

[root@ek02 config]# vim elasticsearch.yml
# 集群名称
cluster.name: elkbigdata
# 当前节点名称
node.name: server2
# 是否可被选择为master
node.master: true
# 是否为数据节点
node.data: true
# 数据存储位置，可指定多个存储位置
path.data: /data1/elasticsearch,/data2/elasticsearch
# 日志位置
path.logs: /usr/local/elasticsearch/logs
# 使用物理内存，不使用swap内存
bootstrap.memory_lock: true
network.host: 0.0.0.0
# http服务端口
http.port: 9200
# 最小主节点数
discovery.zen.minimum_master_nodes: 1
# master节点列表
discovery.zen.ping.unicast.hosts: ["192.168.10.105:9300","192.168.10.106:9300"]

[root@ek03 config]# vim elasticsearch.yml
# 集群名称
cluster.name: elkbigdata
# 当前节点名称
node.name: server3
# 是否可被选择为master
node.master: true
# 是否为数据节点
node.data: true
# 数据存储位置，可指定多个存储位置
path.data: /data1/elasticsearch,/data2/elasticsearch
# 日志位置
path.logs: /usr/local/elasticsearch/logs
# 使用物理内存，不使用swap内存
bootstrap.memory_lock: true
network.host: 0.0.0.0
# http服务端口
http.port: 9200
# 最小主节点数
discovery.zen.minimum_master_nodes: 1
# master节点列表
discovery.zen.ping.unicast.hosts: ["192.168.10.105:9300","192.168.10.106:9300"]

创建数据存储目录

[root@ek01 config]# mkdir -p /data1/elasticsearch
[root@ek01 config