EFK 安装配置笔记

----------------------------------------------elasticsearch---------------------------------------------------

查看elasticsearch版本:
curl -XGET localhost:9200
安装elasticsearch:
sudo apt-get install elasticsearch

安装成功后提示:

### NOT starting on installation, please execute the following statements to configure elasticsearch service to start automatically using systemd
 sudo systemctl daemon-reload
 sudo systemctl enable elasticsearch.service
### You can start elasticsearch service by executing
 sudo systemctl start elasticsearch.service
Created elasticsearch keystore in /etc/elasticsearch/elasticsearch.keystore

查看运行日志
sudo journalctl -u elasticsearch
重启 sudo service elasticsearch restart

查看当前所有索引
curl 'http://10.39.xx.xx:9200/_cat/indices?v'

删除指定索引

curl -XDELETE 10.39.xx.xx:9200/es-test-log-

###其他配置:

由于系统日志量太大，我这里提供了一个清理指定日期前多少天ES中日志索引信息的脚本startDeleteOldLog.sh
#!/bin/bash

###################################

###################################
#清除最近多少天的日志，默认30天
past_day_count=$1
if [ ! $past_day_count ]; then
    past_day_count=30;
fi
#待清除的索引匹配规则
index_prefix=$2
if [ ! $index_prefix ]; then
    index_prefix="efk-*-*";
fi
#ES地址
es_host=$3
if [ ! $es_host ]; then
    es_host=localhost:9200;
fi
echo "准备清理掉ES[$es_host]内索引前缀为[$index_prefix]的超过当前时间前$past_day_count天的信息......"

function delete_indices() {
    index_name=$1
    index_date=$2
 comp_date=`date -d "$past_day_count day ago" +"%Y-%m-%d"`
 date1="$index_date 00:00:00"
 date2="$comp_date 00:00:00"

 t1=`date -d "$date1" +%s`
 t2=`date -d "$date2" +%s`

 if [ $t1 -le $t2 ]; then        
     curl -XDELETE http://$es_host/$index_name
 fi
}

curl -XGET http://$es_host/_cat/indices | awk -F" " '{print $3}' | egrep "$index_prefix" | sort | while read LINE
do   
    index_name=$LINE;
    index_date=`echo $LINE | awk -F"-" '{print $NF}' | egrep "[0-9]*.[0-9]*.[0-9]*" | uniq  | sed 's/./-/g'`
    if [ $index_date ]; then
        delete_indices $index_name $index_date
 fi
done

echo "清理完成！"

将我们的定时脚本加入到linux定时任务中，自动清理ES中的过期日志信息

# crontab -e
0 2 */10 * * sh /data/bash/task/es/startDeleteOldLog.sh

###其他安装方式:

关闭防火墙
安装EPEL源
yum -y install epel-release
yum clean all
yum makecache
安装系统工具
yum -y install vim wget telnet
JDK安装
java -version

加入ELK yum仓库
sudo rpm --import https://packages.elastic.co/GPG-KEY-elasticsearch
vi /etc/yum.repos.d/elastic.repo

[elastic-7.x]
name=Elastic repository for 7.x packages
baseurl=https://artifacts.elastic.co/packages/7.x/yum
gpgcheck=1
gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch
enabled=1
autorefresh=1
type=rpm-md

wget -P/usr/local/src/-c  https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-7.2.1-linux-x86_64.tar.gz

tar -zxvf ./elasticsearch-7.15.0    -C /usr/local
检查目录的权限
ll /usr/local/
将目录的权限改成appadmin:appadmin
chown  -R appadmin:appadmin elasticsearch-5.6.0
启动
./elasticsearch -d
//查看启动状态
sudo netstat -tlunp|grep 9200
sudo vi config/elasticsearch.yml

修改/etc/security/limits.conf文件，增加配置，用户退出后重新登录生效
*               soft    nofile          65536
*               hard    nofile          65536

执行命令：sysctl -w vm.max_map_count=262144
查看结果：sysctl -a|grep vm.max_map_count
重启虚拟机将失效,在/etc/sysctl.conf文件最后添加一行: vm.max_map_count=262144

-------------------------kibana 下载安装------------------------------------------

wget https://artifacts.elastic.co/downloads/kibana/kibana-7.15.0-linux-x86_64.tar.gz
sudo tar -zxvf ./kibana-7.15.0-linux-x86_64.tar.gz -C /usr/local
cd /usr/local/kibana-7.15.0-linux-x86_64
kibana 修改配置文件
 1 cd /config
 2 vim kibana.yml
 3 # 配置kibana的端口
 4 server.port: 5601
 5 # 配置监听ip
 6 server.host: "192.168.xx.xx"
 7 # 配置es服务器的ip，如果是集群则配置该集群中主节点的ip
 8 elasticsearch.hosts: "http://192.168.xx.xx:9200/"
 9 # 配置kibana的日志文件路径，不然默认是messages里记录日志
10 logging.dest:/home/ubuntu/logs/kibana.log

获取权限: sudo chown -R ubuntu:ubuntu /usr/local/kibana-7.15.0-linux-x86_64

启动: nohup /usr/local/kibana-7.15.0-linux-x86_64/bin/kibana -c ../conf/kibana.yml >/dev/null 2>&1 &
启动,带日志:nohup /usr/local/kibana-7.15.0-linux-x86_64/bin/kibana -c /usr/local/kibana-7.15.0-linux-x86_64/config/kibana.yml >/home/ubuntu/logs/kibana/kibana.log &

访问验证:http://10.39.xx.xx:5601/

---------------------------------------logstash安装配置------------------------------------------------------
wget https://artifacts.elastic.co/downloads/logstash/logstash-7.15.0-linux-x86_64.tar.gz
sudo tar -zxvf ./logstash-7.15.0-linux-x86_64.tar.gz -C /usr/local
sudo chown -R ubuntu:ubuntu /usr/local/logstash-7.15.0

ps -ef |grep logstash
nohup /usr/local/logstash-7.15.0/bin/logstash -f /usr/local/logstash-7.15.0/config/logstash.conf >/home/ubuntu/logs/logstash/logstash.log &
nohup /usr/local/logstash-7.15.0/bin/logstash -f /usr/local/logstash-7.15.0/config/logstash.conf >/dev/null 2>&1 &

-----------------------------------------fileBeat 启动----------------------------------------
wget https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-7.15.0-linux-x86_64.tar.gz
sudo tar -zxvf ./filebeat-7.15.0-linux-x86_64.tar.gz -C /usr/local
sudo chown -R ubuntu:ubuntu /usr/local/filebeat-7.15.0-linux-x86_64 或者sudo chmod 755 /usr/local/filebeat-7.15.0/filebeat-7.15.0-linux-x86_64/filebeat.yml

ps -ef |grep filebeat
nohup /usr/local/filebeat-7.15.0-linux-x86_64/filebeat -e -c /usr/local/filebeat-7.15.0-linux-x86_64/filebeat.yml > /data/logs/filebeat.log &

nohup /usr/local/filebeat-7.15.0-linux-x86_64/filebeat -e -c /usr/local/filebeat-7.15.0-linux-x86_64/filebeat.yml  >/dev/null 2>&1 &  //启动并清空日志