elk环境构建及其应用日志收集

服务器及其角色

使用的组件及其版本

elasticsearch-6.3.0.deb

filebeat-6.3.0-amd64.deb

jdk-8u65-linux-x64.tar.gz

kibana-6.3.0-amd64.deb

logstash-6.3.0.deb

各个组件之间有版本依赖关系，这个注意一下（尽量用同版本的设置）

JDK环境设置

1、jdk路径 /opt/jdk1.8.0_65
2、/etc/profile的新增配置
export JAVA_HOME=/opt/jdk1.8.0_65/
export JAVA_BIN=$JAVA_HOME/bin
export JAVA_LIB=$JAVA_HOME/lib
export CLASSPATH=.:$JAVA_LIB/tools.jar:$JAVA_LIB/dt.jar
export PATH=$JAVA_BIN:$PATH
3、source /etc/profile
4、#java -version
java version "1.8.0_65"
Java(TM) SE Runtime Environment (build 1.8.0_65-b17)
Java HotSpot(TM) 64-Bit Server VM (build 25.65-b01, mixed mode)

elasticsearch的安装配置

#dpkg -i elasticsearch-6.3.0.deb
配置文件
#grep -Ev '^#|^$' elasticsearch.yml
cluster.name: my-application
node.name: elk-01
path.data: /var/lib/elasticsearch
path.logs: /var/log/elasticsearch
network.host: 172.16.30.98
http.port: 9200

logstash的安装配置

dpkg -i logstash-6.3.0.deb

配置文件

# cat  /etc/logstash/conf.d/logstash.conf
input {
  beats {
    port => 5044
  }
}
output {
if [fields][service] == 'dw_cloud_slave' {
  elasticsearch {
    hosts => "172.16.30.98:9200"
    manage_template => false
    index => "dw_cloud_slave"
    document_type => "%{[@metadata][type]}"
  }
}
if [fields][service] == 'dw_cloud_weixin' {
  elasticsearch {
    hosts => "172.16.30.98:9200"
    manage_template => false
    index => "dw_cloud_weixin"
    document_type => "%{[@metadata][type]}"
  }
}
if [fields][service] == 'dw_cloud_ris' {
  elasticsearch {
    hosts => "172.16.30.98:9200"
    manage_template => false
    index => "dw_cloud_ris"
    document_type => "%{[@metadata][type]}"
  }
}
if [fields][service] == 'dw_cloud_dfs' {
  elasticsearch {
    hosts => "172.16.30.98:9200"
    manage_template => false
    index => "dw_cloud_dfs"
    document_type => "%{[@metadata][type]}"
  }
}
if [fields][service] == 'dw_cloud_admin' {
  elasticsearch {
    hosts => "172.16.30.98:9200"
    manage_template => false
    index => "dw_cloud_admin"
    document_type => "%{[@metadata][type]}"
  }
}
if [fields][service] == 'dw_cloud_scheduler' {
  elasticsearch {
    hosts => "172.16.30.98:9200"
    manage_template => false
    index => "dw_cloud_scheduler"
    document_type => "%{[@metadata][type]}"
  }
}
if [fields][service] == 'dw_cloud_sso' {
  elasticsearch {
    hosts => "172.16.30.98:9200"
    manage_template => false
    index => "dw_cloud_sso"
    document_type => "%{[@metadata][type]}"
  }
}
if [fields][service] == 'dw_cloud_statistics' {
  elasticsearch {
    hosts => "172.16.30.98:9200"
    manage_template => false
    index => "dw_cloud_statistics"
    document_type => "%{[@metadata][type]}"
  }
}
}

kibana安装配置

dpkg -i kibana-6.3.0-amd64.deb
配置文件
# grep -Ev '^#|^$' kibana.yml
server.port: 5601
server.host: "0.0.0.0"
server.name: "elk-03"
elasticsearch.url: "http://172.16.30.98:9200"

filebeat安装配置

dpkg -i filebeat-6.3.0-amd64.deb
# cat /etc/filebeat/filebeat.yml
filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /root/dw_cloud_ai_agent/log/dw_ai.log
    - /root/dw_body_part_webservice/log/dw_body_part_webservice.log
    - /root/aiclassifier/log/nohup_*
    - /root/lung/log/nohup_*
  fields:
    service: "dw_cloud_slave"
- type: log
  enabled: true
  paths:
    - /root/dw_weixin/log/dw_weixin_service.log
  fields:
    service: "dw_cloud_weixin"
- type: log
  enabled: true
  paths:
    - /root/dw_cloud_ris_web/log/dw_cloud_ris_web.log
    - /root/dw_cloud_ris/log/dw_cloud_ris_service.log
    - /root/dw_cloud_qa/log/dw_cloud_qa.log
    - /root/dw_cloud_workstation/log/dw_cloud_workstation.log
  fields:
    service: "dw_cloud_ris"
- type: log
  enabled: true
  paths:
    - /root/dw_cloud_dfs_job/log/dw_cloud_dfs_job.log
    - /root/dw_cloud_dfs_web/log/dw_cloud_dfs_web.log
  fields:
    service: "dw_cloud_dfs"
- type: log
  enabled: true
  paths:
    - /root/dw_cloud_admin_service/log/dw_cloud_admin_service.log
    - /root/dw_cloud_admin_service/log/dw_cloud_admin_service.log
  fields:
    service: "dw_cloud_admin"
- type: log
  enabled: true
  paths:
    - /root/dw_cloud_scheduler/log/dw_cloud_scheduler.log
  fields:
    service: "dw_cloud_scheduler"
- type: log
  enabled: true
  paths:
    - /root/dw_cloud_sso/log/dw_cloud_sso_service.log
  fields:
    service: "dw_cloud_sso"
- type: log
  enabled: true
  paths:
    - /root/dw_statistics_service/log/dw_statistics_service.log
  fields:
    service: "dw_cloud_statistics"
#----------------------------------------------------------------------------------------
filebeat.config.modules:
  path: ${path.config}/modules.d/*.yml
  reload.enabled: false
setup.template.settings:
  index.number_of_shards: 3
setup.kibana:
#----------------------------------------------------------------------------------------
output.logstash:
  hosts: ["172.16.30.166:5044"]

往日志文件里写一些东西

接着控制台会显示既定的索引

创建Discover里可以查看的目录索引