注：各个服务logstash之间无通信，通过kafka的group_id一致实现消息只消费一次的功能的伪集群
logstash采集与清洗数据到elasticsearch https://www.cnblogs.com/dyh004/p/9641377.html
https://www.jianshu.com/p/443f1ea7b640
在server和client上都安装logstash
[root@client src]# tar xf logstash-7.12.0-linux-x86_64.tar.gz -C /usr/local/
[root@client src]# mv /usr/local/logstash-7.12.0/ /usr/local/logstash
测试logstash
//输入采用标准输入，输出采用标准输出
[root@client src]# cd /usr/local/logstash/bin/
[root@client bin]# ./logstash -e 'input{ stdin{} } output{ stdout{} }'
The stdin plugin is now waiting for input:
hello world ---手动输入hello world
{
"message" => "hello wohello world",
"@version" => "1",
"host" => "client",
"@timestamp" => 2021-08-03T03:45:28.335Z
}
//使用Rubydebug显示详细输出，codec为一种编译器
[root@client bin]# ./logstash -e 'input{ stdin{} } output{ stdout{ codec=>rubydebug } }'
{
"message" => "hello world",
"@version" => "1",
"host" => "client",
"@timestamp" => 2021-08-03T03:49:21.673Z
}
用logstash将信息写入elasticsearch中
[root@client bin]# ./logstash -e 'input{ stdin{} } output{ stdout{ hosts=>["192.168.100.99:9200"] } }'
修改logstash配置文件，让其收集系统日志/var/log/messages,并将其输出到屏幕上。
条件判断（condition）
Logstash从1.3.0版本开始支持条件判断和表达式。
表达式支持下面这些操作符：
- ==,!=,<,>,<=,>=
- =（匹配正则），!（不匹配正则）
- in（包含），not in（不包含）
- and，or，nand(非与)，xor（非或）
- （）（复合表达式），!()（对复合表达式取反）
[root@server bin]# vim logstash.conf
input {
file{
path => "/usr/local/nginx/logs/access.log" //可以定义多个日志类型或目录
type => "nginx_access" //自定义日志的类型
start_position => "beginning"
}
}
input {
kafka {
bootstrap_servers => "192.168.108.173:9093"
auto_offset_reset => "latest"
group_id => "intfitms-log"
topics => ["intf-logs"]
codec => multiline {
pattern => "^%{TIMESTAMP_ISO8601} "
negate => true
what => "previous"
}
}
}
filter {
grok {
match => {
"message"=> "%{TIMESTAMP_ISO8601:logtime} %{GREEDYDATA:appName} [%{GREEDYDATA:thread}] %{LOGLEVEL:loglevel} %{GREEDYDATA:class} - %{GREEDYDATA:loginfo}"
}
}
date {
match => ["logtime","yyyy-MM-dd HH:mm:ss.SSS"]
target => "@timestamp"
locale => "cn"
timezone => "+08:00"
remove_field => "logtime"
}
mutate{
remove_field => ["thread","thread.keyword"] //移除字段
}
}
output {
if "AdviceInfo" in [message] {
file {
path => "/data/intf-%{+yyyy.MM.dd}.log"
codec => line { format => "%{message}"} //输出为本地文件，日志格式不变
}
}
else{
elasticsearch {
hosts => ["192.168.100.99:9200"，"192.168.100.100:9200"，"192.168.100.101:9200"]
index => "test-%{+YYYY.MM.dd}" --可以根据主机名、ip、应用名称等来命名
}
}
}
[root@server bin]# ./logstash -f logstash.conf --执行
[root@server bin]# nohup ./logstash -f logstash.conf & --后台运行
如果日志过多，可以将日志缓存到redis，redis再写入elasticsearch
[root@server src]# curl -L -O https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-6.5.0-x86_64.rpm
[root@server src]# rpm -vi filebeat-6.5.0-x86_64.rpm
[root@server src]# vim /etc/filebeat/filebeat.yml
filebeat.inputs:
- type: log
encoding: GBK
paths:
- /data01/filebeat/test.out
include_lines: ["AdviceInfo"]
scan_frequency: 10s
fields_under_root: true
fields:
log_ip: 133.38.137.168
log_name: intf-app
output.redis:
hosts: ["192.168.108.173"]
password: "7o&Oghn2Ou36"
port: 8001
db: 0
key: "intf-app-list"
[root@server src]# filebeat modules enable elasticsearch
[root@server src]# filebeat setup
[root@server src]# service filebeat start
grok匹配
性能
前段时间分析日志，由于印象中logstash的性能有限，在github上找到一段go写的日志采集且能发送ES的程序，在日志分割部分，想利用grok的灵活，继续在github上查找go实现grok的代码，组合形成一个新的日志采集程序。简单测试，功能实现。
在部署生产环境，发现日志采集的速度为1000EPS，修改参数也很难提升，开始意味是Elasticsearch所限，搭建elasticsearch集群后，依旧受限。之后，放弃grok功能，发现直接转发并写入ES，可以轻松达到30K EPS，相差30倍的性能且无法满足基本性能需求，直接丢弃grok。
在grok调试中，由于采用是整个字段匹配，如果一处无法匹配，就会导致整个匹配失败。编写grok语句蛮痛苦的，需要不断尝试，更有可能是同样的字段，出现调试器异常有时匹配不成功的现象（包括kibana 6.3自带的）。
测试中遇到的坑如下：
- 涉及正则语法的字符需要使用转义，如’('、 ‘)’、 ‘[’、 ‘]’…
- 其他实现的grok语法，需要对‘’再次转义，相当于“”转义
- 如果DATA无法匹配，可以使用GREEDYDATA，或许有惊喜
- 记不清字符‘|’是否有转义，可以尝试下
经历几次艰苦的调试，已经对grok好感大幅降低，grok依附与logstash等，对于数据采集分割性能堪忧，基于性能，考虑放弃logstash，自然放弃这种模式。
此外，发现针对字段处理，可能使用mapping是更好的模式，将尝试使用新的方式。
grok预定字段
USERNAME [a-zA-Z0-9._-]+
USER %{USERNAME}
INT (?:[+-]?(?:[0-9]+))
BASE10NUM (?[+-]?(?:(?:[0-9]+(?:.[0-9]+)?)|(?:.[0-9]+)))
NUMBER (?:%{BASE10NUM})
BASE16NUM (?(?"(?>\.|[^\"]+)+"|""|(?>'(?>\.|[^\']+)+')|''|(?>`(?>\.|[^\`]+)+`)|``))
UUID [A-Fa-f0-9]{8}-(?:[A-Fa-f0-9]{4}-){3}[A-Fa-f0-9]{12}
# Networking
MAC (?:%{CISCOMAC}|%{WINDOWSMAC}|%{COMMONMAC})
CISCOMAC (?:(?:[A-Fa-f0-9]{4}.){2}[A-Fa-f0-9]{4})
WINDOWSMAC (?:(?:[A-Fa-f0-9]{2}-){5}[A-Fa-f0-9]{2})
COMMONMAC (?:(?:[A-Fa-f0-9]{2}:){5}[A-Fa-f0-9]{2})
IPV6 ((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]d|1dd|[1-9]?d)(.(25[0-5]|2[0-4]d|1dd|[1-9]?d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]d|1dd|[1-9]?d)(.(25[0-5]|2[0-4]d|1dd|[1-9]?d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]d|1dd|[1-9]?d)(.(25[0-5]|2[0-4]d|1dd|[1-9]?d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]d|1dd|[1-9]?d)(.(25[0-5]|2[0-4]d|1dd|[1-9]?d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]d|1dd|[1-9]?d)(.(25[0-5]|2[0-4]d|1dd|[1-9]?d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]d|1dd|[1-9]?d)(.(25[0-5]|2[0-4]d|1dd|[1-9]?d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]d|1dd|[1-9]?d)(.(25[0-5]|2[0-4]d|1dd|[1-9]?d)){3}))|:)))(%.+)?
IPV4 (?1,2})[.](?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2})[.](?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2})[.](?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2}))(?![0-9])
IP (?:%{IPV6}|%{IPV4})
HOSTNAME b(?:[0-9A-Za-z][0-9A-Za-z-]{0,62})(?:.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(.?|b)
HOST %{HOSTNAME}
IPORHOST (?:%{HOSTNAME}|%{IP})
HOSTPORT %{IPORHOST}:%{POSINT}
# paths
PATH (?:%{UNIXPATH}|%{WINPATH})
UNIXPATH (?>/(?>[w_%!$@:.,-]+|\.)*)+
TTY (?:/dev/(pts|tty([pq])?)(w+)?/?(?:[0-9]+))
WINPATH (?>[A-Za-z]+:|\)(?:\[^\?*]*)+
URIPROTO [A-Za-z]+(+[A-Za-z+]+)?
URIHOST %{IPORHOST}(?::%{POSINT:port})?
# uripath comes loosely from RFC1738, but mostly from what Firefox
# doesn't turn into %XX
URIPATH (?:/[A-Za-z0-9$.+!*'(){},~:;=@#%_-]*)+
#URIPARAM ?(?:[A-Za-z0-9]+(?:=(?:[^&]*))?(?:&(?:[A-Za-z0-9]+(?:=(?:[^&]*))?)?)*)?
URIPARAM ?[A-Za-z0-9$.+!*'|(){},~@#%&/=:;_?-[]]*
URIPATHPARAM %{URIPATH}(?:%{URIPARAM})?
URI %{URIPROTO}://(?:%{USER}(?::[^@]*)?@)?(?:%{URIHOST})?(?:%{URIPATHPARAM})?
# Months: January, Feb, 3, 03, 12, December
MONTH b(?:Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:tember)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)b
MONTHNUM (?:0?[1-9]|1[0-2])
MONTHNUM2 (?:0[1-9]|1[0-2])
MONTHDAY (?:(?:0[1-9])|(?:[12][0-9])|(?:3[01])|[1-9])
# Days: Monday, Tue, Thu, etc...
DAY (?:Mon(?:day)?|Tue(?:sday)?|Wed(?:nesday)?|Thu(?:rsday)?|Fri(?:day)?|Sat(?:urday)?|Sun(?:day)?)
# Years?
YEAR (?>dd){1,2}
HOUR (?:2[0123]|[01]?[0-9])
MINUTE (?:[0-5][0-9])
# '60' is a leap second in most time standards and thus is valid.
SECOND (?:(?:[0-5]?[0-9]|60)(?:[:.,][0-9]+)?)
TIME (?!<[0-9])%{HOUR}:%{MINUTE}(?::%{SECOND})(?![0-9])
# datestamp is YYYY/MM/DD-HH:MM:SS.UUUU (or something like it)
DATE_US %{MONTHNUM}[/-]%{MONTHDAY}[/-]%{YEAR}
DATE_EU %{MONTHDAY}[./-]%{MONTHNUM}[./-]%{YEAR}
ISO8601_TIMEZONE (?:Z|[+-]%{HOUR}(?::?%{MINUTE}))
ISO8601_SECOND (?:%{SECOND}|60)
TIMESTAMP_ISO8601 %{YEAR}-%{MONTHNUM}-%{MONTHDAY}[T ]%{HOUR}:?%{MINUTE}(?::?%{SECOND})?%{ISO8601_TIMEZONE}?
DATE %{DATE_US}|%{DATE_EU}
DATESTAMP %{DATE}[- ]%{TIME}
TZ (?:[PMCE][SD]T|UTC)
DATESTAMP_RFC822 %{DAY} %{MONTH} %{MONTHDAY} %{YEAR} %{TIME} %{TZ}
DATESTAMP_RFC2822 %{DAY}, %{MONTHDAY} %{MONTH} %{YEAR} %{TIME} %{ISO8601_TIMEZONE}
DATESTAMP_OTHER %{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{TZ} %{YEAR}
DATESTAMP_EVENTLOG %{YEAR}%{MONTHNUM2}%{MONTHDAY}%{HOUR}%{MINUTE}%{SECOND}
# Syslog Dates: Month Day HH:MM:SS
SYSLOGTIMESTAMP %{MONTH} +%{MONTHDAY} %{TIME}
PROG (?:[w._/%-]+)
SYSLOGPROG %{PROG:program}(?:[%{POSINT:pid}])?
SYSLOGHOST %{IPORHOST}
SYSLOGFACILITY <%{NONNEGINT:facility}.%{NONNEGINT:priority}>
HTTPDATE %{MONTHDAY}/%{MONTH}/%{YEAR}:%{TIME} %{INT}
# Shortcuts
QS %{QUOTEDSTRING}
# Log formats
SYSLOGBASE %{SYSLOGTIMESTAMP:timestamp} (?:%{SYSLOGFACILITY} )?%{SYSLOGHOST:logsource} %{SYSLOGPROG}:
COMMONAPACHELOG %{IPORHOST:clientip} %{USER:ident} %{USER:auth} [%{HTTPDATE:timestamp}] "(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})" %{NUMBER:response} (?:%{NUMBER:bytes}|-)
COMBINEDAPACHELOG %{COMMONAPACHELOG} %{QS:referrer} %{QS:agent}
# Log Levels
LOGLEVEL ([Aa]lert|ALERT|[Tt]race|TRACE|[Dd]ebug|DEBUG|[Nn]otice|NOTICE|[Ii]nfo|INFO|[Ww]arn?(?:ing)?|WARN?(?:ING)?|[Ee]rr?(?:or)?|ERR?(?:OR)?|[Cc]rit?(?:ical)?|CRIT?(?:ICAL)?|[Ff]atal|FATAL|[Ss]evere|SEVERE|EMERG(?:ENCY)?|[Ee]merg(?:ency)?)
