【记录】Telegraf-解析nginx日志

本文档只记录解析nginx access.log相关内容，关于telegraf后期整理一篇文档单独介绍。

1、Tail Input Plugin

Tail Input Plugin 是 Telegraf项目众多Inputs插件之一。
Tail Input Plugin 监听一个日志文件并解析该文件的每个日志变更。
默认情况下，tail 插件的行为类似于以下 Unix tail 命令：

	tail -F --lines=0 myfile.log

Telegraf收集nginx access.log 相当于Unix tail命令：

	tail -F --lines=0 access.log

并使用grok格式来解析日志。
除此之外，Telegraf Tail Input Plugin支持多种 Data Format
细节的东西可以跟下源码，下面直接记录运行配置。

2、配置 (1) telegraf.conf

[agent]
  interval = "10s"
  round_interval = true
  metric_batch_size = 1000
  metric_buffer_limit = 10000
  collection_jitter = "0s"
  flush_interval = "10s"
  flush_jitter = "0s"
  precision = ""
  hostname = ""
  omit_hostname = false

[[inputs.tail]]
  files = ["/Users/enic/github/telegraf/testdata/access.log"]
#   from_beginning = false
  from_beginning = true
  grok_patterns = ["%{COMMON_LOG_FORMAT}"]
  name_override = "nginx_access_log"
  grok_custom_pattern_files = []
  grok_custom_patterns = '''
      NGINX_ACCESS_LOG %{IP:remote_addr} - (-|%{WORD:remote_user}) [%{HTTPDATE:time_local}] %{QS:request} %{NUMBER:status:int} %{NUMBER:body_bytes_sent:int} %{QS:referrer} %{QS:agent} %{IPORHOST:xforwardedfor} %{IPORHOST:host} %{QS:upstream_addr} (-|%{NUMBER:upstream_status:int}) %{BASE10NUM:request_time:float} (-|%{BASE10NUM:upstream_response_time:float}) %{QS:cookie_uin} %{QS:cookie_luin}
  '''
  grok_timezone = "Local"
  data_format = "grok"

[[outputs.file]]
  files = ["/Users/enic/github/telegraf/testdata/outputs.log"]

(2)access.log

10.10.10.10 - - [19/Apr/2022:03:17:34 +0800] "POST /user/login HTTP/1.1" 400 63 "-" "Go-http-client/1.1" "-"
10.10.10.10 - - [19/Apr/2022:03:17:34 +0800] "POST /user/login HTTP/1.1" 400 63 "-" "Go-http-client/1.1" "-"
10.10.10.10 - - [19/Apr/2022:03:17:34 +0800] "POST /user/login HTTP/1.1" 400 63 "-" "Go-http-client/1.1" "-"
10.10.10.10 - - [19/Apr/2022:03:17:34 +0800] "POST /user/login HTTP/1.1" 400 63 "-" "Go-http-client/1.1" "-"
10.10.10.10 - - [19/Apr/2022:03:17:34 +0800] "POST /user/login HTTP/1.1" 400 63 "-" "Go-http-client/1.1" "-"
10.10.10.10 - - [19/Apr/2022:03:17:34 +0800] "POST /user/login HTTP/1.1" 400 63 "-" "Go-http-client/1.1" "-"
10.10.10.10 - - [19/Apr/2022:03:17:44 +0800] "POST /qps HTTP/1.1" 504 3693 "-" "Go-http-client/1.1" "-"
10.10.10.10 - - [19/Apr/2022:03:17:54 +0800] "POST /latency HTTP/1.1" 504 3693 "-" "Go-http-client/1.1" "-"
10.10.10.10 - - [19/Apr/2022:03:18:34 +0800] "POST /user/login HTTP/1.1" 400 63 "-" "Go-http-client/1.1" "-"
10.10.10.10 - - [19/Apr/2022:03:18:34 +0800] "POST /user/login HTTP/1.1" 400 63 "-" "Go-http-client/1.1" "-"
10.10.10.10 - - [19/Apr/2022:03:18:34 +0800] "POST /user/login HTTP/1.1" 400 63 "-" "Go-http-client/1.1" "-"
10.10.10.10 - - [19/Apr/2022:03:18:34 +0800] "POST /user/login HTTP/1.1" 400 63 "-" "Go-http-client/1.1" "-"
10.10.10.10 - - [19/Apr/2022:03:18:34 +0800] "POST /user/login HTTP/1.1" 400 63 "-" "Go-http-client/1.1" "-"
10.10.10.10 - - [19/Apr/2022:03:18:34 +0800] "POST /user/login HTTP/1.1" 400 63 "-" "Go-http-client/1.1" "-"
10.10.10.100 - - [19/Apr/2022:03:18:37 +0800] "GET / HTTP/1.1" 200 1839 "-" "Expanse, a Palo Alto Networks company, searches across the global IPv4 space multiple times per day to identify customers' presences on the Internet. If you would like to be excluded from our scans, please send IP addresses/domains to: scaninfo@paloaltonetworks.com" "-"
10.10.10.10 - - [19/Apr/2022:03:18:44 +0800] "POST /qps HTTP/1.1" 504 3693 "-" "Go-http-client/1.1" "-"
10.10.10.10 - - [19/Apr/2022:03:18:54 +0800] "POST /latency HTTP/1.1" 504 3693 "-" "Go-http-client/1.1" "-"
10.10.10.10 - - [19/Apr/2022:03:19:34 +0800] "POST /user/login HTTP/1.1" 400 63 "-" "Go-http-client/1.1" "-"
10.10.10.10 - - [19/Apr/2022:03:19:34 +0800] "POST /user/login HTTP/1.1" 400 63 "-" "Go-http-client/1.1" "-"
10.10.10.10 - - [19/Apr/2022:03:19:34 +0800] "POST /user/login HTTP/1.1" 400 63 "-" "Go-http-client/1.1" "-"
10.10.10.10 - - [19/Apr/2022:03:19:34 +0800] "POST /user/login HTTP/1.1" 400 63 "-" "Go-http-client/1.1" "-"
10.10.10.10 - - [19/Apr/2022:03:18:34 +0800] "POST /user/login HTTP/1.1" 400 63 "-" "Go-http-client/1.1" "-"
10.10.10.10 - - [19/Apr/2022:03:18:34 +0800] "POST /user/login HTTP/1.1" 400 63 "-" "Go-http-client/1.1" "-"
10.10.10.10 - - [19/Apr/2022:03:18:34 +0800] "POST /user/login HTTP/1.1" 400 63 "-" "Go-http-client/1.1" "-"
10.10.10.10 - - [19/Apr/2022:03:18:34 +0800] "POST /user/login HTTP/1.1" 400 63 "-" "Go-http-client/1.1" "-"
10.10.10.10 - - [19/Apr/2022:03:19:34 +0800] "POST /user/login HTTP/1.1" 400 63 "-" "Go-http-client/1.1" "-"
10.10.10.10 - - [19/Apr/2022:03:19:34 +0800] "POST /user/login HTTP/1.1" 400 63 "-" "Go-http-client/1.1" "-"
10.10.10.10 - - [19/Apr/2022:03:19:34 +0800] "POST /user/login HTTP/1.1" 400 63 "-" "Go-http-client/1.1" "-"
10.10.10.10 - - [19/Apr/2022:03:19:34 +0800] "POST /user/login HTTP/1.1" 400 63 "-" "Go-http-client/1.1" "-"
10.10.10.10 - - [19/Apr/2022:03:18:34 +0800] "POST /user/login HTTP/1.1" 400 63 "-" "Go-http-client/1.1" "-"
10.10.10.10 - - [19/Apr/2022:03:18:34 +0800] "POST /user/login HTTP/1.1" 400 63 "-" "Go-http-client/1.1" "-"
10.10.10.10 - - [19/Apr/2022:03:18:34 +0800] "POST /user/login HTTP/1.1" 400 63 "-" "Go-http-client/1.1" "-"
10.10.10.10 - - [19/Apr/2022:03:18:34 +0800] "POST /user/login HTTP/1.1" 400 63 "-" "Go-http-client/1.1" "-"
10.10.10.10 - - [19/Apr/2022:03:18:34 +0800] "POST /user/login HTTP/1.1" 400 63 "-" "Go-http-client/1.1" "-"
10.10.10.10 - - [19/Apr/2022:03:18:34 +0800] "POST /user/login HTTP/1.1" 400 63 "-" "Go-http-client/1.1" "-"
10.10.10.10 - - [19/Apr/2022:03:18:34 +0800] "POST /user/login HTTP/1.1" 400 63 "-" "Go-http-client/1.1" "-"
10.10.10.10 - - [19/Apr/2022:03:18:34 +0800] "POST /user/login HTTP/1.1" 400 63 "-" "Go-http-client/1.1" "-"
10.10.10.10 - - [19/Apr/2022:03:18:34 +0800] "POST /user/login HTTP/1.1" 400 63 "-" "Go-http-client/1.1" "-"

3、操作

cd telegraf
make
./telegraf --config=/Users/enic/github/telegraf/testdata/telegraf.conf

4、output展示

nginx_access_log,host=enicMBP,path=/Users/enic/github/telegraf/testdata/access.log,resp_code=400,verb=POST request="/user/login",http_version=1.1,ident="-",auth="-",resp_bytes=63i,client_ip="10.10.10.10" 1650309454000000000
nginx_access_log,host=enicMBP,path=/Users/enic/github/telegraf/testdata/access.log,resp_code=400,verb=POST request="/user/login",resp_bytes=63i,client_ip="10.10.10.10",ident="-",auth="-",http_version=1.1 1650309454001000000
nginx_access_log,host=enicMBP,path=/Users/enic/github/telegraf/testdata/access.log,resp_code=400,verb=POST ident="-",request="/user/login",auth="-",http_version=1.1,resp_bytes=63i,client_ip="10.10.10.10" 1650309454002000000
nginx_access_log,host=enicMBP,path=/Users/enic/github/telegraf/testdata/access.log,resp_code=400,verb=POST http_version=1.1,client_ip="10.10.10.10",ident="-",resp_bytes=63i,auth="-",request="/user/login" 1650309454003000000
nginx_access_log,host=enicMBP,path=/Users/enic/github/telegraf/testdata/access.log,resp_code=400,verb=POST client_ip="10.10.10.10",auth="-",ident="-",request="/user/login",http_version=1.1,resp_bytes=63i 1650309454004000000
nginx_access_log,host=enicMBP,path=/Users/enic/github/telegraf/testdata/access.log,resp_code=400,verb=POST ident="-",auth="-",request="/user/login",resp_bytes=63i,client_ip="10.10.10.10",http_version=1.1 1650309454005000000
nginx_access_log,host=enicMBP,path=/Users/enic/github/telegraf/testdata/access.log,resp_code=504,verb=POST auth="-",request="/qps",http_version=1.1,resp_bytes=3693i,client_ip="10.10.10.10",ident="-" 1650309464000000000
nginx_access_log,host=enicMBP,path=/Users/enic/github/telegraf/testdata/access.log,resp_code=504,verb=POST ident="-",auth="-",request="/latency",http_version=1.1,resp_bytes=3693i,client_ip="10.10.10.10" 1650309474000000000
nginx_access_log,host=enicMBP,path=/Users/enic/github/telegraf/testdata/access.log,resp_code=400,verb=POST auth="-",request="/user/login",resp_bytes=63i,client_ip="10.10.10.10",http_version=1.1,ident="-" 1650309514000000000
nginx_access_log,host=enicMBP,path=/Users/enic/github/telegraf/testdata/access.log,resp_code=400,verb=POST ident="-",auth="-",request="/user/login",client_ip="10.10.10.10",resp_bytes=63i,http_version=1.1 1650309514001000000
nginx_access_log,host=enicMBP,path=/Users/enic/github/telegraf/testdata/access.log,resp_code=400,verb=POST resp_bytes=63i,client_ip="10.10.10.10",ident="-",auth="-",request="/user/login",http_version=1.1 1650309514002000000
nginx_access_log,host=enicMBP,path=/Users/enic/github/telegraf/testdata/access.log,resp_code=400,verb=POST request="/user/login",resp_bytes=63i,ident="-",auth="-",http_version=1.1,client_ip="10.10.10.10" 1650309514003000000
nginx_access_log,host=enicMBP,path=/Users/enic/github/telegraf/testdata/access.log,resp_code=400,verb=POST client_ip="10.10.10.10",auth="-",ident="-",request="/user/login",http_version=1.1,resp_bytes=63i 1650309514004000000
nginx_access_log,host=enicMBP,path=/Users/enic/github/telegraf/testdata/access.log,resp_code=400,verb=POST resp_bytes=63i,ident="-",request="/user/login",client_ip="10.10.10.10",auth="-",http_version=1.1 1650309514005000000
nginx_access_log,host=enicMBP,path=/Users/enic/github/telegraf/testdata/access.log,resp_code=200,verb=GET ident="-",request="/",resp_bytes=1839i,auth="-",http_version=1.1,client_ip="10.10.10.100" 1650309517000000000
nginx_access_log,host=enicMBP,path=/Users/enic/github/telegraf/testdata/access.log,resp_code=504,verb=POST request="/qps",auth="-",ident="-",http_version=1.1,resp_bytes=3693i,client_ip="10.10.10.10" 1650309524000000000
nginx_access_log,host=enicMBP,path=/Users/enic/github/telegraf/testdata/access.log,resp_code=504,verb=POST resp_bytes=3693i,auth="-",request="/latency",http_version=1.1,client_ip="10.10.10.10",ident="-" 1650309534000000000
nginx_access_log,host=enicMBP,path=/Users/enic/github/telegraf/testdata/access.log,resp_code=400,verb=POST http_version=1.1,resp_bytes=63i,auth="-",client_ip="10.10.10.10",ident="-",request="/user/login" 1650309574000000000
nginx_access_log,host=enicMBP,path=/Users/enic/github/telegraf/testdata/access.log,resp_code=400,verb=POST ident="-",auth="-",resp_bytes=63i,client_ip="10.10.10.10",request="/user/login",http_version=1.1 1650309574001000000
nginx_access_log,host=enicMBP,path=/Users/enic/github/telegraf/testdata/access.log,resp_code=400,verb=POST ident="-",request="/user/login",auth="-",http_version=1.1,resp_bytes=63i,client_ip="10.10.10.10" 1650309574002000000
nginx_access_log,host=enicMBP,path=/Users/enic/github/telegraf/testdata/access.log,resp_code=400,verb=POST auth="-",request="/user/login",client_ip="10.10.10.10",ident="-",http_version=1.1,resp_bytes=63i 1650309574003000000
nginx_access_log,host=enicMBP,path=/Users/enic/github/telegraf/testdata/access.log,resp_code=400,verb=POST http_version=1.1,resp_bytes=63i,client_ip="10.10.10.10",auth="-",request="/user/login",ident="-" 1650309514000000000
nginx_access_log,host=enicMBP,path=/Users/enic/github/telegraf/testdata/access.log,resp_code=400,verb=POST ident="-",auth="-",client_ip="10.10.10.10",request="/user/login",http_version=1.1,resp_bytes=63i 1650309514001000000
nginx_access_log,host=enicMBP,path=/Users/enic/github/telegraf/testdata/access.log,resp_code=400,verb=POST http_version=1.1,client_ip="10.10.10.10",request="/user/login",resp_bytes=63i,auth="-",ident="-" 1650309514002000000
nginx_access_log,host=enicMBP,path=/Users/enic/github/telegraf/testdata/access.log,resp_code=400,verb=POST resp_bytes=63i,ident="-",request="/user/login",http_version=1.1,client_ip="10.10.10.10",auth="-" 1650309514003000000
nginx_access_log,host=enicMBP,path=/Users/enic/github/telegraf/testdata/access.log,resp_code=400,verb=POST resp_bytes=63i,client_ip="10.10.10.10",ident="-",auth="-",request="/user/login",http_version=1.1 1650309574000000000
nginx_access_log,host=enicMBP,path=/Users/enic/github/telegraf/testdata/access.log,resp_code=400,verb=POST client_ip="10.10.10.10",ident="-",auth="-",resp_bytes=63i,request="/user/login",http_version=1.1 1650309574001000000
nginx_access_log,host=enicMBP,path=/Users/enic/github/telegraf/testdata/access.log,resp_code=400,verb=POST client_ip="10.10.10.10",resp_bytes=63i,ident="-",auth="-",request="/user/login",http_version=1.1 1650309574002000000
nginx_access_log,host=enicMBP,path=/Users/enic/github/telegraf/testdata/access.log,resp_code=400,verb=POST resp_bytes=63i,client_ip="10.10.10.10",ident="-",auth="-",request="/user/login",http_version=1.1 1650309574003000000
nginx_access_log,host=enicMBP,path=/Users/enic/github/telegraf/testdata/access.log,resp_code=400,verb=POST request="/user/login",resp_bytes=63i,client_ip="10.10.10.10",auth="-",http_version=1.1,ident="-" 1650309514000000000
nginx_access_log,host=enicMBP,path=/Users/enic/github/telegraf/testdata/access.log,resp_code=400,verb=POST auth="-",http_version=1.1,resp_bytes=63i,client_ip="10.10.10.10",request="/user/login",ident="-" 1650309514001000000
nginx_access_log,host=enicMBP,path=/Users/enic/github/telegraf/testdata/access.log,resp_code=400,verb=POST client_ip="10.10.10.10",auth="-",resp_bytes=63i,ident="-",request="/user/login",http_version=1.1 1650309514002000000
nginx_access_log,host=enicMBP,path=/Users/enic/github/telegraf/testdata/access.log,resp_code=400,verb=POST auth="-",resp_bytes=63i,request="/user/login",http_version=1.1,client_ip="10.10.10.10",ident="-" 1650309514003000000
nginx_access_log,host=enicMBP,path=/Users/enic/github/telegraf/testdata/access.log,resp_code=400,verb=POST request="/user/login",resp_bytes=63i,client_ip="10.10.10.10",ident="-",http_version=1.1,auth="-" 1650309514004000000
nginx_access_log,host=enicMBP,path=/Users/enic/github/telegraf/testdata/access.log,resp_code=400,verb=POST request="/user/login",client_ip="10.10.10.10",http_version=1.1,resp_bytes=63i,ident="-",auth="-" 1650309514005000000
nginx_access_log,host=enicMBP,path=/Users/enic/github/telegraf/testdata/access.log,resp_code=400,verb=POST request="/user/login",http_version=1.1,resp_bytes=63i,client_ip="10.10.10.10",ident="-",auth="-" 1650309514006000000
nginx_access_log,host=enicMBP,path=/Users/enic/github/telegraf/testdata/access.log,resp_code=400,verb=POST ident="-",request="/user/login",resp_bytes=63i,client_ip="10.10.10.10",auth="-",http_version=1.1 1650309514007000000

【记录】Telegraf-解析nginx日志

Go语言相关栏目本月热门文章