ELK过滤Nginx日志和Java日志

日志样例 JAVA

固定格式

[2022-05-30T14:54:07.579+08:00] [INFO] [,] [cn.git.workflow.util.WorkFlowFactory] [ccms-test-17] [3.1.101.55] [workflow-server] [WorkFlowFactory.java,163,cn.git.workflow.util.WorkFlowFactory,getWebServiceProperties] [webService获取地址类型serviceType为[UAT],选择操作url地址为[http://6.1.14.86:9001/wfms/cxf/processWebService?wsdl]] ## ''
[2022-05-30T14:54:07.624+08:00] [INFO] [,] [cn.git.workflow.service.impl.WorkFlowServiceImpl] [ccms-test-17] [3.1.101.55] [workflow-server] [WorkFlowServiceImpl.java,414,cn.git.workflow.service.impl.WorkFlowServiceImpl,findProcessPic] [查看流程图发送参数: {"processId":"787169"}] ## ''

NGINX access日志

直接配置nginx.conf, 配置为json格式,固定格式

    log_format json escape=json
      '{"@timestamp":"$time_iso8601",'
      '"host":"$hostname",'
      '"server_ip":"$server_addr",'
      '"client_ip":"$remote_addr",'
      '"xff":"$http_x_forwarded_for",'
      '"domain":"$host",'
      '"url":"$uri",'
      '"referer":"$http_referer",'
      '"args":"$args",'
      '"upstreamtime":"$upstream_response_time",'
      '"responsetime":"$request_time",'
      '"request_method":"$request_method",'
      '"status":"$status",'
      '"size":"$body_bytes_sent",'
      '"request_body":"$request_body",'
      '"request_length":"$request_length",'
      '"protocol":"$server_protocol",'
      '"upstreamhost":"$upstream_addr",'
      '"file_dir":"$request_filename",'
      '"http_user_agent":"$http_user_agent"'
    '}';

error日志

默认配置, 输出格式不固定

格式一

2022/05/30 14:48:45 [warn] 13#0: *87124 an upstream response is buffered to a te1/23/0000000231 while reading upstream, client: 3.2.26.199, server: 3.1.101.57, P/1.1", upstream: "http://3.1.101.56:11102/manage/common/syscodes/get", host: "3"

格式二

2022/05/30 10:37:03 [error] 13#0: *81714 connect() failed (111: Connection refused) while connecting to upstream, client: 3.2.26.154, server: 3.1.101.57, request: "GET /sysmanage/manage/organization/listps HTTP/1.1", upstream: "http://3.1.101.56:11102/manage/organization/listps", host: "3.1.101.57:8901", referrer: "http://3.1.101.57:8901/"

格式三

2022/05/30 15:12:33 [emerg] 1#0: unexpected end of file, expecting "}" in /usr/local/openresty/nginx/conf/nginx.conf:105

2022/05/30 22:41:12 [notice] 22802#0: signal process started

filebeat配置

###################### Filebeat Configuration Example #########################
filebeat.name: ccms-test-19
filebeat.idle_timeout: 5s
filebeat.spool_zie: 2048

#----------------------------------input form ccms servers--------------------------------#
filebeat.inputs:
- type: log
  enabled: true
  paths:
   - /opt/ccms-auto-deploy/credit-interface-converter/*/target/logs/*.log
   - /opt/ccms-auto-deploy/credit-gateway/target/logs/*.log
  fields:
    kafka_topic: topic-ccms-dev
    filebeat-server: 3.1.101.57    
  fields_under_root: true

  # filebeat 多行日志的处理
  multiline.pattern: '^['
  multiline.negate: true
  multiline.match: after

  encoding: plain
  tail_files: false

  # 检测指定目录下文件更新时间
  scan_frequency: 3s
  # 每隔1s检测一下文件变化，如果连续检测2次之后文件还没有变化，下一次检测间隔时间变为5s
  backoff: 1s
  max_backoff: 5s
  backoff_factor: 2

#----------------------------------input form nginx access_log--------------------------------#
- type: log
  enabled: true
  paths:
   - /data/openresty-vts/nginx/logs/ccms-*.log
  fields:
    kafka_topic: topic-nginx-access
    filebeat-server: 3.1.101.57
  fields_under_root: true

  encoding: plain
  tail_files: false

  json.keys_under_root: true
  json.overwrite_keys: true
  json.add_error_key: false


  # 检测指定目录下文件更新时间
  scan_frequency: 3s
  # 每隔1s检测一下文件变化，如果连续检测2次之后文件还没有变化，下一次检测间隔时间变为5s
  backoff: 1s
  max_backoff: 5s
  backoff_factor: 2

#----------------------------------input form nginx error_log--------------------------------#
- type: log
  enabled: true
  paths:
   - /data/openresty-vts/nginx/logs/error.log
  fields:
    kafka_topic: topic-nginx-error
    filebeat-server: 3.1.101.57
  fields_under_root: true

  encoding: plain
  tail_files: false

  # 检测指定目录下文件更新时间
  scan_frequency: 3s
  # 每隔1s检测一下文件变化，如果连续检测2次之后文件还没有变化，下一次检测间隔时间变为5s
  backoff: 1s
  max_backoff: 5s
  backoff_factor: 2

#----------------------------------Kafka output--------------------------------#
output.kafka:
  enabled: true
  hosts: ['3.1.101.33:9092','3.1.101.34:9092','3.1.101.35:9092']
  topic: '%{[kafka_topic]}'

logstash配置 JAVA

input {
  beats {
    port => 5044
  }
}

input {
   kafka {
    topics_pattern => "topic-ccms-dev"
    bootstrap_servers => "3.1.101.33:9092,3.1.101.34:9092,3.1.101.35:9092"
    consumer_threads => 4
    decorate_events => true
    group_id => "kafka-elk-ccms"
    add_field => {"logstash-server" => "3.1.101.33"}
   }
}

filter {
    json {
      source => "message"
    }

    grok {
      match => { "message" => "[%{TIMESTAMP_ISO8601:currentDateTime}] [%{LOGLEVEL:level}] [%{DATA:traceInfo}] [%{NOTSPACE:class}] [%{DATA:hostName}] [%{IP:hostIp}] [%{DATA:applicationName}] [%{DATA:location}] [%{DATA:messageInfo}] ## %{QUOTEDSTRING:throwable}" }
    }

    mutate{
      enable_metric => "false"
      remove_field => ["ecs","tags","input","agent","@version","log","port","host","message"]
    }

    date {
      match => [ "currentDateTime", "ISO8601" ]
    }
}

output {
        elasticsearch {
        hosts => ["3.1.101.33:9200","3.1.101.34:9200","3.1.101.35:9200"]
        index => "index-ccms-dev_%{+YYY-MM-dd}"
        sniffing => true
        template_overwrite => true
    }
}

NGINX access日志

input {
   kafka {
    topics_pattern => "topic-nginx-access"
    bootstrap_servers => "3.1.101.33:9092,3.1.101.34:9092,3.1.101.35:9092"
    codec => "json"
    consumer_threads => 4
    decorate_events => true
    group_id => "kafka-nginx-access"
    add_field => {"logstash-server" => "3.1.101.33"}
   }
}

filter {
  geoip {
      source => "client_ip"
      target => "geoip"
      add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ]
      add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}" ]
      remove_field => [ "[geoip][latitude]", "[geoip][longitude]", "[geoip][country_code2]","[geoip][country_code3]", "[geoip][timezone]", "[geoip][continent_code]", "[geoip][dma_code]", "[geoip][region_code]" ]
  }

  mutate {
    convert => [ "size", "integer" ]
    convert => [ "status", "integer" ]
    convert => [ "responsetime", "float" ]
    convert => [ "upstreamtime", "float" ]
    convert => [ "[geoip][coordinates]", "float" ]
    # 过滤 filebeat 没用的字段,这里过滤的字段要考虑好输出到es的，否则过滤了就没法做判断
    remove_field => [ "ecs","agent","host","cloud","@version","input","logs_type" ]
  }


  useragent {
    source => "http_user_agent"
    target => "ua"
    # 过滤useragent没用的字段
    remove_field => [ "[ua][minor]","[ua][major]","[ua][build]","[ua][patch]","[ua][os_minor]","[ua][os_major]" ]
  }

}

output {
        elasticsearch {
        hosts => ["3.1.101.33:9200","3.1.101.34:9200","3.1.101.35:9200"]
        index => "logstash-nginx-access_%{+YYY-MM-dd}"
        sniffing => true
        template_overwrite => true
    }
}

这里是为了适应grafana 11190模板, 将ES数据展示到grafana中, 实际可根据需求调整

error日志

input {
   kafka {
    topics_pattern => "topic-nginx-error"
    bootstrap_servers => "3.1.101.33:9092,3.1.101.34:9092,3.1.101.35:9092"
    consumer_threads => 4
    decorate_events => true
    group_id => "kafka-nginx-error"
    add_field => {"logstash-server" => "3.1.101.33"}
    enable_metric => true
   }
}

filter {
    json {
      source => "message"
    }

    grok {
      match => [
        "message", "%{DATESTAMP:currentDateTime}s{1,}[%{LOGLEVEL:level}]s{1,}(%{NUMBER:pid:int}#%{NUMBER}:s{1,}*%{NUMBER})s{1,}(%{GREEDYDATA:messageInfo})(?:,s{1,}client:s{1,}(?%{IP}|%{HOSTNAME}))(?:,s{1,}server:s{1,}%{IPORHOST:server})(?:, request: %{QS:request})?(?:, upstream: "%{URI:endpoint}")?(?:, host: "%{HOSTPORT:host}")?(?:, referrer: "%{URI:referrer}")?",
        "message", "%{DATESTAMP:currentDateTime}s{1,}[%{DATA:level}]s{1,}%{GREEDYDATA:messageInfo}"]
    }

    date{
      match => ["currentDateTime", "yy/MM/dd HH:mm:ss", "ISO8601"]
      timezone => "+08:00"
      target => "@timestamp"
    }

    mutate{
      enable_metric => "false"
      remove_field => [ "ecs","tags","input","agent","@version","log","port","host","message" ]
    }
}

output {
        elasticsearch {
        hosts => ["3.1.101.33:9200","3.1.101.34:9200","3.1.101.35:9200"]
        index => "logstash-nginx-error_%{+YYY-MM-dd}"
        sniffing => true
        template_overwrite => true
    }
}

配置解析:

json插件: 将message中所有字段信息提取到顶级目录, 包括filebeat中定义的字段
grok插件: 做了一个match匹配列表, 来适应nginx错误日志的多种情况
date: 
  match: 将提取的currentDateTime时间转换为ISO8601
  timezone: 调整时区, 到kibana里会加8小时
  target: 将currentDateTime时间覆盖@timestamp时间戳
mutate: 
  enable_metric: 关闭指标
  remove_field: 删除无用字段

过滤结果 JAVA

{
"_index": "index-ccms-dev_2022-05-31",
"_type": "_doc",
"_id": "MliXF4EBEBn_sZq0n-NG",
"_version": 1,
"_score": 1,
"_source": {
    "class": "cn.git.collateral.manage.CollateralEsbApiImpl",
    "kafka_topic": "topic-ccms-dev",
    "logstash-server": "3.1.101.33",
    "level": "ERROR",
    "location": "CollateralEsbApiImpl.java,44,cn.git.collateral.manage.CollateralEsbApiImpl,esbOptionServer$original$rX5ytHc4",
    "@timestamp": "2022-05-31T00:50:14.741Z",
    "filebeat-server": "3.1.101.56",
    "hostIp": "3.1.101.56",
    "traceInfo": "TID: 97353afd28624015baa3f69c98ef23a7.93.16539582147090015",
    "hostName": "ccms-test-18",
    "currentDateTime": "2022-05-31T08:50:14.741+08:00",
    "throwable": "''",
    "messageInfo": "esb服务调用COLLATERAL模块异常，错误信息[nested exception is org.apache.ibatis.type.TypeException: Could not set parameters for mapping: ParameterMapping{property='colTpCd', mode=IN, javaType=class java.lang.Object, jdbcType=null, numericScale=null, resultMapId='null', jdbcTypeName='null', expression='null'}. Cause: org.apache.ibatis.type.TypeException: Error setting null for parameter #1 with JdbcType OTHER . Try setting a different JdbcType for this parameter or a different jdbcTypeForNull configuration property. Cause: java.sql.SQLException: 无效的列类型: 1111]",
    "applicationName": "collateral-server"
    }
}

NGINX access

{
"_index": "logstash-nginx-access_2022-05-30",
"_type": "_doc",
"_id": "Q1RpEoEBEBn_sZq0ngYQ",
"_version": 1,
"_score": 1,
"_source": {
    "file_dir": "/opt/ccms-auto-deploy/front-bank-credit/static/js/app.eef69865.js",
    "log": {
        "offset": 2433145,
        "file": {
        "path": "/data/openresty-vts/nginx/logs/ccms-credit-access.log"
    }
    },
    "upstreamhost": "",
    "request_body": "",
    "logstash-server": "3.1.101.33",
    "geoip": {
    "postal_code": "98109",
    "region_name": "Washington",
    "location": {
        "lat": 47.6348,
        "lon": -122.3451
    },
    "city_name": "Seattle",
    "coordinates": [
        -122.3451
        ,
        47.6348
    ],
    "country_name": "United States",
    "ip": "3.2.26.243"
    },
    "referer": "http://3.1.101.57:8901/",
    "protocol": "HTTP/1.1",
    "http_user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.5005.62 Safari/537.36",
    "request_length": "448",
    "client_ip": "3.2.26.243",
    "server_ip": "3.1.101.57",
    "domain": "3.1.101.57",
    "kafka_topic": "topic-nginx-access",
    "url": "/static/js/app.eef69865.js",
    "@timestamp": "2022-05-30T00:41:56.000Z",
    "request_method": "GET",
    "status": 304,
    "responsetime": 0,
    "xff": "",
    "args": "",
    "upstreamtime": 0,
    "size": 0,
    "ua": {
        "name": "Chrome",
        "os_name": "Windows",
        "device": "Other",
        "os": "Windows"
    }
    }
}

error

{
"_index": "logstash-nginx-error_2022-05-30",
"_type": "_doc",
"_id": "d_vBE4EBA-RkhKPLS3nJ",
"_version": 1,
"_score": 1,
"_source": {
    "logstash-server": "3.1.101.34",
    "request": ""GET /sysmanage/manage/organization/tree HTTP/1.1"",
    "@timestamp": "2022-05-30T06:57:23.485Z",
    "messageInfo": "an upstream response is buffered to a temporary file /usr/local/openresty/nginx/proxy_temp/3/14/0000000143 while reading upstream",
    "kafka_topic": "topic-nginx-error",
    "level": "warn",
    "referrer": "http://3.1.101.57:8901/",
    "client": "3.2.26.91",
    "filebeat-server": "3.1.101.55",
    "pid": 13,
    "currentDateTime": "2022/05/28 09:57:46",
    "endpoint": "http://3.1.101.56:11102/manage/organization/tree",
    "server": "3.1.101.57"
    }
}

ELK过滤Nginx日志和Java日志

Linux相关栏目本月热门文章