ELK分布式日志收集

es 要求Linux 的 参数 vm.max_map_count 至少为 262144，目录需要赋权限: chmod -R 777 search

新建docker-compose.yml

version: '3.1'
services:
  elasticsearch:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.6.2
    container_name: elasticsearch
    restart: always
    volumes:
      - ./search/es/esdata:/usr/share/elasticsearch/data
      #- ./search/config/es01/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml
      - ./search/es/logs/elasticsearch:/usr/share/elasticsearch/logs
      - ./search/es/plugins/analysis-ik:/usr/share/elasticsearch/plugins/analysis-ik
    environment:
      - discovery.type=single-node
      - "ES_JAVA_OPTS=-Xms4g -Xmx4g"
      - TZ=Asia/Shanghai
    ports:
      - "9200:9200"
      - "9300:9300"
    networks:
      - elk

  kibana:
    image: docker.elastic.co/kibana/kibana:7.6.2
    links:
      - elasticsearch
    environment:
      ELASTICSEARCH_HOSTS: http://elasticsearch:9200
    volumes:
      - ./kibana_config/:/usr/local/kibana/config/
    ports:
      - "5601:5601"
    depends_on:
      - elasticsearch 
    networks:
      - elk
    container_name: kibana
    restart: always

  logstash:
    image: docker.elastic.co/logstash/logstash:7.6.2
    links:
      - elasticsearch
    command: logstash -f /etc/logstash/config/logstash.conf  #logstash 启动时使用的配置文件
    volumes:
      - ./search/logstash/config/logstash.conf:/etc/logstash/config/logstash.conf  #logstash 配文件位置
      - ./search/logstash/config/logstash.yml:/usr/share/logstash/config/logstash.yml  #logstash 配文件位置
     # - ./search/logstash/log/pv.log:/logs/pv.log #挂载要读取日志文件
    depends_on:
      - elasticsearch  #后于elasticsearch启动
    ports:
      - "4560:4560"
      # - "8065:8065"
      # - "9600:9600"
    networks:
      - elk 
    container_name: logstash

networks:
  elk:

新建./search/es/plugins/analysis-ik安装ik分词器

https://github.com/medcl/elasticsearch-analysis-ik/releases/download/v7.6.2/elasticsearch-analysis-ik-7.6.2.zip

新建./search/logstash/config/logstash.conf

input {
	tcp {
		mode => "server"
		port => 4560
		codec => json #json格式读取
	}
}
filter {
	json{
		source => "message"
		remove_field => ["host","port","message","@version"] #去掉不要的字段
  }
}

output {
	elasticsearch {
		action => "index"
		hosts  => "172.18.42.142:9200"
		index  => "test_log"
	}
}

新建./search/logstash/config/logstash.yml

http.host: "0.0.0.0"
xpack.monitoring.elasticsearch.hosts: ["http://172.0.0.1:9200"] #设置es地址

springboot微服务的pom文件添加

    org.springframework.boot
    >spring-boot-starter-data-elasticsearch


    net.logstash.logback
    >logstash-logback-encoder
    6.6

resources/application.yml加入

spring:
    elasticsearch:
        rest:
          uris: http://172.0.0.1:9200 #es地址

resources/logback.xml

	


	

	
	
		
			INFO
		
		
		
			
				
					
						{
						"date":"%date{"yyyy-MM-dd HH:mm:ss.SSS",UTC}",
						"level":"%level",
						"msg":"%msg"
						}
					
				
			
		
	

	
		172.0.0.1:4560
		
		
	

	
	
		
		
	

	

		
	

	
	
	    
	

es的整合

# domain/WebLog
@Data
@document(indexName = "test_log",shards = 1,replicas = 1)
public class WebLog {

    @Id
    private String id;

    @Field(type = FieldType.Integer)
    private Integer port;

    @Field(type = FieldType.Text)
    private String message;

    @Field(name = "@version",type = FieldType.Keyword)
    private String version;

    @Field(name = "@timestamp", type = FieldType.Date,format = DateFormat.date_time)
    private Date timestamp;

    @Field(type = FieldType.Keyword)
    private String host;

    private LogMessage lm;
}

# repository/TestRepository
public interface TestRepository extends ElasticsearchRepository {
}

# controller/WebLogController
@Api(tags = "es信息接口")
@RestController
public class WebLogController {
    @Autowired
    private TestRepository testRepository;
    
    @ApiOperation(value = "test详情")
    @GetMapping("/testfindById/{id}")
    public R> testfindById(@PathVariable String id) {
        return R.ok(testRepository.findById(id));
    }
}