docker-compose部署EFK

root@shutang:/home/shutang/docker-scripts# tree log-scripts/
log-scripts/
|-- README.MD
|-- docker-compose.yml
|-- es
|   `-- elasticsearch.yml
|-- fluentd
|   |-- Dockerfile
|   `-- conf
|       `-- fluent.conf
`-- kibana
    |-- Dockerfile
    `-- kibana.yml

version: '3'

services:
  fluentd:
    build: ./fluentd
    restart: always
    volumes:
      - /mnt/disks/append-disk/fluentd/log:/fluentd/log
      - ./fluentd/conf/fluent.conf:/fluentd/etc/fluent.conf
    ports:
      - "24224:24224"
      - "24224:24224/udp"
    environment:
      - "discovery.zen.ping.unicast.hosts=elasticsearch"
    networks:
      - esnet

  elasticsearch:
    image: docker.elastic.co/elasticsearch/elasticsearch:6.8.22
    container_name: elasticsearch
    restart: always
    environment:
      - node.name=master1
      - bootstrap.memory_lock=true
      - "ES_JAVA_OPTS=-Xms6g -Xmx6g"
      - network.publish_host=10.175.100.100
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes:
      - esdata1:/usr/share/elasticsearch/data
      - ./es/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml
    expose:
      - 9200
      - 9300
    ports:
      - 9200:9200
      - 9300:9300
    networks:
      - esnet

  kibana:
    build: ./kibana
    restart: always
    environment:
      - "discovery.zen.ping.unicast.hosts=elasticsearch"
    ports:
      - "5601:5601"
    networks:
      - esnet
    volumes:
      - ./kibana/kibana.yml:/usr/share/kibana/config/kibana.yml
volumes:
  esdata1:
    driver: local
networks:
  esnet:

cluster.name: "docker-cluster"
network.host: 0.0.0.0

# minimum_master_nodes need to be explicitly set when bound on a public IP
# set to 1 to allow single node clusters
# Details: https://github.com/elastic/elasticsearch/pull/17288
discovery.zen.minimum_master_nodes: 1
discovery.zen.ping.unicast.hosts: ["10.175.100.100", "10.175.100.101"]
xpack.security.enabled: false

FROM fluent/fluentd:v1.5

USER root

RUN echo "source 'https://mirrors.tuna.tsinghua.edu.cn/rubygems/'" > Gemfile && gem install bundler

RUN gem install fluent-plugin-elasticsearch -v 4.0.3   --no-document
#RUN gem install fluent-plugin-elasticsearch  --no-document
RUN gem install fluent-plugin-concat

USER fluent

CMD ["fluentd", "-o", "/fluentd/log/fluentd.log", "--log-rotate-age", "daily", "--log-rotate-size", "102400"]

  @type forward
  port 24224
  bind 0.0.0.0



  @type concat
  @log_level trace
  key log
  multiline_start_regexp /^d{1,3}.d{1,3}.d{1,3}.d{1,3}|^- GET|^d{4}-d{1,2}-d{1,2}.d{1,2}:d{1,2}:d{1,2}/
  multiline_end_regexp /n$/



  @type copy
  
    @type elasticsearch
    host 10.175.100.100
    port 9200
    logstash_format true
    logstash_prefix mxlog
    logstash_dateformat %Y%m%d
    include_tag_key true
    tag_key @log_name
    flush_interval 1s
  
  
    @type stdout
  



  @type copy
  
    @type elasticsearch
    host 10.175.100.100
    port 9200
    logstash_format true
    logstash_prefix log
    logstash_dateformat %Y%m%d
    include_tag_key true
    tag_key @log_name
    flush_interval 1s
  
  
    @type stdout
  

root@shutang:/home/shutang/docker-scripts/log-scripts# cat es2-dc.yml
version: '3'

services:
  elasticsearch:
    image: docker.elastic.co/elasticsearch/elasticsearch:6.8.22
    container_name: elasticsearch-slave
    restart: always
    environment:
      - node.name=master2
      - bootstrap.memory_lock=true
      - "ES_JAVA_OPTS=-Xms2g -Xmx2g"
      - network.publish_host=10.175.100.101
    expose:
      - 9200
      - 9300
    ports:
      - 9200:9200
      - 9300:9300
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes:
      - ./es/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml
      - /mnt/disks/append/es-data:/usr/share/elasticsearch/data

cluster.name: "docker-cluster"
network.host: 0.0.0.0

# minimum_master_nodes need to be explicitly set when bound on a public IP
# set to 1 to allow single node clusters
# Details: https://github.com/elastic/elasticsearch/pull/17288
discovery.zen.minimum_master_nodes: 1
discovery.zen.ping.unicast.hosts: ["10.175.100.100", "10.176.100.101"]