docker部署ELK日志系统+kafka

#拉取镜像
docker pull elasticsearch:7.6.2

# 创建文件夹
mkdir -p /data/elk/es/config
# 创建 elasticsearch.yml 文件
vi /data/elk/es/config/elasticsearch.yml
------------------------写入---------------------------
http.port: 9200
network.host: 0.0.0.0 
cluster.name: "my-el" 
http.cors.enabled: true 
http.cors.allow-origin: "*" 
------------------------结束---------------------------
#启动镜像
docker run -itd --name es -p elasticsearch:7.6.2
# 赋值容器中的data、logs文件
docker cp 容器id:/usr/share/elasticsearch/data /data/elk/es
docker cp 容器id:/usr/share/elasticsearch/logs /data/elk/es
#赋予权限
chown -R 1000:1000 /data/elk/es

#启动镜像
docker run -itd -p 9200:9200 -p 9300:9300 --name es -e ES_JAVA_OPTS="-Xms512m -Xmx512m" -e "discovery.type=single-node" --restart=always -v /data/elk/es/config/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml -v /data/elk/es/data:/usr/share/elasticsearch/data -v /data/elk/es/logs:/usr/share/elasticsearch/logs elasticsearch:7.6.2

# 拉取镜像
docker pull  mobz/elasticsearch-head:5

#启动镜像
docker run -d --restart=always -p 9100:9100 --name elasticsearch-head mobz/elasticsearch-head:5

# 创建文件夹
mkdir -p /data/elk/elasticsearch-head
# 进入文件夹下
cd /data/elk/elasticsearch-head
# 拷贝 vendor.js 文件到宿主机上
docker cp elasticsearch-head:/usr/src/app/_site/vendor.js ./
# 修改 vendor.js 文件内容
sed -i '/contentType:/s/application/x-www-form-urlencoded/application/json;charset=UTF-8/' vendor.js

sed -i '/var inspectData = s.contentType/s/application/x-www-form-urlencoded/application/json;charset=UTF-8/' vendor.js

# 删除容器
docker rm -f elasticsearch-head
# 运行容器
docker run -d --restart=always -p 9100:9100 -v /data/elk/elasticsearch-head/vendor.js:/usr/src/app/_site/vendor.js --name elasticsearch-head mobz/elasticsearch-head:5

#拉取镜像,版本与elasticsearch一致，避免出现不兼容问题
docker pull kibana:7.6.2

#获取elasticsearch容器ip
docker inspect --format '{{ .NetworkSettings.IPAddress }}' es

# 创建挂载配置
mkdir -p /data/elk/kibana
vi /data/elk/kibana/kibana.yml

#配置内容
#Default Kibana configuration for docker target
server.name: kibana
server.host: "0"
elasticsearch.hosts: ["http://elasticsearch容器ip:9200"]
xpack.monitoring.ui.container.elasticsearch.enabled: true
i18n.locale: "zh-CN"
kibana.index: ".kibana"

#启动镜像
docker run -d --restart=always --log-driver json-file --log-opt max-size=100m --log-opt max-file=2 --name kibana -p 5601:5601 -v /data/elk/kibana/kibana.yml:/usr/share/kibana/config/kibana.yml kibana:7.6.2

#拉取zookeeper镜像,因为kafka依赖zookeeper
docker pull wurstmeister/zookeeper

#启动zookeeper镜像
#-v /etc/localtime:/etc/localtime 与服务器使用一样的时间设置
docker run -d  --name zookeeper -p 2181:2181 -v /etc/localtime:/etc/localtime wurstmeister/zookeeper

#拉取kafka镜像
docker pull wurstmeister/kafka

#启动kafka镜像
docker run -d --name kafka -p 9092:9092 
-e KAFKA_BROKER_ID=0 
-e KAFKA_ZOOKEEPER_CONNECT=localhost:2181 
-e KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://localhost:9092 
-e KAFKA_LISTENERS=PLAINTEXT://0.0.0.0:9092 
-v /etc/localtime:/etc/localtime 
wurstmeister/kafka

#进入容器
docker exec -it 容器id bash
#进入到bin目录
cd /opt/kafka/bin/
#创建topic测试
./kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic elk-log
./kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic elk-log
#查看topic
kafka-topics.sh --list --zookeeper localhost:2181
#删除topic
kafka-topics.sh --delete --zookeeper localhost -topic elk-log

#测试是否成功
#生产者,发送消息
./kafka-console-producer.sh --broker-list localhost:9092 --topic elk-log
#消费者,订阅消息
./kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic elk-log

#拉取镜像,跟前面两个保持版本一样
docker pull logstash:7.6.2

#创建挂载配置
mkdir -p /data/elk/logstash
vi /data/elk/logstash/logstash.yml

#配置内容
http.host: "0.0.0.0"
xpack.monitoring.elasticsearch.hosts: [ "http://172.17.0.2:9200" ]
xpack.monitoring.elasticsearch.username: elastic
xpack.monitoring.elasticsearch.password: changeme
path.config: /data/docker/logstash/conf.d
@Slf4j
public class LogIpConfig extends ClassicConverter {
    private static String webIP;

    static {
        try {
            webIP = InetAddress.getLocalHost().getHostAddress();
        } catch (UnknownHostException e) {
            log.error("获取日志Ip异常", e);
            webIP = null;
        }
    }

    @Override
    public String convert(ILoggingEvent event) {
        return webIP;
    }
}

    
    
    
    

    
    
    
    
    
    
    

    
    
        
        
            ${CONSOLE_LOG_PATTERN}
            
        
    

    
        
            
                
                    GMT+8
                
                
                
                    
                        {
                        "ip":"%ip",
                        "severity": "%level",
                        "service": "${springAppName:-}",
                        "trace": "%X{X-B3-TraceId:-}",
                        "span": "%X{X-B3-SpanId:-}",
                        "parent": "%X{X-B3-ParentSpanId:-}",
                        "exportable": "%X{X-Span-Export:-}",
                        "pid": "${PID:-}",
                        "thread": "%thread",
                        "class": "%logger{40}",
                        "rest": "%message",
                        "stack_trace": "%exception{30}"
                        }
                    
                
            
        
        
        ${springAppName:-}-log
        
        
        
        
        
        
        
        bootstrap.servers=localhost:9092
        
        acks=0
        
        linger.ms=1000
        
        max.block.ms=0
        
        client.id=0