CentOS-使用K8S搭建ELK

ELK Elasticsearch Logstash Kibana 日志采集系统
Elasticsearch 是一个搜索和分析引擎，可以获取信息。
Logstash 是服务器端数据处理管道，可以同时从多个来源采集和转换数据，然后将数据发送到如 Elasticsearch 等“存储库”中。
Kibana 是一个页面，可以让用户在 Elasticsearch 中使用图形和图表对数据进行可视化。

实验需要可创建新namespace
vim namespace.yaml

apiVersion: v1
kind: Namespace
metadata:
  name: ELK

创建deployment & service，安装elasticsearch程序并开放访问端口
vim 1.elasticsearch.yaml

apiVersion: apps/v1
kind: Deployment  #资源类型
metadata:  #定义Pod的元数据, 至少定义Labels(key: value)
  name: elasticsearch
  namespace: ELK  #默认Default
  labels :  #定义Pod的Labels(key: value)
    app: elasticsearch
spec:  #定义Pod的规格
  replicas: 1  #Pod的个数，默认1
  selector:  #通过Selector选择被控制的Pod
    matchLabels:
      app: elasticsearch
  template:  #Pod的模板
    metadata:
      name: elasticsearch
      labels:  #给Service通过Labels查询并连接应用
        app: elasticsearch
    spec:
      containers:  #定义Pod的容器
#      - image: docker.io/elasticsearch:7.1.1  #Docker镜像，注意统一ELK三应用版本号
      - image: elasticsearch7.1.1-log4j2.17.1
        imagePullPolicy: IfNotPresent  #拉取容器镜像策略(Nerver/IfNotPresent/Always), 默认Always(从远程仓库拉取)
        name: elasticsearch  #容器Name
        resources:  #定义容器请求的资源
          limits:  #限制容器请求的资源
            cpu: 1
            memory: 2Gi
          requests:
            cpu: 0.5
            memory: 500Mi
        env:  #环境变量, 相当于docker run的-e参数
        - name: "discovery.type"  #单节点elasticsearch
          value: "single-node"
        - name: ES_JAVA_OPTS
          value: "-Xms512m -Xmx2g"
        ports:  #容器暴露端口, 相当于设置Pod端口9200,9300 tcp可用
        - containerPort: 9200
          protocol: TCP
        - containerPort: 9300
          protocol: TCP
---
apiVersion: v1
kind: Service
metadata:
  name: elasticsearch-service
  namespace: ELK
spec:
  type: NodePort  #默认ClusterIP，分ExternalName, ClusterIP, NodePort, and LoadBalancer，Nodepost（Pod端口映射Node端口）外部访问通过NodeIP+NodePort就可以连到应用
  ports:
  - name: elasticsearch-port
    port: 9200  #Service连接Pod端口
    targetPort: 9200  #Elasticsearch Pod端口
    protocol: TCP
    nodePort: 30001  ###K8s外部访问端口, 自定义(0-65535)
  - name: elasticsearch-portlink
    port: 9300
    targetPort: 9300
    protocol: TCP
    nodePort: 30010
  selector:  #服务绑定到指定K:V形式的Pod, 上面配置好的
    app: elasticsearch

创建deployment & service，安装kibana程序并开放访问端口
vim 2.kibana.yaml

apiVersion: apps/v1
kind: Deployment
metadata:
  name: kibana
  namespace: ELK
  labels:
    app: kibana
spec:
  replicas: 1
  selector:
    matchLabels:
      app: kibana
  template:
    metadata:
      name: kibana
      labels:
        app: kibana
    spec:
      containers:
      - name: kibana
        image: docker.io/kibana:7.1.1
        imagePullPolicy: IfNotPresent
        env:
        - name: ELASTICSEARCH_HOSTS 
          value: "http://10.102.218.212:9200"  #elasticsearch service, 与elasticsearch通信
        - name: XPACK_SECURITY_ENABLED
          value: "true"
        ports:
        - name: ui
          containerPort: 5601
          protocol: TCP
        resources:
          limits:
            cpu: 1
            memory: 2Gi
          requests:
            cpu: 0.5
            memory: 500Mi
---
apiVersion: v1
kind: Service
metadata:
  name: kibana-service
  namespace: ELK
spec:
  type: NodePort
  ports:
  - port: 5601
    protocol: TCP
    targetPort: 5601
    nodePort: 30002
  selector:
    app: kibana

创建configmap，保存logstash的input/output/filter文件信息
vim 3.logstash-configmap.yaml

apiVersion: v1
data:
  input-conf: |-
    input {
      tcp {
        port => 5044
        mode => "server"
      }
     }
  output-conf: |-
    output {
      elasticsearch {
        hosts => ["10.106.13.24:9200"] 
        index => "apidemo"
      }

      stdout{
        codec => rubydebug
      }
    }
`
kind: ConfigMap
metadata:
  name: logstash-configmap
  namespace: ELK

创建deployment & service，安装logstash程序并开放访问端口
vim 3.logstash.yaml

apiVersion: apps/v1
kind: Deployment
metadata:
  name: logstash
  namespace: ELK
  labels:
    app: logstash
spec:
  replicas: 1
  selector:
    matchLabels:
      app: logstash
  template:
    metadata:
      name: logstash
      labels:
        app: logstash
    spec:
      containers:
      - name: logstash
       # image: docker.io/logstash:7.1.1
        image: logstash7.1.1-log4j2.17.1
        imagePullPolicy: IfNotPresent
        ports:
        - containerPort: 5044
          protocol: TCP
        volumeMounts:  #Logstash日志接收和转发信息都在此文件配置/usr/share/logstash/pipeline/logstash.conf
        - name: logstash-configmap
          mountPath: /usr/share/logstash/pipeline
        env:  #参数名称和logstash/config/中配置文件一样
        - name: XPACK_MONITORING_ELASTICSEARCH_HOSTS
          value: "http://10.102.218.212:9200"  #elasticsearch service, 与elasticsearch通信
          #value: "elasticsearch-service.test-sjy:9200"
        securityContext:
          privileged: true
      volumes:
      - name: logstash-configmap
        configMap:
          name: logstash-configmap
#        emptyDir: {}
---
apiVersion: v1
kind: Service
metadata:
  name: logstash-service
  namespace: ELK
spec:
  type: NodePort
  ports:
  - port: 5044
    protocol: TCP
    targetPort: 5044
    nodePort: 30003
  selector:
    app: logstash