java定时任务删除ElasticSearch没有用的索引

问题描述

1、在es集群上发现，集群每天会自动生成一堆没有用的索引，百度了下大概是监控，日志类型的记录索引，大概的名称如下。单台的环境上没有看到这些索引。

2、这些自动生成的索引是不是由于elasticsearch.yml文件中配置了action.auto_create_index，这个还需要测试，目前没有去验证，大家可以验证了把结果告诉我。百度后发现可以直接设置action.auto_create_index: false，这个需要大家验证。

action.auto_create_index: .security,.monitoring*,.watches,.triggered_watches,.watcher-history*,app-a-*,app-b-*

--------------------------------------删除无用索引代码如下---------------

1、pom文件添加

    org.springframework.boot
    spring-boot-starter-data-elasticsearch
    2.5.3

2、bootstrap.yml配置文件

spring:
  elasticsearch:
    # 多个服务用英文逗号分开
    hostAndPort: 192.168.1.10:9201,192.168.1.11:9201,192.168.1.12:9201
    username: elastic
    password: changeme

3、增加ElasticSearchConfig.java

package com.config;

import lombok.extern.slf4j.Slf4j;
import org.elasticsearch.client.RestHighLevelClient;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.data.elasticsearch.client.ClientConfiguration;
import org.springframework.data.elasticsearch.client.RestClients;




@Configuration
@Slf4j
public class ElasticSearchConfig {


    @Value("${spring.elasticsearch.hostAndPort}")
    private String hostAndPort;
    @Value("${spring.elasticsearch.username}")
    private String username;
    @Value("${spring.elasticsearch.password}")
    private String password;


    @Bean
    public RestHighLevelClient elasticsearchClient() {
        // 多台部署
        String[] hosts = hostAndPort.split(",");
        ClientConfiguration configuration = ClientConfiguration.builder()
                .connectedTo(hosts)
                .withBasicAuth(username, password)
                .build();
        RestHighLevelClient client = RestClients.create(configuration).rest();

        return client;
    }


}

4、创建定时任务DeleteJob.java

package com.job;


import lombok.extern.slf4j.Slf4j;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.client.indices.GetIndexRequest;
import org.elasticsearch.client.indices.GetIndexResponse;
import org.elasticsearch.common.unit.Timevalue;
import org.springframework.scheduling.annotation.Async;
import org.springframework.scheduling.annotation.EnableScheduling;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;

import javax.annotation.Resource;
import java.text.SimpleDateFormat;
import java.util.*;



@Component
@EnableScheduling
@Async
@Slf4j
public class DeleteJob {


    @Resource
    private RestHighLevelClient restHighLevelClient;


    //@Scheduled(cron = "${spring.elasticsearch.deleteNoUseIndex}")
    @Scheduled(cron = "0 0 4 * * ?")
    public void deleteIndexJob() {
        // 每天早上4点执行
        log.info("deleteIndexJob begin ...");
        try {
            deleteEsIndexByDay();
        } catch (Exception e) {
            e.printStackTrace();
        }
    }

    public void deleteEsIndexByName() {
        try {
            // 按照名字删除索引
            GetIndexRequest getIndexRequest = new GetIndexRequest("cloud_es_*");
            deleteEsIndex(getIndexRequest);
        } catch (Exception e) {
            e.printStackTrace();
        }
    }

    public void deleteEsIndexByDay() {
        try {
            Date today = new Date();
            Calendar c = Calendar.getInstance();
            c.setTime(today);
            c.add(Calendar.DAY_OF_MONTH, -1);
            SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd");
            String day = sdf.format(c.getTime());
            GetIndexRequest getIndexRequest = new GetIndexRequest("cloud_es_*");
            // 只删除昨天的，就是【cloud_es_*-day】
            GetIndexResponse getIndexResponse = restHighLevelClient.indices().get(getIndexRequest, RequestOptions.DEFAULT);
            String[] indices = getIndexResponse.getIndices();
            List deleteIds = new ArrayList<>();
            for(String key:indices){
                if(key.length()>8){
                    try {
                        Integer date = Integer.parseInt(key.substring(key.length()-8));
                        if(date<=Integer.parseInt(day)){
                            // 删除昨天和以前的所有没有用的索引
                            deleteIds.add(key);
                        }
                    }catch (Exception e){
                        e.printStackTrace();
                    }
                }
            }
            if(deleteIds.size() > 0){
                deleteEsIndex(deleteIds.toArray(new String[deleteIds.size()]));
            }
        } catch (Exception e) {
            e.printStackTrace();
        }
    }

    public void deleteEsIndex(GetIndexRequest getIndexRequest){
        // 构建获取所有索引的请求
        GetIndexResponse getIndexResponse = null;
        try {
            getIndexResponse = restHighLevelClient.indices().get(getIndexRequest, RequestOptions.DEFAULT);
            String[] indices = getIndexResponse.getIndices();
            for(String key:indices){
                log.info("no use index, key={}",key);
            }
            DeleteIndexRequest request = new DeleteIndexRequest(indices);
            request.timeout(Timevalue.timevalueMinutes(2));
            restHighLevelClient.indices().delete(request, RequestOptions.DEFAULT);
        } catch (Exception e) {
            e.printStackTrace();
        }

    }

    public void deleteEsIndex(String[] indices){
        try {
            for(String key:indices){
                log.info("no use index, key={}",key);
            }
            DeleteIndexRequest request = new DeleteIndexRequest(indices);
            request.timeout(Timevalue.timevalueMinutes(2));
            restHighLevelClient.indices().delete(request, RequestOptions.DEFAULT);
        } catch (Exception e) {
            e.printStackTrace();
        }

    }

}