栏目分类:
子分类:
返回
名师互学网用户登录
快速导航关闭
当前搜索
当前分类
子分类
实用工具
热门搜索
名师互学网 > IT > 软件开发 > 后端开发 > Java

使用 通配符+Lua脚本 删除大批量的Key

Java 更新时间: 发布时间: IT归档 最新发布 模块sitemap 名妆网 法律咨询 聚返吧 英语巴士网 伯小乐 网商动力

使用 通配符+Lua脚本 删除大批量的Key

在不阻塞业务的前提下,使用 Lua脚本删除百万/千万级的key
在lua脚本中使用scan扫描key,并依次删除
当删除数量达到1万时,脚本直接返回,完成本次调用
如果删除的key数量大于0,就循环调用脚本进行删除

工具类
import org.springframework.data.redis.connection.RedisConnection;
import org.springframework.data.redis.connection.ReturnType;
import org.springframework.data.redis.core.RedisCallback;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.data.redis.core.script.DefaultRedisScript;
import org.springframework.data.redis.core.script.RedisScript;
import org.springframework.data.redis.serializer.RedisSerializer;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
import java.util.List;


public class RedisBatchDeleteLua {

    private static volatile RedisScript script;
    private static final int batchSize = 10000;

    
    public static long delete(String pattern, StringRedisTemplate redisTemplate) {
        return delete(pattern, batchSize, 0, redisTemplate);
    }

    
    public static long delete(String pattern, int interval, StringRedisTemplate redisTemplate) {
        return delete(pattern, batchSize, interval, redisTemplate);
    }

    
    public static long delete(String pattern, int bachSize, int interval, StringRedisTemplate redisTemplate) {
        try {
            RedisScript script = getScript(redisTemplate);
            RedisSerializer keySerializer = redisTemplate.getKeySerializer();
            byte[] KEY1 = keySerializer.serialize(pattern);
            byte[] ARGV = keySerializer.serialize(String.valueOf(bachSize));
            long total = 0;
            Long batchDelete;
            do {
                batchDelete = (Long) redisTemplate.execute((RedisCallback) connection -> connection.evalSha(script.getSha1(), ReturnType.VALUE, 1, KEY1, ARGV));
                batchDelete = batchDelete == null ? 0L : batchDelete;
                total += batchDelete;
                if (interval > 0) {
                    Thread.sleep(interval);
                }
            } while (batchDelete > 0);
            return total;
        } catch (IOException | InterruptedException x) {
            throw new RuntimeException(x);
        }
    }

    private static RedisScript getScript(StringRedisTemplate redisTemplate) throws IOException {
        if (script != null) {
            return script;
        }
        synchronized (RedisBatchDeleteLua.class) {
            if (script != null) {
                return script;
            }
            String lua = getScript();
            script = new DefaultRedisScript(lua);
            redisTemplate.execute((RedisCallback) connection -> {
                if (isScriptExist(script.getSha1(), connection)) {
                    return null;
                }
                RedisSerializer keySerializer = redisTemplate.getKeySerializer();
                connection.scriptLoad(keySerializer.serialize(lua));
                return null;
            });
        }
        return script;
    }


    private static String getScript() throws IOException {
        InputStream input = RedisBatchDeleteLua.class.getClassLoader().getResourceAsStream("redis/RedisBatchDelete.lua");
        BufferedReader reader = new BufferedReader(new InputStreamReader(input, StandardCharsets.UTF_8));
        StringBuilder sb = new StringBuilder();
        String line;
        while ((line = reader.readLine()) != null) {
            sb.append("n").append(line);
        }
        return sb.toString();
    }

    private static boolean isScriptExist(String sha, RedisConnection connection) {
        List booleans = connection.scriptExists(sha);
        if (booleans == null || booleans.isEmpty()) {
            return false;
        }
        Boolean scriptExist = booleans.get(0);
        return scriptExist != null && scriptExist;
    }

}

 
lua脚本 
local limitSize = tonumber(ARGV[1]) -- 最多删除多少个key
local batchSize = limitSize -- scan一次最多扫描多少个key

if (batchSize > 10000) then -- 一次扫描不能超过1w条
    batchSize = 10000
end

local function scan(key)
    local cursor = 0
    local keynum = 0

    repeat
        local res = redis.call("scan", cursor, "match", key, 'COUNT', batchSize)

        if (res ~= nil and #res >= 0) then
            redis.replicate_commands()
            cursor = tonumber(res[1])
            local ks = res[2]
            local size = #ks
            for i=1,size,1 do
                redis.call("del", tostring(ks[i]))
                keynum = keynum + 1
                if (keynum >= limitSize) then -- 已经删除了指定数量的key, 返回
                  return keynum
                end
            end
        end
    until (cursor <= 0)

    return keynum
end

local total = scan(KEYS[1])

return total
转载请注明:文章转载自 www.mshxw.com
我们一直用心在做
关于我们 文章归档 网站地图 联系我们

版权所有 (c)2021-2022 MSHXW.COM

ICP备案号:晋ICP备2021003244-6号