栏目分类:
子分类:
返回
名师互学网用户登录
快速导航关闭
当前搜索
当前分类
子分类
实用工具
热门搜索
名师互学网 > IT > 软件开发 > 后端开发 > Java

【kafka】一个工具类轻松解决kafka幂等及并发问题

Java 更新时间: 发布时间: IT归档 最新发布 模块sitemap 名妆网 法律咨询 聚返吧 英语巴士网 伯小乐 网商动力

【kafka】一个工具类轻松解决kafka幂等及并发问题

KafkaConsumerUtil.java
import com.alibaba.fastjson.JSON;
import com.kdniao.bigdataplatcommon.utils.LoggerUtil;
import com.kdniao.bigdataplatdataredis.RedisCache;
import com.kdniao.bigdataplatdataredis.entity.RedisKeyInit;
import com.kdniao.depen.common.KafkaMessageRS;
import com.kdniao.depen.enums.base.BusinessType;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.kafka.support.Acknowledgment;
import org.springframework.stereotype.Component;

import java.time.LocalDateTime;
import java.util.Arrays;
import java.util.concurrent.TimeUnit;
import java.util.function.Consumer;


@Data
@Component
public class KafkaConsumerUtil {

    @Autowired
    private RedisCache redisCache;

    
    @Data
    public static class KafkaContent {

        @ApiModelProperty(value = "队列名")
        private String topic;

        @ApiModelProperty(value = "groupID")
        private String groupID;

        @ApiModelProperty(value = "分区")
        private int partition;

        @ApiModelProperty(value = "分区键")
        private Object key;

        @ApiModelProperty(value = "消息内容")
        private Object message;

        @ApiModelProperty(value = "消费开始时间")
        private LocalDateTime startTime;

        @ApiModelProperty(value = "消费结束时间")
        private LocalDateTime endTime;

        @ApiModelProperty(value = "消费时长,毫秒")
        private long duration;

        @ApiModelProperty(value = "消费成功失败")
        private boolean success;
    }

    
    private void consumer(ConsumerRecord record, Acknowledgment acknowledgment, String groupID, Consumer consumer) {
        KafkaContent kafkaContent = new KafkaContent();
        String topic = record.topic();
        kafkaContent.setTopic(topic);
        kafkaContent.setGroupID(groupID);
        kafkaContent.setPartition(record.partition());
        kafkaContent.setKey(record.key());
        String prefix = topic + "_" + groupID + "_";
        Object message = record.value();
        kafkaContent.setMessage(message);
        long startCurrentTimeMillis = System.currentTimeMillis();
        kafkaContent.setStartTime(LocalDateTime.now());
        boolean success = true;
        String waybillCode = null;
        String shipperCode = null;
        if (ObjectUtils.isNotEmpty(message)) {
            KafkaMessageRS kafkaMessageRS = JSON.parseObject(message + "", KafkaMessageRS.class);
            waybillCode = kafkaMessageRS.getWaybillCode();
            shipperCode = kafkaMessageRS.getShipperCode();
        }
        try {
            consumer.accept(record);
        } catch (Exception e) {
            success = false;
            LoggerUtil.error("【消息队列:" + prefix + ":消费" + "失败" + "】\n" + Arrays.toString(e.getStackTrace()));
            throw new RuntimeException(e);
        } finally {
            long endCurrentTimeMillis = System.currentTimeMillis();
            kafkaContent.setEndTime(LocalDateTime.now());
            long duration = endCurrentTimeMillis - startCurrentTimeMillis;
            kafkaContent.setDuration(duration);
            kafkaContent.setSuccess(success);
            LoggerUtil.info("【消息队列:" + prefix + ":消费" + (success ? "成功" : "失败") + "】", "【消息队列:" + prefix + ":消费" + (success ? "成功" : "失败") + "】 n消息上下文:" + JSON.toJSONString(kafkaContent), BusinessType.A8008, waybillCode, shipperCode, duration);
        }
    }

    
    public void noRepeatConsumer(ConsumerRecord record, Acknowledgment acknowledgment, String groupID, boolean duplicateRemoval, Consumer consumer) {
        Object message = record.value();
        String repeatConsumerlockKey = null;
        String businessConsumerLockKey = null;
        try {
            //幂等处理
            if (ObjectUtils.isNotEmpty(message)) {
                KafkaMessageRS kafkaMessageRS = JSON.parseObject(message + "", KafkaMessageRS.class);
                String id = kafkaMessageRS.getId();
                if (ObjectUtils.isNotEmpty(kafkaMessageRS) && ObjectUtils.isNotEmpty(id)) {
                    String topic = record.topic();
                    String prefix = topic + "_" + groupID + "_";
                    String lockId = null;
                    //防止同一条消息同时消费
                    if (ObjectUtils.isNotEmpty(record.key())) {
                        //如果key不等于null,说明要保证相同key的消费同步,为了防止由于rebalance引发的并发问题,此处消息消费加分布式锁
                        lockId = prefix + record.key();
                    } else {
                        //防止同一条消息同时消费
                        lockId = prefix + id;
                    }

                    repeatConsumerlockKey = RedisKeyInit.RedisKey.getRepeatConsumerLockKey(lockId);
                    boolean lock = redisCache.lock(repeatConsumerlockKey, 0L, 180L);
                    if (!lock) {
                        LoggerUtil.info("【消息队列:" + prefix + ":消费获取锁失败】 n消息上下文:" + message);
                        return;
                    }

                    //处理幂等
                    if (duplicateRemoval) {
                        businessConsumerLockKey = RedisKeyInit.RedisKey.getBusinessConsumerLockKey(topic, groupID, id);
                        //判断是否消费过,如果已经消费过直接退出
                        Object hasConsumer = redisCache.getCacheObject(businessConsumerLockKey);
                        if (ObjectUtils.isNotEmpty(hasConsumer)) {
                            LoggerUtil.error("【" + topic + "_" + groupID + "重复消费】" + businessConsumerLockKey + ",直接手动提交。");
                            acknowledgment.acknowledge();
                            return;
                        }
                    }
                }
            }
            consumer(record, acknowledgment, groupID, consumer);

            if (duplicateRemoval && ObjectUtils.isNotEmpty(businessConsumerLockKey)) {
                //统计成功,将结果存到redis
                redisCache.setCacheObject(businessConsumerLockKey, true, 4, TimeUnit.HOURS);
            }
            acknowledgment.acknowledge();
        } catch (Exception e) {
            throw new RuntimeException(e);
        } finally {
            if (ObjectUtils.isNotEmpty(repeatConsumerlockKey)) {
                redisCache.deleteObject(repeatConsumerlockKey);
            }
        }
    }
}
使用方式
    // 消费监听
    @KafkaListener(topics = {"#{'${waybillReport}'}"}, groupId = KafkaConfigConstant.KA_GROUP_ID, containerFactory = KafkaConfigConstant.KA_KAFKA_CONTAINER_FACTORY)
    public void onMessage(ConsumerRecord record, Acknowledgment acknowledgment) {
        kafkaConsumerUtil.noRepeatConsumer(record, acknowledgment, KafkaConfigConstant.KA_GROUP_ID,true, consumerRecord -> {
            consumerMessage(record);
        });
    }
转载请注明:文章转载自 www.mshxw.com
本文地址:https://www.mshxw.com/it/961756.html
我们一直用心在做
关于我们 文章归档 网站地图 联系我们

版权所有 (c)2021-2022 MSHXW.COM

ICP备案号:晋ICP备2021003244-6号