KafkaConsumerUtil.java
import com.alibaba.fastjson.JSON;
import com.kdniao.bigdataplatcommon.utils.LoggerUtil;
import com.kdniao.bigdataplatdataredis.RedisCache;
import com.kdniao.bigdataplatdataredis.entity.RedisKeyInit;
import com.kdniao.depen.common.KafkaMessageRS;
import com.kdniao.depen.enums.base.BusinessType;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.kafka.support.Acknowledgment;
import org.springframework.stereotype.Component;
import java.time.LocalDateTime;
import java.util.Arrays;
import java.util.concurrent.TimeUnit;
import java.util.function.Consumer;
@Data
@Component
public class KafkaConsumerUtil {
@Autowired
private RedisCache redisCache;
@Data
public static class KafkaContent {
@ApiModelProperty(value = "队列名")
private String topic;
@ApiModelProperty(value = "groupID")
private String groupID;
@ApiModelProperty(value = "分区")
private int partition;
@ApiModelProperty(value = "分区键")
private Object key;
@ApiModelProperty(value = "消息内容")
private Object message;
@ApiModelProperty(value = "消费开始时间")
private LocalDateTime startTime;
@ApiModelProperty(value = "消费结束时间")
private LocalDateTime endTime;
@ApiModelProperty(value = "消费时长,毫秒")
private long duration;
@ApiModelProperty(value = "消费成功失败")
private boolean success;
}
private void consumer(ConsumerRecord, ?> record, Acknowledgment acknowledgment, String groupID, Consumer consumer) {
KafkaContent kafkaContent = new KafkaContent();
String topic = record.topic();
kafkaContent.setTopic(topic);
kafkaContent.setGroupID(groupID);
kafkaContent.setPartition(record.partition());
kafkaContent.setKey(record.key());
String prefix = topic + "_" + groupID + "_";
Object message = record.value();
kafkaContent.setMessage(message);
long startCurrentTimeMillis = System.currentTimeMillis();
kafkaContent.setStartTime(LocalDateTime.now());
boolean success = true;
String waybillCode = null;
String shipperCode = null;
if (ObjectUtils.isNotEmpty(message)) {
KafkaMessageRS kafkaMessageRS = JSON.parseObject(message + "", KafkaMessageRS.class);
waybillCode = kafkaMessageRS.getWaybillCode();
shipperCode = kafkaMessageRS.getShipperCode();
}
try {
consumer.accept(record);
} catch (Exception e) {
success = false;
LoggerUtil.error("【消息队列:" + prefix + ":消费" + "失败" + "】\n" + Arrays.toString(e.getStackTrace()));
throw new RuntimeException(e);
} finally {
long endCurrentTimeMillis = System.currentTimeMillis();
kafkaContent.setEndTime(LocalDateTime.now());
long duration = endCurrentTimeMillis - startCurrentTimeMillis;
kafkaContent.setDuration(duration);
kafkaContent.setSuccess(success);
LoggerUtil.info("【消息队列:" + prefix + ":消费" + (success ? "成功" : "失败") + "】", "【消息队列:" + prefix + ":消费" + (success ? "成功" : "失败") + "】 n消息上下文:" + JSON.toJSONString(kafkaContent), BusinessType.A8008, waybillCode, shipperCode, duration);
}
}
public void noRepeatConsumer(ConsumerRecord, ?> record, Acknowledgment acknowledgment, String groupID, boolean duplicateRemoval, Consumer consumer) {
Object message = record.value();
String repeatConsumerlockKey = null;
String businessConsumerLockKey = null;
try {
//幂等处理
if (ObjectUtils.isNotEmpty(message)) {
KafkaMessageRS kafkaMessageRS = JSON.parseObject(message + "", KafkaMessageRS.class);
String id = kafkaMessageRS.getId();
if (ObjectUtils.isNotEmpty(kafkaMessageRS) && ObjectUtils.isNotEmpty(id)) {
String topic = record.topic();
String prefix = topic + "_" + groupID + "_";
String lockId = null;
//防止同一条消息同时消费
if (ObjectUtils.isNotEmpty(record.key())) {
//如果key不等于null,说明要保证相同key的消费同步,为了防止由于rebalance引发的并发问题,此处消息消费加分布式锁
lockId = prefix + record.key();
} else {
//防止同一条消息同时消费
lockId = prefix + id;
}
repeatConsumerlockKey = RedisKeyInit.RedisKey.getRepeatConsumerLockKey(lockId);
boolean lock = redisCache.lock(repeatConsumerlockKey, 0L, 180L);
if (!lock) {
LoggerUtil.info("【消息队列:" + prefix + ":消费获取锁失败】 n消息上下文:" + message);
return;
}
//处理幂等
if (duplicateRemoval) {
businessConsumerLockKey = RedisKeyInit.RedisKey.getBusinessConsumerLockKey(topic, groupID, id);
//判断是否消费过,如果已经消费过直接退出
Object hasConsumer = redisCache.getCacheObject(businessConsumerLockKey);
if (ObjectUtils.isNotEmpty(hasConsumer)) {
LoggerUtil.error("【" + topic + "_" + groupID + "重复消费】" + businessConsumerLockKey + ",直接手动提交。");
acknowledgment.acknowledge();
return;
}
}
}
}
consumer(record, acknowledgment, groupID, consumer);
if (duplicateRemoval && ObjectUtils.isNotEmpty(businessConsumerLockKey)) {
//统计成功,将结果存到redis
redisCache.setCacheObject(businessConsumerLockKey, true, 4, TimeUnit.HOURS);
}
acknowledgment.acknowledge();
} catch (Exception e) {
throw new RuntimeException(e);
} finally {
if (ObjectUtils.isNotEmpty(repeatConsumerlockKey)) {
redisCache.deleteObject(repeatConsumerlockKey);
}
}
}
}
使用方式
// 消费监听
@KafkaListener(topics = {"#{'${waybillReport}'}"}, groupId = KafkaConfigConstant.KA_GROUP_ID, containerFactory = KafkaConfigConstant.KA_KAFKA_CONTAINER_FACTORY)
public void onMessage(ConsumerRecord, ?> record, Acknowledgment acknowledgment) {
kafkaConsumerUtil.noRepeatConsumer(record, acknowledgment, KafkaConfigConstant.KA_GROUP_ID,true, consumerRecord -> {
consumerMessage(record);
});
}