本文只作记录,原文在相应的后面都附上了。如有侵权,请联系进行删除。文章目录
- Kafka的assign和subscribe订阅模式和手动提交偏移量
- 前言
- 一、topic只有一个分区
- 二、topic有多分区
- 三、topic有多分区(自己改了下)
前言
使用Apache Kafka消费者组时,有一个为消费者分配对应分区partition的过程,可使用“自动”subscribe和“手动”assign的方式处理消息。 KafkaConsumer.subscribe():为consumer自动分配partition,有内部算法保证topic-partition以最优的方式均匀分配给同group下的不一样consumer。 KafkaConsumer.assign():为consumer手动、显示的指定须要消费的topic-partitions,不受group.id限制,至关与指定的group无效(this method does not use the consumer’s group management)。 注意: 1.consumer.assign()是不会被消费者的组管理功能管理的,他相对因而一个临时的,不会改变当前group.id的offset,好比:在使用consumer.subscribe(Arrays.asList(topicName));时offset为20,若是再经过assign方式已经获取了消息后,在下次经过consumer.subscribe(Arrays.asList(topicName));来获取消息时offset仍是20,仍是会获取20之后的消息。 2.若是两种模式同时使用的话会报错 3.需要引入依赖,pom文件配置如下:(手敲的,有问题请自行百度)
org.apache.kafka kafka-clients 0.9.0.0
一、topic只有一个分区
原文地址:http://www.javashuo.com/article/p-nsgkrmkb-ga.html
package com.guoxin.sydjtxry;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.boot.CommandLineRunner;
import org.springframework.stereotype.Component;
@Component
//实现CommandLineRunner接口,在springboot启动加载bean后自动运行其run方法。
public class KafkaConsumerTask implements CommandLineRunner {
private static final Logger LOG = LoggerFactory.getLogger(KafkaConsumerTask.class);
@Override
public void run(String... args) {
// 全量消费
SydjtxryConsumer.consumer();
}
}
package com.guoxin.sydjtxry;
import kafka.utils.ShutdownableThread;
import net.sf.json.JSONObject;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.apache.kafka.common.TopicPartition;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Arrays;
import java.util.Collections;
import java.util.Properties;
import java.util.UUID;
public class SydjtxryConsumer extends ShutdownableThread
{
private static final Logger LOG = LoggerFactory.getLogger(SydjtxryConsumer.class);
private final KafkaConsumer consumer;
private final String topic;
private Long rowcount = 0L;
// 一次请求的最大等待时间
private final int waitTime = 10000;
// consumer从指定的offset处理
private Long seekOffset = 1936170L;
// Broker链接地址
private final String bootstrapServers = "bootstrap.servers";
public SydjtxryConsumer(String topic)
{
super("KafkaConsumerExample", false);
Properties props = new Properties();
KafkaProperties kafkaProc = KafkaProperties.getInstance();
// Broker链接地址
props.put(bootstrapServers,
kafkaProc.getValues(bootstrapServers, "192.110.110.33:9092"));
props.put("enable.auto.commit", "true"); // 自动提交
props.put("auto.commit.interval.ms", "1000");
props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
props.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
consumer = new KafkaConsumer(props);
this.topic = topic;
}
public void doWork()
{
// 订阅
TopicPartition partition = new TopicPartition(this.topic, 0);
consumer.assign(Arrays.asList(partition));
consumer.seek(partition, seekOffset);
compareOffset = seekOffset;
// 消息消费请求
ConsumerRecords records = consumer.poll(waitTime);
if (records.isEmpty()) {
System.out.println("消费者没有消费到数据------->");
} else {
// 消息处理
for (ConsumerRecord record : records) {
try {
JSONObject jsondata = JSONObject.fromObject(record.value().toString());
String table = jsondata.getString("table"); // 库名.表名
if (table.equals("BJSX_OGG.GR_XX")) {
rowcount++;
// 业务逻辑
}
LOG.info("数据偏移量为-->" + record.offset());
} catch (Exception e) {
e.printStackTrace();
LOG.warn("偏移量为" + record.offset() + "的数据处理有问题,请排查-->" + record.value().toString());
}
seekOffset = record.offset();
if (seekOffset % 10000 == 0) {
LOG.info("offset-->" + seekOffset);
}
}
}
}
public static void consumer()
{
SydjtxryConsumer consumerThread = new SydjtxryConsumer("heheda");
consumerThread.start();
}
}
遇到的问题:当消费到该topic最后一条数据后,之后的消费会循环消费该数据。改进:当消费完最后一条数据,之后的订阅模式都由assign改成subscribe
package com.guoxin.sydjtxry;
import kafka.utils.ShutdownableThread;
import net.sf.json.JSONObject;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.apache.kafka.common.TopicPartition;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Arrays;
import java.util.Collections;
import java.util.Properties;
public class SydjtxryConsumer extends ShutdownableThread
{
private static final Logger LOG = LoggerFactory.getLogger(SydjtxryConsumer.class);
private final KafkaConsumer consumer;
private final String topic;
private Long rowcount = 0L;
// 一次请求的最大等待时间
private final int waitTime = 10000;
// consumer从指定的offset处理
private Long seekOffset = 1936170L;
private Long compareOffset = 0L;
private boolean flag = false;
// Broker链接地址
private final String bootstrapServers = "bootstrap.servers";
// Group id
private final String groupId = "group.id";
private StringBuilder grxxdata = new StringBuilder(); // 要批量插入gauss表中的数据
private StringBuilder grylzfdata = new StringBuilder(); // 要批量插入gauss表中的数据
public SydjtxryConsumer(String topic)
{
super("KafkaConsumerExample", false);
Properties props = new Properties();
KafkaProperties kafkaProc = KafkaProperties.getInstance();
// Broker链接地址
props.put(bootstrapServers,
kafkaProc.getValues(bootstrapServers, "192.110.110.33:9092"));
props.put("enable.auto.commit", "true"); // 自动提交
props.put("auto.commit.interval.ms", "1000");
props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
props.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
props.put("auto.offset.reset", "latest");
// Group id
props.put(groupId, UUID.randomUUID().toString());
consumer = new KafkaConsumer(props);
this.topic = topic;
}
public void doWork()
{
// 订阅
if (compareOffset.equals(seekOffset) && flag == false) {
// 暂停kafka的消费 暂停分区的分配
consumer.unsubscribe(); // 此处不取消订阅暂停过久会出现订阅超时的错误
consumer.pause(consumer.assignment());
consumer.subscribe(Collections.singletonList(this.topic));
flag = true;
} else if (flag == true) {
consumer.subscribe(Collections.singletonList(this.topic));
} else {
TopicPartition partition = new TopicPartition(this.topic, 0);
consumer.assign(Arrays.asList(partition));
consumer.seek(partition, seekOffset);
compareOffset = seekOffset;
}
// 消息消费请求
ConsumerRecords records = consumer.poll(waitTime);
if (compareOffset.equals(seekOffset + records.count() - 1) && flag == false) {
System.out.println("指定offset消费已结束,此条为末尾的重复消费数据,跳过业务处理,此后由assign改成subscribe订阅模式-->");
} else {
if (records.isEmpty()) {
System.out.println("消费者没有消费到数据------->");
} else {
// 消息处理
for (ConsumerRecord record : records) {
try {
JSONObject jsondata = JSONObject.fromObject(record.value().toString());
String table = jsondata.getString("table"); // 库名.表名
if (table.equals("BJSX_OGG.GR_XX")) {
// 业务逻辑
}
LOG.info("数据偏移量为-->" + record.offset());
} catch (Exception e) {
e.printStackTrace();
LOG.warn("偏移量为" + record.offset() + "的数据处理有问题,请排查-->" + record.value().toString());
}
seekOffset = record.offset();
if (seekOffset % 10000 == 0) {
LOG.info("offset-->" + seekOffset);
}
}
}
}
}
public static void consumer()
{
SydjtxryConsumer consumerThread = new SydjtxryConsumer("heheda");
consumerThread.start();
}
}
二、topic有多分区
原文地址:https://www.cnblogs.com/dongxishaonian/p/12038500.html
@Service
@Slf4j
public class TspLogbookAnalysisService implements CommandLineRunner {
//声明kafka分区数相等的消费线程数,一个分区对应一个消费线程
private static final int consumeThreadNum = 9;
//特殊指定每个分区开始消费的offset,这个地方可以优化,写个方法获取数据库中记录的offset,没有的话,写固定的值(例如:0)
private List partitionOffsets = Lists.newArrayList(1111,1112,1113,1114,1115,1116,1117,1118,1119);
private ExecutorService executorService = Executors.newFixedThreadPool(consumeThreadNum);
@Override
public void run(String... args) {
//循环遍历创建消费线程
IntStream.range(0, consumeThreadNum)
.forEach(partitionIndex -> executorService.submit(() -> startConsume(partitionIndex)));
}
}
kafka consumer对offset的处理(方法都是在TspLogbookAnalysisService 类里面的)
声明kafka consumer的配置类(并不是都需要配置,按需写) ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG需要配置为false,因为代码里是使用的手动提交
private Properties buildKafkaConfig() {
Properties kafkaConfiguration = new Properties();
kafkaConfiguration.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "");
kafkaConfiguration.put(ConsumerConfig.GROUP_ID_CONFIG, "");
kafkaConfiguration.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, "");
kafkaConfiguration.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, "");
kafkaConfiguration.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, "");
kafkaConfiguration.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, "");
kafkaConfiguration.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG,"");
kafkaConfiguration.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, false);
...更多配置项
return kafkaConfiguration;
}
创建kafka consumer,处理offset,开始消费数据任务
private void startConsume(int partitionIndex) {
//创建kafka consumer
KafkaConsumer consumer = new KafkaConsumer<>(buildKafkaConfig());
try {
//指定该consumer对应的消费分区
TopicPartition partition = new TopicPartition(kafkaProperties.getKafkaTopic(), partitionIndex);
consumer.assign(Lists.newArrayList(partition));
//consumer的offset处理
if (collectionUtils.isNotEmpty(partitionOffsets) && partitionOffsets.size() == consumeThreadNum) {
Long seekOffset = partitionOffsets.get(partitionIndex);
log.info("partition:{} , offset seek from {}", partition, seekOffset);
consumer.seek(partition, seekOffset);
}
//开始消费数据任务
kafkaRecordConsume(consumer, partition);
} catch (Exception e) {
log.error("kafka consume error:{}", ExceptionUtils.getFullStackTrace(e));
} finally {
try {
consumer.commitSync();
} finally {
consumer.close();
}
}
}
消费数据逻辑,offset操作
private void kafkaRecordConsume(KafkaConsumerconsumer, TopicPartition partition) { while (true) { try { ConsumerRecords records = consumer.poll(TspLogbookConstants.POLL_TIMEOUT); //具体的处理流程 records.forEach((k) -> handleKafkaInput(k.key(), k.value())); //很重要:日志记录当前consumer的offset,partition相关信息(之后如需重新指定offset消费就从这里的日志中获取offset,partition信息) if (records.count() > 0) { String currentOffset = String.valueOf(consumer.position(partition)); log.info("current records size is:{}, partition is: {}, offset is:{}", records.count(), consumer.assignment(), currentOffset); } //offset提交 consumer.commitAsync(); } catch (Exception e) { log.error("handlerKafkaInput error{}", ExceptionUtils.getFullStackTrace(e)); } } }
三、topic有多分区(自己改了下)
因为本人脑海里没有byte[]转为对象的方法,所以将获取的records 由ConsumerRecords
@Service
@Slf4j
public class TspLogbookAnalysisService implements CommandLineRunner {
@Autowird
private MongoTemplate mongoTemplate;
//声明kafka分区数相等的消费线程数,一个分区对应一个消费线程
private static final int consumeThreadNum = 9;
//特殊指定每个分区开始消费的offset,这个地方可以优化,写个方法获取数据库中记录的offset,没有的话,写固定的值(例如:0)
private List partitionOffsets = Lists.newArrayList(1111,1112,1113,1114,1115,1116,1117,1118,1119);
private ExecutorService executorService = Executors.newFixedThreadPool(consumeThreadNum);
@Override
public void run(String... args) {
//循环遍历创建消费线程
IntStream.range(0, consumeThreadNum)
.forEach(partitionIndex -> executorService.submit(() -> startConsume(partitionIndex)));
}
}
kafka consumer对offset的处理(方法都是在TspLogbookAnalysisService 类里面的)
声明kafka consumer的配置类(并不是都需要配置,按需写) ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG需要配置为false,因为代码里是使用的手动提交
private Properties buildKafkaConfig() {
Properties kafkaConfiguration = new Properties();
kafkaConfiguration.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "");
kafkaConfiguration.put(ConsumerConfig.GROUP_ID_CONFIG, "");
kafkaConfiguration.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, "");
kafkaConfiguration.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, "");
kafkaConfiguration.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, "");
kafkaConfiguration.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, "");
kafkaConfiguration.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG,"");
kafkaConfiguration.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "");
...更多配置项
return kafkaConfiguration;
}
创建kafka consumer,处理offset,开始消费数据任务
private void startConsume(int partitionIndex) {
//创建kafka consumer
ConsumerRecords consumer = new KafkaConsumer<>(buildKafkaConfig());
try {
//指定该consumer对应的消费分区
TopicPartition partition = new TopicPartition(kafkaProperties.getKafkaTopic(), partitionIndex);
consumer.assign(Lists.newArrayList(partition));
//consumer的offset处理
if (collectionUtils.isNotEmpty(partitionOffsets) && partitionOffsets.size() == consumeThreadNum) {
Long seekOffset = partitionOffsets.get(partitionIndex);
log.info("partition:{} , offset seek from {}", partition, seekOffset);
consumer.seek(partition, seekOffset);
}
//开始消费数据任务
kafkaRecordConsume(consumer, partition);
} catch (Exception e) {
log.error("kafka consume error:{}", ExceptionUtils.getFullStackTrace(e));
} finally {
try {
consumer.commitSync();
} finally {
consumer.close();
}
}
}
消费数据逻辑,offset操作
private void kafkaRecordConsume(ConsumerRecordsconsumer, TopicPartition partition) { while (true) { try { ConsumerRecords records = consumer.poll(TspLogbookConstants.POLL_TIMEOUT); //具体的处理流程 这个地方我也改了,方便我去debug //records.forEach((k) -> handleKafkaInput(k.key(), k.value())); for(ConsumerRecord record : records){ log.info("record info :"+record); //然后进行其他业务操作 mongoTemlate.upsert(query,update,entityClass.class ) } //很重要:日志记录当前consumer的offset,partition相关信息(之后如需重新指定offset消费就从这里的日志中获取offset,partition信息) 可以将信息存入数据库中 if (records.count() > 0) { String currentOffset = String.valueOf(consumer.position(partition)); log.info("current records size is:{}, partition is: {}, offset is:{}", records.count(), consumer.assignment(), currentOffset); } //offset提交 consumer.commitAsync(); } catch (Exception e) { log.error("handlerKafkaInput error{}", ExceptionUtils.getFullStackTrace(e)); } } }



