消费者初始化流程图(整体流程图在上篇博客)
消费者用例
KafkaConsumer(ConsumerConfig config, DeserializerkeyDeserializer, Deserializer valueDeserializer) { try { //消费者组平衡 GroupRebalanceConfig groupRebalanceConfig = new GroupRebalanceConfig(config, GroupRebalanceConfig.ProtocolType.CONSUMER); //组id this.groupId = Optional.ofNullable(groupRebalanceConfig.groupId); //consumer和producer一样都是broker的client端 this.clientId = config.getString(CommonClientConfigs.CLIENT_ID_CONFIG); //...... //请求等待时间30S this.requestTimeoutMs = config.getInt(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG); this.defaultApiTimeoutMs = config.getInt(ConsumerConfig.DEFAULT_API_TIMEOUT_MS_CONFIG); this.time = Time.SYSTEM; this.metrics = buildMetrics(config, time, clientId); //重试时间 100毫秒 this.retryBackoffMs = config.getLong(ConsumerConfig.RETRY_BACKOFF_MS_CONFIG); //拦截器 List > interceptorList = (List) config.getConfiguredInstances( ConsumerConfig.INTERCEPTOR_CLASSES_CONFIG, ConsumerInterceptor.class, Collections.singletonMap(ConsumerConfig.CLIENT_ID_CONFIG, clientId)); this.interceptors = new ConsumerInterceptors<>(interceptorList); //反序列化 if (keyDeserializer == null) { this.keyDeserializer = config.getConfiguredInstance(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, Deserializer.class); this.keyDeserializer.configure(config.originals(Collections.singletonMap(ConsumerConfig.CLIENT_ID_CONFIG, clientId)), true); } else { config.ignore(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG); this.keyDeserializer = keyDeserializer; } if (valueDeserializer == null) { this.valueDeserializer = config.getConfiguredInstance(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, Deserializer.class); this.valueDeserializer.configure(config.originals(Collections.singletonMap(ConsumerConfig.CLIENT_ID_CONFIG, clientId)), false); } else { config.ignore(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG); this.valueDeserializer = valueDeserializer; } //offset消费策略 //latest 增量 默认 //earliest 最早 //none OffsetResetStrategy offsetResetStrategy = OffsetResetStrategy.valueOf(config.getString(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG).toUpperCase(Locale.ROOT)); this.subscriptions = new SubscriptionState(logContext, offsetResetStrategy); ClusterResourceListeners clusterResourceListeners = configureClusterResourceListeners(keyDeserializer, valueDeserializer, metrics.reporters(), interceptorList); //broker元数据,消费者需要直到从那个主题那个分区消费 this.metadata = new ConsumerMetadata(retryBackoffMs, config.getLong(ConsumerConfig.METADATA_MAX_AGE_CONFIG), //是否允许访问系统主题,默认TRUE不允许,取反为FALSE !config.getBoolean(ConsumerConfig.EXCLUDE_INTERNAL_TOPICS_CONFIG), //是都允许自动创建不存在的topic,默认TRUE允许 //建议关闭,如程序员输入错误会产生大量无效topic,导致运维成本增加,增加不可控性 config.getBoolean(ConsumerConfig.ALLOW_AUTO_CREATE_TOPICS_CONFIG), subscriptions, logContext, clusterResourceListeners); //连接kafka集群信息 List addresses = ClientUtils.parseAndValidateAddresses( config.getList(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG), config.getString(ConsumerConfig.CLIENT_DNS_LOOKUP_CONFIG)); this.metadata.bootstrap(addresses); String metricGrpPrefix = "consumer"; FetcherMetricsRegistry metricsRegistry = new FetcherMetricsRegistry(Collections.singleton(CLIENT_ID_METRIC_TAG), metricGrpPrefix); ChannelBuilder channelBuilder = ClientUtils.createChannelBuilder(config, time, logContext); this.isolationLevel = IsolationLevel.valueOf( config.getString(ConsumerConfig.ISOLATION_LEVEL_CONFIG).toUpperCase(Locale.ROOT)); Sensor throttleTimeSensor = Fetcher.throttleTimeSensor(metrics, metricsRegistry); int heartbeatIntervalMs = config.getInt(ConsumerConfig.HEARTBEAT_INTERVAL_MS_CONFIG); ApiVersions apiVersions = new ApiVersions(); //consumer和kafka集群交互必备 NetworkClient netClient = new NetworkClient( new Selector(config.getLong(ConsumerConfig.CONNECTIONS_MAX_IDLE_MS_CONFIG), metrics, time, metricGrpPrefix, channelBuilder, logContext), this.metadata, clientId, 100, //连接kafka集群重试时间50毫秒 config.getLong(ConsumerConfig.RECONNECT_BACKOFF_MS_CONFIG), //最大重试时间 1S,总的超时时间 config.getLong(ConsumerConfig.RECONNECT_BACKOFF_MAX_MS_CONFIG), //consumer发送缓冲区 128KB config.getInt(ConsumerConfig.SEND_BUFFER_CONFIG), //接收缓冲区 64KB config.getInt(ConsumerConfig.RECEIVE_BUFFER_CONFIG), //客户端请求kafka集群的等待时间 config.getInt(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG), config.getLong(ConsumerConfig.SOCKET_CONNECTION_SETUP_TIMEOUT_MS_CONFIG), config.getLong(ConsumerConfig.SOCKET_CONNECTION_SETUP_TIMEOUT_MAX_MS_CONFIG), time, true, apiVersions, throttleTimeSensor, logContext); //ConsumerNetClient封装 this.client = new ConsumerNetworkClient( logContext, netClient, metadata, time, retryBackoffMs, config.getInt(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG), heartbeatIntervalMs); //消费者分区分配策略 //range //轮询 //粘性 this.assignors = ConsumerPartitionAssignor.getAssignorInstances( config.getList(ConsumerConfig.PARTITION_ASSIGNMENT_STRATEGY_CONFIG), config.originals(Collections.singletonMap(ConsumerConfig.CLIENT_ID_CONFIG, clientId)) ); //每个消费者需要和协调器通信,每个消费者中也存在一个coordinator this.coordinator = !groupId.isPresent() ? null : new ConsumerCoordinator(groupRebalanceConfig, logContext, this.client, assignors, this.metadata, this.subscriptions, metrics, metricGrpPrefix, this.time, enableAutoCommit, //自动条offset 默认5S config.getInt(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG), this.interceptors, config.getBoolean(ConsumerConfig.THROW_ON_FETCH_STABLE_OFFSET_UNSUPPORTED)); //抓取数据 this.fetcher = new Fetcher<>( logContext, this.client, //最少抓取字节 默认1byte config.getInt(ConsumerConfig.FETCH_MIN_BYTES_CONFIG), //最大50M config.getInt(ConsumerConfig.FETCH_MAX_BYTES_CONFIG), //未抓取数据等待时间500毫秒 config.getInt(ConsumerConfig.FETCH_MAX_WAIT_MS_CONFIG), //单条数据抓取默认1M config.getInt(ConsumerConfig.MAX_PARTITION_FETCH_BYTES_CONFIG), //默认每次处理500条 config.getInt(ConsumerConfig.MAX_POLL_RECORDS_CONFIG), config.getBoolean(ConsumerConfig.CHECK_CRCS_CONFIG), config.getString(ConsumerConfig.CLIENT_RACK_CONFIG), this.keyDeserializer, this.valueDeserializer, this.metadata, this.subscriptions, metrics, metricsRegistry, this.time, this.retryBackoffMs, this.requestTimeoutMs, isolationLevel, apiVersions); this.kafkaConsumerMetrics = new KafkaConsumerMetrics(metrics, metricGrpPrefix); config.logUnused(); AppInfoParser.registerAppInfo(JMX_PREFIX, clientId, metrics, time.milliseconds()); log.debug("Kafka consumer initialized"); } catch (Throwable t) { // call close methods if internal objects are already constructed; this is to prevent resource leak. see KAFKA-2121 // we do not need to call `close` at all when `log` is null, which means no internal objects were initialized. if (this.log != null) { close(0, true); } // now propagate the exception throw new KafkaException("Failed to construct kafka consumer", t); } }
消费者主题订阅源码
public void subscribe(Collectiontopics, ConsumerRebalanceListener listener) { acquireAndEnsureOpen(); try { maybeThrowInvalidGroupIdException(); //判空 if (topics == null) throw new IllegalArgumentException("Topic collection to subscribe to cannot be null"); if (topics.isEmpty()) { this.unsubscribe(); } else { //循环所有的topic集合 for (String topic : topics) { if (Utils.isBlank(topic)) throw new IllegalArgumentException("Topic collection to subscribe to cannot contain null or empty topic"); } throwIfNoAssignorsConfigured(); fetcher.clearBufferedDataForUnassignedTopics(topics); log.info("Subscribed to topic(s): {}", Utils.join(topics, ", ")); //订阅操作 //listener主要监听消费者状态,某一个挂了需要再平衡 if (this.subscriptions.subscribe(new HashSet<>(topics), listener)) metadata.requestUpdateForNewTopics(); } } finally { release(); } }
consumer拉取数据源码
//初始化消费者组 //consumer和协调器建立连接 //拉取数据 //回调 //处理数据 private ConsumerRecordspoll(final Timer timer, final boolean includeMetadataInTimeout) { acquireAndEnsureOpen(); try { //....... do { client.maybeTriggerWakeup(); if (includeMetadataInTimeout) { // 消费者/组初始化 //当前consumer进组 //每个consumer和协调器建立通信 updateAssignmentMetadataIfNeeded(timer, false); } else { while (!updateAssignmentMetadataIfNeeded(time.timer(Long.MAX_VALUE), true)) { log.warn("Still waiting for metadata"); } } //抓取数据,按照分区TopicPartition final Map >> records = pollForFetches(timer); if (!records.isEmpty()) { if (fetcher.sendFetches() > 0 || client.hasPendingRequests()) { client.transmitSends(); } //抓取到了数据交给拦截器处理 return this.interceptors.onConsume(new ConsumerRecords<>(records)); } } while (timer.notExpired()); return ConsumerRecords.empty(); } finally { release(); this.kafkaConsumerMetrics.recordPollEnd(timer.currentTimeMs()); } } private Map >> pollForFetches(Timer timer) { // 判断是否有现成的数据,有直接返回 //第一次拉取不到 final Map >> records = fetcher.fetchedRecords(); if (!records.isEmpty()) { return records; } //发送拉取数据请求,开始拉取数据 //成功获取到数据后onSuccess回调 //consumer---->send----->kafka-cluster----->onSuccess //返回的数据放入队列中 completedFetches fetcher.sendFetches(); //....... //从completedFetches队列中拉取数据到consumer,默认500条 最大50M return fetcher.fetchedRecords(); } public Map >> fetchedRecords() { Map >> fetched = new HashMap<>(); Queue pausedCompletedFetches = new ArrayDeque<>(); //每次处理最大条数 500 int recordsRemaining = maxPollRecords; try { while (recordsRemaining > 0) { //....... //队列取数据 Fetcher.CompletedFetch records = completedFetches.peek(); //取完了,退出循环 if (records == null) break; //..... //拉取数据 completedFetches.poll(); //....... } } catch (KafkaException e) { if (fetched.isEmpty()) throw e; } finally { completedFetches.addAll(pausedCompletedFetches); } return fetched; }



