- Flink消费Kafka插入Clickhouse
- 一、Maven依赖
- 二、Job类
- 二、Kafka FlatMap算子
- 三、Clickhouse FlatMap算子
- 四、Clickhouse建表
一、Maven依赖
二、Job类org.apache.flink flink-clients_2.11 1.13.1 org.apache.flink flink-java 1.13.1 org.apache.flink flink-streaming-java_2.11 1.13.1 org.apache.kafka kafka-clients 3.1.0 org.apache.flink flink-connector-kafka_2.11 1.13.1 org.slf4j slf4j-simple 1.7.36 com.alibaba fastjson 2.0.3 ru.yandex.clickhouse clickhouse-jdbc 0.3.2
public static void main(String[] args) {
try {
Properties properties = new Properties();
properties.setProperty(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, BROKERS);
properties.setProperty(ConsumerConfig.GROUP_ID_CONFIG, GROUP_ID);
//自动检测topic分区变化时间间隔
properties.put("flink.partition-discovery.interval-millis", INTERVAL_MILLIS);
properties.put("refresh.leader.backoff.ms", REFRESH_MS);
KafkaSource kafkaSource = KafkaSource.builder()
.setTopics(TOPIC)
.setValueOnlyDeserializer(new SimpleStringSchema())
.setProperties(properties)
.setStartingOffsets(OffsetsInitializer.latest())
.build();
final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.enableCheckpointing(120000L, CheckpointingMode.EXACTLY_ONCE);
env.getCheckpointConfig().setMinPauseBetweenCheckpoints(180000L);
env.getConfig().setAutoWatermarkInterval(0);
env.fromSource(kafkaSource, WatermarkStrategy.noWatermarks(), "kafkaSource_sourceName")
.uid("kafkaSource_sourceName_uid").name("kafkaSource_sourceName_name").setParallelism(1)
.flatMap(new KafkaSourceFlatMap()).uid("KafkaSourceFlatMap_uid").name("KafkaSourceFlatMap_name").setParallelism(1)
.flatMap(new ClickHouseSinkFlatMap(Constants.PARALL)).uid("ClickHouseSinkFlatMap_uid").name("ClickHouseSinkFlatMap_name").setParallelism(1);
env.execute("KafkaToClickHouseJob");
} catch (Exception e) {
LOG.error("KafkaToClickHouseJob启动失败!", e);
}
}
二、Kafka FlatMap算子
public class KafkaSourceFlatMap extends RichFlatMapFunction三、Clickhouse FlatMap算子> { private static final Logger log = LoggerFactory.getLogger(KafkaSourceFlatMap.class); @Override public void flatMap(String str, Collector
public class ClickHouseSinkFlatMap extends RichFlatMapFunction四、Clickhouse建表
-- 本地表
CREATE TABLE IF NOT EXISTS rawdata.user_tag_store_local ON CLUSTER default
(
user_id String,
event_type String,
label_name String,
label_value String,
platform_type String,
recv_date Date,
recv_time DateTime64(3, 'Asia/Istanbul')
)
ENGINE = MergeTree()
PARTITION BY recv_date
ORDER BY recv_time
--分布式表
CREATE TABLE IF NOT EXISTS rawdata.user_tag_store ON CLUSTER default
AS rawdata.user_tag_store_local
Engine = Distributed("default" , "rawdata" , "user_tag_store_local" , rand());



