栏目分类:
子分类:
返回
名师互学网用户登录
快速导航关闭
当前搜索
当前分类
子分类
实用工具
热门搜索
名师互学网 > IT > 软件开发 > 后端开发 > Java

Flink消费Kafka插入Clickhouse

Java 更新时间: 发布时间: IT归档 最新发布 模块sitemap 名妆网 法律咨询 聚返吧 英语巴士网 伯小乐 网商动力

Flink消费Kafka插入Clickhouse

Flink消费Kafka插入Clickhouse

目录
  • Flink消费Kafka插入Clickhouse
  • 一、Maven依赖
  • 二、Job类
  • 二、Kafka FlatMap算子
  • 三、Clickhouse FlatMap算子
  • 四、Clickhouse建表


一、Maven依赖
        
            org.apache.flink
            flink-clients_2.11
            1.13.1
        

        
            org.apache.flink
            flink-java
            1.13.1
        

        
            org.apache.flink
            flink-streaming-java_2.11
            1.13.1
        

        
            org.apache.kafka
            kafka-clients
            3.1.0
        

        
            org.apache.flink
            flink-connector-kafka_2.11
            1.13.1
        

        
            org.slf4j
            slf4j-simple
            1.7.36
        

        
        
            com.alibaba
            fastjson
            2.0.3
        

        
            ru.yandex.clickhouse
            clickhouse-jdbc
            0.3.2
        
二、Job类
 public static void main(String[] args) {
        try {
            
            Properties properties = new Properties();

            properties.setProperty(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, BROKERS);
            properties.setProperty(ConsumerConfig.GROUP_ID_CONFIG, GROUP_ID);
            //自动检测topic分区变化时间间隔
            properties.put("flink.partition-discovery.interval-millis", INTERVAL_MILLIS);
            properties.put("refresh.leader.backoff.ms", REFRESH_MS);

            
            KafkaSource kafkaSource = KafkaSource.builder()
                    .setTopics(TOPIC)
                    .setValueOnlyDeserializer(new SimpleStringSchema())
                    .setProperties(properties)
                    .setStartingOffsets(OffsetsInitializer.latest())
                    .build();

            
            final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
            env.enableCheckpointing(120000L, CheckpointingMode.EXACTLY_ONCE);
            env.getCheckpointConfig().setMinPauseBetweenCheckpoints(180000L);
            env.getConfig().setAutoWatermarkInterval(0);

            
            env.fromSource(kafkaSource, WatermarkStrategy.noWatermarks(), "kafkaSource_sourceName")
                    .uid("kafkaSource_sourceName_uid").name("kafkaSource_sourceName_name").setParallelism(1)

                    .flatMap(new KafkaSourceFlatMap()).uid("KafkaSourceFlatMap_uid").name("KafkaSourceFlatMap_name").setParallelism(1)
                    .flatMap(new ClickHouseSinkFlatMap(Constants.PARALL)).uid("ClickHouseSinkFlatMap_uid").name("ClickHouseSinkFlatMap_name").setParallelism(1);


            env.execute("KafkaToClickHouseJob");
        } catch (Exception e) {
            LOG.error("KafkaToClickHouseJob启动失败!", e);
        }
    }
二、Kafka FlatMap算子
public class KafkaSourceFlatMap extends RichFlatMapFunction> {
    private static final Logger log = LoggerFactory.getLogger(KafkaSourceFlatMap.class);

    @Override
    public void flatMap(String str, Collector> out) {
        try {
            //CK字段初始化
            String user_id;                                 //用户id
            String event_type;                              //时间类型
            String label_name;                              //标签名称
            String label_value;                             //标签值
            String platform_type;                           //平台类型
            String recv_timestamp;                          //接收日期
            String recv_date;                               //接收日期
            String recv_time;                               //接收时间
            String properties;                              //属性

			//清洗逻辑...
			
            out.collect(listToMap(user_id.trim(),
                    event_type.trim(),
                    label_name.trim(),
                    label_value.trim(),
                    platform_type.trim(),
                    recv_date,
                    recv_time));

    }
    
    public Map listToMap(String... str) {
        Map resMap = new HashMap<>();
        for (int i = 0; i < str.length; i++) {
            resMap.put((i + 1) + Constants.NULL, str[i]);
        }
        return resMap;
    }
} 
三、Clickhouse FlatMap算子
public class ClickHouseSinkFlatMap extends RichFlatMapFunction, String> {

    private static final Logger LOG = LoggerFactory.getLogger(ClickHouseSinkFlatMap.class);
    private int batch_size;
    private int count = 0;
    private int total = 0;
    private ClickHouseConnection connection = null;
    private PreparedStatement preparedStatement = null;

    public ClickHouseSinkFlatMap(int batch_size) {
        this.batch_size = batch_size;
    }

    @Override
    public void flatMap(Map map, Collector collector) {
        try {

            if (connection == null) {
                connection = getConn();

                preparedStatement = connection.prepareStatement(Constants.INSERT_SQL);
            }

            for (int i = 1; i < map.size()+1; i++) {
                    preparedStatement.setString(i, map.get(i + Constants.NULL));
            }

            preparedStatement.addBatch();

            count = count + 1;
            try {
                if (count >= batch_size) {
                    preparedStatement.executeBatch();

                    total += count;
                    LOG.info("已经成功插入{}条数据!!!",total);

                    preparedStatement.clearBatch();
                    count = 0;
                }
            } catch (Exception ee) {
                LOG.error("数据插入clickhouse 报错:", ee);
            }
        } catch (Exception ex) {
            LOG.error("ClickhouseSink插入报错====", ex);
        }
    }

    public static ClickHouseConnection getConn() {

        int socketTimeout = 600000;
        ClickHouseProperties properties = new ClickHouseProperties();
        properties.setUser(Constants.USERNAME);
        properties.setPassword(Constants.PASSWORD);
        properties.setDatabase(Constants.DATABASE);
        properties.setMaxRetries(3);
        properties.setSocketTimeout(socketTimeout);
        ClickHouseDataSource clickHouseDataSource = new ClickHouseDataSource(Constants.URL, properties);
        ClickHouseConnection conn = null;
        try {
            conn = clickHouseDataSource.getConnection();
            return conn;
        } catch (SQLException e) {
        }
        return null;
    }

}
四、Clickhouse建表
-- 本地表
CREATE TABLE IF NOT EXISTS rawdata.user_tag_store_local ON CLUSTER default
(
    user_id String,
    event_type String,
    label_name String,
    label_value String,
    platform_type String,
    recv_date Date,
    recv_time DateTime64(3, 'Asia/Istanbul')
)
ENGINE = MergeTree()
PARTITION BY recv_date
ORDER BY recv_time


--分布式表
CREATE TABLE IF NOT EXISTS rawdata.user_tag_store ON CLUSTER default
AS rawdata.user_tag_store_local
Engine = Distributed("default" , "rawdata" , "user_tag_store_local" , rand());

转载请注明:文章转载自 www.mshxw.com
本文地址:https://www.mshxw.com/it/940198.html
我们一直用心在做
关于我们 文章归档 网站地图 联系我们

版权所有 (c)2021-2022 MSHXW.COM

ICP备案号:晋ICP备2021003244-6号