栏目分类:
子分类:
返回
名师互学网用户登录
快速导航关闭
当前搜索
当前分类
子分类
实用工具
热门搜索
名师互学网 > IT > 软件开发 > 后端开发 > Java

flinkSQL-iceberg测试案例

Java 更新时间: 发布时间: IT归档 最新发布 模块sitemap 名妆网 法律咨询 聚返吧 英语巴士网 伯小乐 网商动力

flinkSQL-iceberg测试案例

kafka-flinkSQL-iceberg测试案例

使用flink中datagen connector进行数据的模拟生成,同时将生成的数据写入到kafka中,模拟流式数据的生成;然后再使用flink对kafka中的数据进行消费,存入到数据湖iceberg中。
该流程类似与上一篇文章:kafka-flinkSQL-hudi的测试,有兴趣请参考上篇文章。

1、使用flink模拟生成数据:
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;

import java.util.Date;


public class CreateDataAll {
    public static void main(String[] args) {


        // datagen source 随机生成数据

        final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        final StreamTableEnvironment tableEnv = StreamTableEnvironment.create(env);

        String sourceDDL = "create table datagen_source (n" +
                " id int,n" +
                " data string,n" +
                "price double,n" +
                " ts as localtimestamp,n" +
                " watermark for ts as tsn" +
                ") with (n" +
                " 'connector' = 'datagen',n" +
                " 'rows-per-second'='" + args[0] + "',n" +  // 每秒生成多少条数据,传入参数
                " 'fields.id.kind'='sequence',n" +
                " 'fields.id.start'='" + args[1] + "',n" +  // id的起始位置
                " 'fields.id.end'='" + args[2] + "',n" +  // id的结束位置
                " 'fields.data.length'='50'n" +
                ")";
        System.out.println(sourceDDL);
        tableEnv.executeSql(sourceDDL);



        // kafka sink
        String sinkDDL = "create table kafka_sink (n" +
                " id int,n" +
                " data string,n" +
                "price double,n" +
                " ts timestampn" +
                ") with (n" +
                " 'connector' = 'kafka',n" +
                " 'topic' = '" + args[3] + "',n" +
                " 'properties.bootstrap.servers' = '192.168.100.14:9092',n" +
                " 'properties.group.id' = 'flinkToKafka3',n" +
                " 'scan.startup.mode' = 'earliest-offset',n" +
                " 'format' = 'json',n" +
                " 'json.fail-on-missing-field' = 'false',n" +
                " 'json.ignore-parse-errors' = 'true'n" +
                ")";
        System.out.println(sinkDDL);
        tableEnv.executeSql(sinkDDL);

        // insert into kafka,其实就是利用flink的自己模拟生成的数据,来发送给kafka,模拟流数据id>1000000 or
        String insertKafka = "insert into kafka_sinkn" +
                "select * from datagen_source ";
        System.out.println(insertKafka);
        tableEnv.executeSql(insertKafka);

        // print

        final Date date_end = new Date();
        System.out.println("起始时间:" + date_end.toString() + "*********************************************");


    }
}
2、使用flink处理kafka数据实时写入iceberg
import org.apache.flink.streaming.api.environment.CheckpointConfig;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.table.api.TableResult;
import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;


public class KafkaToIceberg {
    public static void main(String[] args) {

        System.setProperty("HADOOP_USER_NAME", "root");
        System.setProperty("user.name", "root");
        final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism(4);
        env.enableCheckpointing(6000);
        env.getCheckpointConfig().setMinPauseBetweenCheckpoints(5000);
        env.getCheckpointConfig().setCheckpointTimeout(60000);
        env.getCheckpointConfig().setTolerableCheckpointFailureNumber(2);
        env.getCheckpointConfig().setMaxConcurrentCheckpoints(1);

        env.getCheckpointConfig().enableExternalizedCheckpoints(
                CheckpointConfig.ExternalizedCheckpointCleanup.RETAIN_ON_CANCELLATION);

        env.getCheckpointConfig().enableUnalignedCheckpoints();
        env.getCheckpointConfig().setCheckpointStorage("hdfs://192.168.100.11:9000/tmp/hudi/testFlinkHudi/checkpoint/dir");

        final StreamTableEnvironment tableEnv = StreamTableEnvironment.create(env);


//        // kafka source
        String kafkaSourceDDL = "create table kafka_source (n" +
                " id int,n" +
                " data string,n" +
                "price double,n" +
                " ts timestampn" +
                ") with (n" +
                " 'connector' = 'kafka',n" +
                " 'topic' = '" + args[0] + "',n" +  // 消费topic
                " 'properties.bootstrap.servers' = '192.168.100.14:9092',n" +
                " 'properties.group.id' = 'flinkToKafka3',n" +
                " 'scan.startup.mode' = 'earliest-offset',n" +
                " 'format' = 'json',n" +
                " 'json.fail-on-missing-field' = 'false',n" +
                " 'json.ignore-parse-errors' = 'true'n" +
                ")";
        tableEnv.executeSql(kafkaSourceDDL);
        System.out.println(kafkaSourceDDL);


        // 使用table api 创建 hadoop catalog
        TableResult tableResult = tableEnv.executeSql("CREATE CATALOG hadoop_catalog WITH (n" +
                "  'type'='iceberg',n" +
                "  'catalog-type'='hadoop',n" +
                "  'warehouse'='hdfs://192.168.100.11:9000/tmp/iceberg/flink_iceberg/FlinkClientTable/',n" +
                "  'property-version'='2', n" +
                "  'format-version'='2'n" +
                ")");



        // 建立一张新表
        String createHudiTable = "create table if not exists hadoop_catalog.iceberg_hadoop_db."+args[1]+" (n" +
                " id int,n" +
                " data string,n" +
                " price double,n" +
                " ts timestamp(3),n" +
                " part int,n"+
                " event_time string,n" +
                " event_date string, n" +
                " PRIMARY KEY (`id`) NOT ENFORCED n"+
                ") partitioned by ( part ) n" +
                "WITH (n" +
                " 'property-version'='2', n" +  
                " 'format-version'='2' n" + 
                ")";
        System.out.println(createHudiTable);
        tableEnv.executeSql(createHudiTable);

//        // 流式摄取到iceberg中
        String insertDML = "insert into hadoop_catalog.iceberg_hadoop_db."+args[1]+" n" +
                "selectn" +
                " id,n" +
                " data,n" +
                " price,n " +
                " ts,n" +
                " id/100000 as part,n"+
                " date_format(ts, 'yyyy-MM-dd HH:mm:ss') as event_time,n" +
                " date_format(ts, 'yyyy-MM-dd') as event_daten" +
                "from kafka_source";
        tableEnv.executeSql(insertDML);
    }
}
POM文件


    4.0.0

    org.example
    FlinkToIceberg
    1.0-SNAPSHOT

    
        
        UTF-8
        1.8
        1.8
        
        3.2.2
        3.8.1
        3.1.1
        
        1.8
        
        2.9.2
        1.13.6
        0.13.1
        
        
        2.3.9

        
        compile
    

    

        
        
            org.apache.flink
            flink-runtime-web_2.11
            ${flink.version}
            ${scope.type}
        

        
        
            org.apache.flink
            flink-core
            ${flink.version}
            ${scope.type}
        

        
        
            org.apache.flink
            flink-table-common
            ${flink.version}
            ${scope.type}
        
        
        
            org.apache.flink
            flink-table-api-java-bridge_2.11
            ${flink.version}
            ${scope.type}
        
        
        
            org.apache.flink
            flink-streaming-scala_2.11
            ${flink.version}
            ${scope.type}
        

        
            org.apache.flink
            flink-streaming-java_2.11
            ${flink.version}
            ${scope.type}
        

        
            org.apache.flink
            flink-table-planner-blink_2.11
            ${flink.version}
            ${scope.type}
        

        
            org.apache.flink
            flink-clients_2.11
            ${flink.version}
            ${scope.type}
        

        
            org.apache.flink
            flink-json
            ${flink.version}
            ${scope.type}
        

        
            org.apache.flink
            flink-connector-kafka_2.11
            1.13.6
        

        
        
            org.apache.hadoop
            hadoop-common
            ${hadoop.version}
            ${scope.type}
        
        
            org.apache.hadoop
            hadoop-hdfs
            ${hadoop.version}
            ${scope.type}
        
        
            org.apache.hadoop
            hadoop-client
            ${hadoop.version}
            ${scope.type}
        

        
        
            org.apache.iceberg
            iceberg-flink-runtime-1.13
            0.13.1
        


        
        
            org.apache.iceberg
            iceberg-api
            0.13.1
        

        
        
            org.apache.iceberg
            iceberg-data
            0.13.1
        

        
            org.apache.iceberg
            iceberg-flink
            0.13.1
        

    

    
        
            
            
                org.apache.maven.plugins
                maven-jar-plugin
                
                    
                        
                            true
                            libs/
                            
                                KafkaToIceberg
                            
                        
                    
                
            
        
    


谢谢观看,如果感兴趣flink-hudi案例测试,可以参考笔者之前的文章。

转载请注明:文章转载自 www.mshxw.com
本文地址:https://www.mshxw.com/it/935916.html
我们一直用心在做
关于我们 文章归档 网站地图 联系我们

版权所有 (c)2021-2022 MSHXW.COM

ICP备案号:晋ICP备2021003244-6号