1、使用flink模拟生成数据:使用flink中datagen connector进行数据的模拟生成,同时将生成的数据写入到kafka中,模拟流式数据的生成;然后再使用flink对kafka中的数据进行消费,存入到数据湖iceberg中。
该流程类似与上一篇文章:kafka-flinkSQL-hudi的测试,有兴趣请参考上篇文章。
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
import java.util.Date;
public class CreateDataAll {
public static void main(String[] args) {
// datagen source 随机生成数据
final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
final StreamTableEnvironment tableEnv = StreamTableEnvironment.create(env);
String sourceDDL = "create table datagen_source (n" +
" id int,n" +
" data string,n" +
"price double,n" +
" ts as localtimestamp,n" +
" watermark for ts as tsn" +
") with (n" +
" 'connector' = 'datagen',n" +
" 'rows-per-second'='" + args[0] + "',n" + // 每秒生成多少条数据,传入参数
" 'fields.id.kind'='sequence',n" +
" 'fields.id.start'='" + args[1] + "',n" + // id的起始位置
" 'fields.id.end'='" + args[2] + "',n" + // id的结束位置
" 'fields.data.length'='50'n" +
")";
System.out.println(sourceDDL);
tableEnv.executeSql(sourceDDL);
// kafka sink
String sinkDDL = "create table kafka_sink (n" +
" id int,n" +
" data string,n" +
"price double,n" +
" ts timestampn" +
") with (n" +
" 'connector' = 'kafka',n" +
" 'topic' = '" + args[3] + "',n" +
" 'properties.bootstrap.servers' = '192.168.100.14:9092',n" +
" 'properties.group.id' = 'flinkToKafka3',n" +
" 'scan.startup.mode' = 'earliest-offset',n" +
" 'format' = 'json',n" +
" 'json.fail-on-missing-field' = 'false',n" +
" 'json.ignore-parse-errors' = 'true'n" +
")";
System.out.println(sinkDDL);
tableEnv.executeSql(sinkDDL);
// insert into kafka,其实就是利用flink的自己模拟生成的数据,来发送给kafka,模拟流数据id>1000000 or
String insertKafka = "insert into kafka_sinkn" +
"select * from datagen_source ";
System.out.println(insertKafka);
tableEnv.executeSql(insertKafka);
// print
final Date date_end = new Date();
System.out.println("起始时间:" + date_end.toString() + "*********************************************");
}
}
2、使用flink处理kafka数据实时写入iceberg
import org.apache.flink.streaming.api.environment.CheckpointConfig;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.table.api.TableResult;
import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
public class KafkaToIceberg {
public static void main(String[] args) {
System.setProperty("HADOOP_USER_NAME", "root");
System.setProperty("user.name", "root");
final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(4);
env.enableCheckpointing(6000);
env.getCheckpointConfig().setMinPauseBetweenCheckpoints(5000);
env.getCheckpointConfig().setCheckpointTimeout(60000);
env.getCheckpointConfig().setTolerableCheckpointFailureNumber(2);
env.getCheckpointConfig().setMaxConcurrentCheckpoints(1);
env.getCheckpointConfig().enableExternalizedCheckpoints(
CheckpointConfig.ExternalizedCheckpointCleanup.RETAIN_ON_CANCELLATION);
env.getCheckpointConfig().enableUnalignedCheckpoints();
env.getCheckpointConfig().setCheckpointStorage("hdfs://192.168.100.11:9000/tmp/hudi/testFlinkHudi/checkpoint/dir");
final StreamTableEnvironment tableEnv = StreamTableEnvironment.create(env);
// // kafka source
String kafkaSourceDDL = "create table kafka_source (n" +
" id int,n" +
" data string,n" +
"price double,n" +
" ts timestampn" +
") with (n" +
" 'connector' = 'kafka',n" +
" 'topic' = '" + args[0] + "',n" + // 消费topic
" 'properties.bootstrap.servers' = '192.168.100.14:9092',n" +
" 'properties.group.id' = 'flinkToKafka3',n" +
" 'scan.startup.mode' = 'earliest-offset',n" +
" 'format' = 'json',n" +
" 'json.fail-on-missing-field' = 'false',n" +
" 'json.ignore-parse-errors' = 'true'n" +
")";
tableEnv.executeSql(kafkaSourceDDL);
System.out.println(kafkaSourceDDL);
// 使用table api 创建 hadoop catalog
TableResult tableResult = tableEnv.executeSql("CREATE CATALOG hadoop_catalog WITH (n" +
" 'type'='iceberg',n" +
" 'catalog-type'='hadoop',n" +
" 'warehouse'='hdfs://192.168.100.11:9000/tmp/iceberg/flink_iceberg/FlinkClientTable/',n" +
" 'property-version'='2', n" +
" 'format-version'='2'n" +
")");
// 建立一张新表
String createHudiTable = "create table if not exists hadoop_catalog.iceberg_hadoop_db."+args[1]+" (n" +
" id int,n" +
" data string,n" +
" price double,n" +
" ts timestamp(3),n" +
" part int,n"+
" event_time string,n" +
" event_date string, n" +
" PRIMARY KEY (`id`) NOT ENFORCED n"+
") partitioned by ( part ) n" +
"WITH (n" +
" 'property-version'='2', n" +
" 'format-version'='2' n" +
")";
System.out.println(createHudiTable);
tableEnv.executeSql(createHudiTable);
// // 流式摄取到iceberg中
String insertDML = "insert into hadoop_catalog.iceberg_hadoop_db."+args[1]+" n" +
"selectn" +
" id,n" +
" data,n" +
" price,n " +
" ts,n" +
" id/100000 as part,n"+
" date_format(ts, 'yyyy-MM-dd HH:mm:ss') as event_time,n" +
" date_format(ts, 'yyyy-MM-dd') as event_daten" +
"from kafka_source";
tableEnv.executeSql(insertDML);
}
}
POM文件
4.0.0 org.example FlinkToIceberg 1.0-SNAPSHOT UTF-8 1.8 1.8 3.2.2 3.8.1 3.1.1 1.8 2.9.2 1.13.6 0.13.1 2.3.9 compile org.apache.flink flink-runtime-web_2.11 ${flink.version} ${scope.type} org.apache.flink flink-core ${flink.version} ${scope.type} org.apache.flink flink-table-common ${flink.version} ${scope.type} org.apache.flink flink-table-api-java-bridge_2.11 ${flink.version} ${scope.type} org.apache.flink flink-streaming-scala_2.11 ${flink.version} ${scope.type} org.apache.flink flink-streaming-java_2.11 ${flink.version} ${scope.type} org.apache.flink flink-table-planner-blink_2.11 ${flink.version} ${scope.type} org.apache.flink flink-clients_2.11 ${flink.version} ${scope.type} org.apache.flink flink-json ${flink.version} ${scope.type} org.apache.flink flink-connector-kafka_2.11 1.13.6 org.apache.hadoop hadoop-common ${hadoop.version} ${scope.type} org.apache.hadoop hadoop-hdfs ${hadoop.version} ${scope.type} org.apache.hadoop hadoop-client ${hadoop.version} ${scope.type} org.apache.iceberg iceberg-flink-runtime-1.13 0.13.1 org.apache.iceberg iceberg-api 0.13.1 org.apache.iceberg iceberg-data 0.13.1 org.apache.iceberg iceberg-flink 0.13.1 org.apache.maven.plugins maven-jar-plugin true libs/ KafkaToIceberg
谢谢观看,如果感兴趣flink-hudi案例测试,可以参考笔者之前的文章。



