栏目分类:
子分类:
返回
名师互学网用户登录
快速导航关闭
当前搜索
当前分类
子分类
实用工具
热门搜索
名师互学网 > IT > 软件开发 > 后端开发 > Java

flume-kafka-flink项目

Java 更新时间: 发布时间: IT归档 最新发布 模块sitemap 名妆网 法律咨询 聚返吧 英语巴士网 伯小乐 网商动力

flume-kafka-flink项目

package com.hao;

import java.util.Random;

public class Main {

    public static void main(String[] args) {
	// write your code here
        FileThread fileThread = new FileThread(args[0]);
        Thread thread = new Thread(fileThread);
        thread.start();
    }
}

 

package com.hao;

import java.io.BufferedOutputStream;
import java.io.FileOutputStream;
import java.nio.charset.StandardCharsets;
import java.util.HashMap;
import java.util.Random;

public class FileThread implements Runnable{

    private Random random=null;
    public HashMap provinces =null;
    private FileOutputStream fileOutputStream=null;
    private BufferedOutputStream bufferedOutputStream=null;

    public FileThread(String path) throws Exception{
        random = new Random();
        initProvinces();
        fileOutputStream = new FileOutputStream(path);
        bufferedOutputStream = new BufferedOutputStream(fileOutputStream,1024);
    }
    private void initProvinces(){
        provinces = new HashMap<>();
        provinces.put("beijing",new String[]{"dongcheng","xichang","chaoyang"
                ,"haidian","fengtai","daxing"});
        provinces.put("hebei",new String[]{"xingtai","handan","langfang","zhangjiakou","dangshan"});
        provinces.put("shanxia",new String[]{"taiyuan","datong","hongtong","linfen"});
        provinces.put("henan",new String[]{"zhengzhou","sanmenxia","anyang","nanyang","zhumadian"});
        provinces.put("shandong",new String[]{"jinan","qingdao","yantai","rizhao","liaocheng","linqing"});
    }

    @Override
    public void run() {
        // 543462,1715,beijing,beijing,1511658000
        while (true) {
            StringBuffer buffer = new StringBuffer();
            int userId = random.nextInt(10000);
            buffer.append(userId);
            buffer.append(",");

            int adId = random.nextInt(30);
            buffer.append(adId);
            buffer.append(",");

            int idx = random.nextInt(provinces.keySet().size());
            String province = provinces.keySet().toArray()[idx].toString();
            buffer.append(province);
            buffer.append(",");

            String[] cities = provinces.get(province);
            idx = random.nextInt(cities.length);
            String city = cities[idx];
            buffer.append(city);
            buffer.append(",");

            buffer.append(System.currentTimeMillis());
            String newLine = System.getProperty("line.separator");//获取当前系统的换行符
            buffer.append(newLine);

            try {
                bufferedOutputStream.write(buffer.toString().getBytes(StandardCharsets.UTF_8));
//                bufferedOutputStream.flush();
                Thread.sleep(300);
            }
            catch (Exception ex){}
        }

    }

    @Override
    protected void finalize() throws Throwable {
        bufferedOutputStream.close();
        fileOutputStream.close();
    }
}

开启虚拟机

开启环境

我是在dragon01下创建的生产者主题,上传jar在02

在02下创建flume配置文件

在03消费数据

创建kafka分区province

kafka-topics.sh --create --bootstrap-server dragon01:9092 --topic province --partitions 3 --replication-factor 2

启动消费者

kafka-console-consumer.sh --bootstrap-server dragon01:9092 --topic province

切换到flume/job下

vim file-flume-kafka.conf

a1.sources=r1
a1.channels=c1

a1.sources.r1.type=exec
a1.sources.r1.command=tail -F /home/flink/adclick.log

a1.channels.c1.type=org.apache.flume.channel.kafka.KafkaChannel
a1.channels.c1.kafka.bootstrap.servers=flink01:9092,flink02:9092
a1.channels.c1.parseAsFlumeEvent=false
a1.channels.c1.kafka.topic=province

a1.sources.r1.channels=c1

在bin目录下启动

bin/flume-ng agent -c conf -f job/file-flume-kafka.conf -n a1

启动jar包文件

java -cp ./mock.jar com.hao.Main ./adclick.log

等待几秒钟

采集到的数据

写flink代码

package hao.day20220606;



import hao.bean.AdClickLog;
import org.apache.flink.api.common.serialization.SimpleStringSchema;
import org.apache.flink.api.common.typeinfo.Types;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer;
import org.apache.kafka.clients.consumer.ConsumerConfig;

import java.util.Properties;

public class Flume_Kafka_Flink {
    public static void main(String[] args) throws Exception {
        //统计各省各市各广告点击次数
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism(3);
        Properties props = new Properties();
        props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG,"dragon01:9092,dragon03:9092");
        props.put(ConsumerConfig.GROUP_ID_CONFIG,"adclick");
        //偏移量
        props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG,"earliest");
        FlinkKafkaConsumer kafkaConsumer =
                new FlinkKafkaConsumer<>("province", new SimpleStringSchema(), props);
        DataStreamSource kafkaSource = env.addSource(kafkaConsumer);

        SingleOutputStreamOperator> map = kafkaSource.map(line -> {
            AdClickLog adClickLog = new AdClickLog(line);
            return Tuple2.of(adClickLog.getProvince() + "-"+adClickLog.getCity()+"-"
                    + adClickLog.getAdId(), 1);
        }).returns(Types.TUPLE(Types.STRING, Types.INT));

        map.keyBy(key -> key.f0).sum(1).print();
        env.execute();
    }
}

 

 成功!!!!!!!!!!!!!

转载请注明:文章转载自 www.mshxw.com
本文地址:https://www.mshxw.com/it/970801.html
我们一直用心在做
关于我们 文章归档 网站地图 联系我们

版权所有 (c)2021-2022 MSHXW.COM

ICP备案号:晋ICP备2021003244-6号