用spark做一个流数据处理框架(spark streaming flink)

大数据系列-SPARK-STREAMING流数据queue

package com.test

import org.apache.spark.SparkConf
import org.apache.spark.rdd.RDD
import org.apache.spark.streaming.{Seconds, StreamingContext}

import scala.collection.mutable

object SparkStreamingQueue {

  def main(args: Array[String]): Unit = {

    val sparkConf = new SparkConf().setAppName("SparkStreamingQueue").setMaster("local[*]")
    val streamingContext = new StreamingContext(sparkConf, Seconds(5))

    val rddQueue = new mutable.Queue[RDD[Int]]()

    streamingContext.queueStream(rddQueue, oneAtATime = false)
      .map((_, 1))
      .reduceByKey(_ + _)
      .print()

    streamingContext.start()

    for (k <- 0 to 5000) {
      rddQueue += streamingContext.sparkContext.makeRDD(1 to 300, 10)
      Thread.sleep(2000)
    }

    streamingContext.awaitTermination()

  }

}