类似于java的三层架构
- controller
- service
- dao
以下以wordcount程序简单模拟spark程序架构,以下只是一个习惯,实际可以在大结构下调整。
package com.atguigu.bigdata.spark.core.rdd.operator.transform
import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}
object Spark24_RDD_Req {
def main(args: Array[String]): Unit = {
val sparkConf = new SparkConf().setMaster("local[*]").setAppName("Operator")
val sc = new SparkContext(sparkConf)
// TODO 案例实操
// 1. 获取原始数据:时间戳,省份,城市,用户,广告
val dataRDD = sc.textFile("datas/agent.log")
// 2. 将原始数据进行结构的转换。方便统计
// 时间戳,省份,城市,用户,广告
// =>
// ( ( 省份,广告 ), 1 )
val mapRDD = dataRDD.map(
line => {
val datas = line.split(" ")
(( datas(1), datas(4) ), 1)
}
)
// 3. 将转换结构后的数据,进行分组聚合
// ( ( 省份,广告 ), 1 ) => ( ( 省份,广告 ), sum )
val reduceRDD: RDD[((String, String), Int)] = mapRDD.reduceByKey(_+_)
// 4. 将聚合的结果进行结构的转换
// ( ( 省份,广告 ), sum ) => ( 省份, ( 广告, sum ) )
val newMapRDD = reduceRDD.map{
case ( (prv, ad), sum ) => {
(prv, (ad, sum))
}
}
// 5. 将转换结构后的数据根据省份进行分组
// ( 省份, 【( 广告A, sumA ),( 广告B, sumB )】 )
val groupRDD: RDD[(String, Iterable[(String, Int)])] = newMapRDD.groupByKey()
// 6. 将分组后的数据组内排序(降序),取前3名
val resultRDD = groupRDD.mapValues(
iter => {
iter.toList.sortBy(_._2)(Ordering.Int.reverse).take(3)
}
)
// 7. 采集数据打印在控制台
resultRDD.collect().foreach(println)
sc.stop()
}
}
WordCountController 接受请求和响应
package com.atguigu.bigdata.spark.core.framework.controller
import com.atguigu.bigdata.spark.core.framework.common.TController
import com.atguigu.bigdata.spark.core.framework.service.WordCountService
class WordCountController extends TController {
private val wordCountService = new WordCountService()
// 调度
def dispatch(): Unit = {
// TODO 执行业务操作
val array = wordCountService.dataAnalysis()
array.foreach(println)
}
}
WordCountService 具体逻辑
package com.atguigu.bigdata.spark.core.framework.service
import com.atguigu.bigdata.spark.core.framework.common.TService
import com.atguigu.bigdata.spark.core.framework.dao.WordCountDao
import org.apache.spark.rdd.RDD
class WordCountService extends TService {
private val wordCountDao = new WordCountDao()
// 数据分析
def dataAnalysis() = {
val lines = wordCountDao.readFile("datas/word.txt")
val words: RDD[String] = lines.flatMap(_.split(" "))
val wordToOne = words.map(word=>(word,1))
val wordToSum: RDD[(String, Int)] = wordToOne.reduceByKey(_+_)
val array: Array[(String, Int)] = wordToSum.collect()
array
}
}
WordCountDao 数据交互
package com.atguigu.bigdata.spark.core.framework.dao
import com.atguigu.bigdata.spark.core.framework.common.TDao
// 持久层
class WordCountDao extends TDao{
}
common 一些抽象,可以看作是java中的接口或者抽象类
TApplication
package com.atguigu.bigdata.spark.core.framework.common
import com.atguigu.bigdata.spark.core.framework.controller.WordCountController
import com.atguigu.bigdata.spark.core.framework.util.EnvUtil
import org.apache.spark.{SparkConf, SparkContext}
trait TApplication {
def start(master:String ="local[*]", app:String = "Application")( op : => Unit ): Unit = {
val sparConf = new SparkConf().setMaster(master).setAppName(app)
val sc = new SparkContext(sparConf)
EnvUtil.put(sc)
try {
op
} catch {
case ex => println(ex.getMessage)
}
// TODO 关闭连接
sc.stop()
EnvUtil.clear()
}
}
TController
package com.atguigu.bigdata.spark.core.framework.common
trait TController {
def dispatch(): Unit
}
TDao
package com.atguigu.bigdata.spark.core.framework.common
import com.atguigu.bigdata.spark.core.framework.util.EnvUtil
trait TDao {
def readFile(path:String) = {
EnvUtil.take().textFile(path)
}
}
TService
package com.atguigu.bigdata.spark.core.framework.common
trait TService {
def dataAnalysis():Any
}
EnvUtil 工具类
存储sc 并传递
package com.atguigu.bigdata.spark.core.framework.util
import org.apache.spark.SparkContext
object EnvUtil {
private val scLocal = new ThreadLocal[SparkContext]()
def put( sc : SparkContext ): Unit = {
scLocal.set(sc)
}
def take(): SparkContext = {
scLocal.get()
}
def clear(): Unit = {
scLocal.remove()
}
}
以上摘自尚硅谷教程



