扩展/综合3

读application数据

        
            org.apache.spark
            spark-sql_2.12
        

        
            mysql
            mysql-connector-java
        

        
            com.typesafe
            config
            1.2.1
        

写入hive ，配置压缩，如果碰到权限问题加上System.setProperty(“HADOOP_USER_NAME”,“hdfs”)，
当然上集群没有这种问题，但是碰到hive不可访问得话，需要加上hive-site和hdfs-site

package com.cityos.spark

import com.typesafe.config.ConfigFactory
import org.apache.spark.internal.Logging
import org.apache.spark.sql.{SaveMode, SparkSession}

object MemberApp extends Logging{
  def main(args: Array[String]): Unit = {

    System.setProperty("HADOOP_USER_NAME","hdfs")

    val spark = SparkSession.builder()
      .master("local[2]")
      .appName(this.getClass.getCanonicalName)
      .enableHiveSupport()
      .getOrCreate()

    val config = ConfigFactory.load()

    val database = config.getString("database")
    val memeberTable = config.getString("memberTable")
    val url = config.getString("url")
    val username = config.getString("username")
    val password = config.getString("password")

    logError("加载会员数据开始。。。。。。。")
    val member = spark.read.format("jdbc")
      .option("url", url)
      .option("dbtable", s"$database.$memeberTable")
      .option("user", username)
      .option("password", password)
      .load()

    member.createOrReplaceTempView("member")
    logError("加载性别维度开始。。。。")
    val sexSql =
      """
        |select sex, count(1) from member  group by sex
        |""".stripMargin

    val sqlResult = spark.sql(sexSql)

    sqlResult.write.format("hive")
      .option("fileFormat","orc")
      .option("compression","zlib")
      .mode(SaveMode.Overwrite).saveAsTable(config.getString("sex.app.output"))

    logError("加载性别维度结束。。。。")

    spark.stop()
  }
}