hive on spark 代码方式实现

    spark 版本  2.4.0-cdh6.3.2
    hadoop 版本 3.0.0-cdh6.3.2
    hive 版本 2.1.1-cdh6.3.2

    4.0.0

    bigdata
    data_shuffle_demo
    1.0-SNAPSHOT
    

        2.11.12
        2.4.0-cdh6.3.2
        3.0.0-cdh6.3.2
        
        provided
        
        
        optimization
    

    

        

            cloudera
            https://repository.cloudera.com/artifactory/cloudera-repos/
            
                true
            
            
                false
            
        

    

        
        
            org.scala-lang
            scala-library
            ${scala.version}
            ${mvn.scop.pro}
        
        
            org.apache.hadoop
            hadoop-client
            ${hadoop.version}
            ${mvn.scop.pro}
        
        
            org.apache.spark
            spark-core_2.11
            ${spark.version}
            
                
                    hadoop-client
                    org.apache.hadoop
                
            
            ${mvn.scop.pro}
        
        
            org.apache.spark
            spark-sql_2.11
            ${spark.version}
            ${mvn.scop.pro}
        


        
            org.apache.hive
            hive-exec
            2.1.1-cdh6.3.2
        

        
        org.apache.hive
        hive-jdbc
        2.1.1-cdh6.3.2
        ${mvn.scop.pro}
        

        
            org.apache.spark
            spark-hive_2.11
            ${spark.version}
            ${mvn.scop.pro}
        

    

    
        ${jar.pre.name}
        


            
            
                org.apache.maven.plugins
                maven-compiler-plugin
                3.8.1
                
                    1.8
                    1.8
                
            


            
            
                net.alchim31.maven
                scala-maven-plugin
                3.2.2
                
                    
                        
                            compile
                            testCompile
                        
                    
                
            


        
    

//spark链接
  def sparkSession(): SparkSession = {

    var conf = new SparkConf()
      .set("spark.sql.legacy.allowCreatingManagedTableUsingNonemptyLocation", "true")
      .set("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
      .set("spark.sql.warehouse.dir", "your hive warehouse")
      .set("hive.metastore.uris", "your hive metastore uris")
    SparkSession.builder()
      .config(conf)
      .enableHiveSupport()
      .getOrCreate()
  }