Spark优化

package other

import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}

import scala.collection.mutable.ListBuffer


object CombineByKeyTest {
  def main(args: Array[String]): Unit = {
    val context = new SparkContext(
      new SparkConf()
        .setMaster("local")
        .setAppName("l")
    )
    context.setLogLevel("Error")
    val rdd1: RDD[(String, Int)] = context.parallelize(List[(String, Int)](
      ("kobe", 24),
      ("james", 25),
      ("kobe", 26),
      ("bob", 27),
      ("kobe", 28),
      ("james", 29)
    ), 2)

    
    rdd1.combineByKey(i=>{"hello"+i},(s:String,i:Int)=>{s+"#"+i},(s1:String,s2:String)=>{s1+"@"+s2})
      .foreach(println)

//    rdd1.mapPartitionsWithIndex((index,iter)=>{
//      val list = new ListBuffer[String]()
//      while(iter.hasNext){
//        val tp = iter.next()
//        list.append("partition index = "+index+",value = "+tp)
//      }
//      list.iterator
//    }).foreach(println)
  }
}