小工具知识网

当前位置: 首页 > 行业知识 > Spark(2)-基础tranform算子(二)

行业知识

Spark(2)-基础tranform算子(二)

2026-11-18 13:35:00 来源:互联网转载

一、算子列表

编号名称
19repartitionAndSortWithinPartitions算子
20sortBy算子
21sortByKey算子
22reparation算子
23coalesce算子
24cogroup算子
25join算子
26leftOuterJoin算子
27rightOuterJoin算子
28fullOuterJoin算子
29intersection算子
30subtract算子
31cartesian算子

二、代码示例

package sparkCoreimport org.apache.spark.rdd.RDDimport org.apache.spark.{HashPartitioner, SparkConf, SparkContext}object basic_transform_03 {  def main(args: Array[String]): Unit = {    val conf: SparkConf = new SparkConf().setAppName("transform").setMaster("local[*]")    val sc:SparkContext = new SparkContext(conf)    sc.setLogLevel("WARN")    //19.repartitionAndSortWithinPartitions:按照值的分区器进行分区,并且将数据按照指定的规则在分区内排序,底层使用的是ShuffledRDD,设置   // 了指定的分区器和排序规则    println("*********19.repartitionAndSortWithinPartitions算子*****")   val lst1: List[(String, Int)] = List(     ("spark", 1), ("spark", 1), ("Hive", 1),     ("Mysql", 1), ("Java", 1), ("Python", 1),     ("Mysql", 1), ("kafka", 1), ("flink", 1)   )    val rdd19: RDD[(String, Int)] = sc.parallelize(lst1, 4)    val partitioner: HashPartitioner = new HashPartitioner(rdd19.partitions.length)    // rdd19按照指定的分区器进行分区,并且每个分区内的结果按照key(spark,hive等)的字母的字典顺序进行排序    val ReSortwithinPatitioner_rdd: RDD[(String, Int)] = rdd19.repartitionAndSortWithinPartitions(partitioner)    println(ReSortwithinPatitioner_rdd.collect().toList)    //20.sortBy    println("*********20.sortBy算子*************")    val lst2: List[String] = List(      "maple", "maple", "kelly", "Avery",      "kelly", "Jacky", "Paul", "Avery",      "maple", "maple", "Paul", "Avery"    )    val rdd20: RDD[String] = sc.parallelize(lst2)    val words: RDD[String] = rdd20.flatMap(_.split(" "))    val wordAndOne: RDD[(String, Int)] = words.map((_, 1))    val reduced: RDD[(String, Int)] = wordAndOne.reduceByKey(_ + _)    // 根据单词出现的次数,从高到低排序    val sorted_rdd: RDD[(String, Int)] = reduced.sortBy(_._2, false)    println(sorted_rdd.collect().toList)    //21.sortByKey    println("*********21.sortByKey算子*************")    val sortedByKey_rdd: RDD[(Int, (String, Int))] = reduced.map(t => (t._2, t)).sortByKey(false)    println(sortedByKey_rdd.collect().toList)    //22.reparation:功能是重新分区,⼀定会shuffle,即将数据打散.    println("*********22.reparation算子*************")    val rdd22: RDD[String] = sc.parallelize(lst2, 3)    //reparation方法一定会shuffle    // 无论将分区数量变多、变少或不变,都会shuffle    // reparation的底层调⽤的是coalesce,shuffle = true    val rep_rdd: RDD[String] = rdd22.repartition(3)    //23.coalesce:可以shuffle,也可以不shuffle,如果将分区数量减少,并且shuffle = false,就是将分区进⾏合并    println("*********23.coalesce算子*************")    val rdd23: RDD[String] = sc.parallelize(lst2,4)    // 与reparation一样,必然会shuffle和重新分区    val coalesce_rdd1: RDD[String] = rdd23.coalesce(4, true)    println("coalesce_rdd1:",coalesce_rdd1.collect().toList)    //分区减少,且shuffle为false,并不会分区    val coalesce_rdd2: RDD[String] = rdd23.coalesce(2, false)    println("coalesce_rdd2:",coalesce_rdd2.collect().toList)    //24.cogroup:使⽤相同的分区器(HashPartitioner),将来⾃多个RDD中的key相同的数据通过⽹络传⼊到同⼀台机器的同⼀个分区中    //注意:两个RDD中对应的数据都必须是对偶元组类型,并且key类型⼀定相同    println("*********24.cogroup算子*************")    val rdd24_1: RDD[(String, Int)] = sc.parallelize(List(("tom", 1), ("tom", 2), ("Jerry", 3), ("Paul", 3)))    val rdd24_2: RDD[(String, Int)] = sc.parallelize(List(("tom", 10), ("tom", 20), ("Jacky", 3), ("Avery", 30)))    val cogroup_rdd: RDD[(String, (Iterable[Int], Iterable[Int]))] = rdd24_1.cogroup(rdd24_2)    println(cogroup_rdd.collect().toList)    //25.join:相当于SQL中的内关联join    println("*********25.join算子*************")    val rdd25_1: RDD[(String, Int)] = sc.parallelize(List(("tom", 1), ("tom", 2), ("Jerry", 3), ("Paul", 3)))    val rdd25_2: RDD[(String, Int)] = sc.parallelize(List(("tom", 10), ("tom", 20), ("Jacky", 3), ("Avery", 30)))    val join_rdd: RDD[(String, (Int, Int))] = rdd25_1.join(rdd25_2)    println(join_rdd.collect().toList)    //26.leftOuterJoin:相当于SQL中的左外关联    println("*********26.leftOuterJoin算子*************")    val rdd26_1: RDD[(String, Int)] = sc.parallelize(List(("tom", 1), ("tom", 2), ("Jerry", 3), ("Paul", 3)))    val rdd26_2: RDD[(String, Int)] = sc.parallelize(List(("tom", 10), ("tom", 20), ("Jacky", 3), ("Avery", 30)))    val leftJoin_rdd: RDD[(String, (Int, Option[Int]))] = rdd26_1.leftOuterJoin(rdd26_2)    println(leftJoin_rdd.collect().toList)    //27.rightOuterJoin:相当于SQL中的右外关联    println("*********27.rightOuterJoin算子*************")    val rdd27_1: RDD[(String, Int)] = sc.parallelize(List(("tom", 1), ("tom", 2), ("Jerry", 3), ("Paul", 3)))    val rdd27_2: RDD[(String, Int)] = sc.parallelize(List(("tom", 10), ("tom", 20), ("Jacky", 3), ("Avery", 30)))    val rightJoin_rdd: RDD[(String, (Option[Int], Int))] = rdd27_1.rightOuterJoin(rdd27_2)    println(rightJoin_rdd.collect().toList)    //28.fullOuterJoin:相当于SQL中的全关联    println("*********28.fullOuterJoin算子*************")    val rdd28_1: RDD[(String, Int)] = sc.parallelize(List(("tom", 1), ("tom", 2), ("Jerry", 3), ("Paul", 3)))    val rdd28_2: RDD[(String, Int)] = sc.parallelize(List(("tom", 10), ("tom", 20), ("Jacky", 3), ("Avery", 30)))    val fullOutJoin_rdd: RDD[(String, (Option[Int], Option[Int]))] = rdd28_1.fullOuterJoin(rdd28_2)    println(fullOutJoin_rdd.collect().toList)    //29.intersection:求交集    println("*********29.intersection算子*************")    val rdd29_1: RDD[Int] = sc.parallelize(List(1, 2, 3, 4, 5))    val rdd29_2: RDD[Int] = sc.parallelize(List(3, 4, 5,6,7))    val rdd29_inter: RDD[Int] = rdd29_1.intersection(rdd29_2)    println("rdd29_inter:", rdd29_inter.collect().toList)    //底层实现使用的是cogroup    val rdd29_11: RDD[(Int, Null)] = rdd29_1.map((_, null))    val rdd29_22: RDD[(Int, Null)] = rdd29_2.map((_, null))    val rdd29_co: RDD[(Int, (Iterable[Null], Iterable[Null]))] = rdd29_11.cogroup(rdd29_22)    //List((1,(CompactBuffer(null),CompactBuffer())), (2,(CompactBuffer(null),CompactBuffer())), (3,(CompactBuffer(null),CompactBuffer(null))), (4,(CompactBuffer(null),CompactBuffer(null))), (5,(CompactBuffer(null),CompactBuffer(null))), (6,(CompactBuffer(),CompactBuffer(null))), (7,(CompactBuffer(),CompactBuffer(null))))    println("rdd29_co:",rdd29_co.collect().toList)    val rdd29_co_inter = rdd29_co.filter(it => it._2._1.nonEmpty & it._2._2.nonEmpty).keys    print("rdd29_co_inter", rdd29_co_inter.collect().toList)    //30.subtract:两个RDD的差集,将第⼀个RDD中的数据,如果在第⼆个RDD中出现了,就从第⼀个RDD中移除    println("*********30.subtract算子*************")    val rdd30_1: RDD[Int] = sc.parallelize(List(1, 2, 3, 4, 5))    val rdd30_2: RDD[Int] = sc.parallelize(List(3, 4, 5,6,7))    val substract_rdd: RDD[Int] = rdd30_1.subtract(rdd30_2)    println(substract_rdd.collect().toList)    //31.cartesian:笛卡尔积    println("*********31.cartesian算子*************")    val rdd31_1: RDD[String] = sc.parallelize(List("Maple", "Kelly", "Avery"))    val rdd31_2: RDD[String] = sc.parallelize(List("Jerry", "Maple", "Tom"))    val cartesian_rdd: RDD[(String, String)] = rdd31_1.cartesian(rdd31_2)    println(cartesian_rdd.collect().toList)  }}

��三、部分算子示意图

1、coalesce算子shuffle参数设置成false

2、cogroup

 

上一篇:轻松网聊视频下载指南,详细步骤教程

下一篇:视频下载到相册的简易指南