Spark(1)-wordCount入门

1. 创建Maven项目

    4.0.0    com.wakedata    code    1.0-SNAPSHOT            8        8        UTF-8        3.4.1        2.12.14                                   org.scala-lang            scala-library            ${scala.version}                                    org.apache.spark            spark-core_2.12            ${spark.version}                                src/main/scala        src/test/scala                                            net.alchim31.maven                scala-maven-plugin                3.2.2                                                                                        compile                            testCompile                                                                                                                                            -dependencyfile                                ${project.build.directory}/.scala_dependencies                                                                                                                                            org.apache.maven.plugins                maven-shade-plugin                2.4.3                                                            package                                                    shade                                                                                                                                                *:*                                                                            META-INF/*.SF                                        META-INF/*.DSA                                        META-INF/*.RSA                                                                                                                                                                                                reference.conf                                                                                                    cn.itcast.rpc.Master                                                                                                                                                 

2.目录结构

3. 代码实现

package sparkCoreimport org.apache.spark.rdd.RDDimport org.apache.spark.{SparkConf, SparkContext}/*** * 1. 创建SparkContext * 2. 创建RDD * 3. 调用RDD的Transformation算子 * 4. 调用Action * 5. 释放资源 */object wordcount_01 {  def main(args: Array[String]): Unit = {    val conf:SparkConf = new SparkConf().setAppName("WordCount").setMaster("local")    //创建SparkContext，使⽤SparkContext来创建RDD    val sc: SparkContext = new SparkContext(conf)    //spark写Spark程序，就是对抽象的神奇的⼤集合【RDD】编程，调⽤它⾼度封装的API //使⽤SparkContext创建RDD    val lines: RDD[String] = sc.textFile("./data/words.txt")    //切分压平    val words: RDD[String] = lines.flatMap(_.split(" "))    将单词和⼀组合放在元组中    val wordsAndOne: RDD[(String, Int)] = words.map((_, 1))    //分组聚合，reduceByKey可以先局部聚合再全局聚合    val reduced: RDD[(String, Int)] = wordsAndOne.reduceByKey(_ + _)    //排序    val sorted: RDD[(String, Int)] = reduced.sortBy(_._2, false)    //打印结果    sorted.foreach(line => print(line))    //释放资源    sc.stop()  }}

运行���果：