SparkStreaming程序设计

object StreamingWordCount {

  def main(args: Array[String]): Unit = {
    val sparkConf: SparkConf = new SparkConf().setMaster("local[*]").setAppName("StreamingWordCount")
    val ssc = new StreamingContext(sparkConf, Duration(10000))

    val rdd = ssc.textFileStream("file:///Users/orco/data/wordCountDir")
    val lines = rdd.flatMap(_.split(" "))
    lines.foreachRDD(rdd => {
      rdd.
        map(x => (x, 1)).
        reduceByKey(_ + _).
        foreach(println)
    })

    ssc.start()
    ssc.awaitTermination()
  }
}