Spark ML 文本的分类

def main(args: Array[String]): Unit = {
    val sparkConf = new SparkConf().setAppName("DecisionTree1").setMaster("local[2]")

    val sc = new SparkContext(sparkConf)
    // Load documents (one per line).

    val documents: RDD[Seq[String]] = sc.textFile("/XXX/sample_libsvm_data.txt").map(_.split(" ").toSeq)

    val hashingTF = new HashingTF()
    val tf: RDD[Vector] = hashingTF.transform(documents)

    // Filter the top 50 features from each feature vector
    val data = tf.map { lp =>
      LabeledPoint(lp(0), Vectors.dense((lp.toArray).tail))
    }

    val splits = data.randomSplit(Array(0.9, 0.1))
    val (trainingData, testData) = (splits(0), splits(1))

    // Train a DecisionTree model.
    //  Empty categoricalFeaturesInfo indicates all features are continuous.
    val numClasses = 2
    val categoricalFeaturesInfo = Map[Int, Int]()
    val impurity = "gini"
    val maxDepth = 5
    val maxBins = 32

    val model = DecisionTree.trainClassifier(trainingData, numClasses, categoricalFeaturesInfo,
      impurity, maxDepth, maxBins)

    // Evaluate model on test instances and compute test error

    val testStr = Array("y z a c")

    val prediction = model.predict(hashingTF.transform(testStr))
    
    println("-----------------------------------------")
    println(prediction)
    println("-----------------------------------------")
  }