sparkml和mllib分别实现KMeans算法

本文主要是介绍sparkml和mllib分别实现KMeans算法，希望对大家解决编程问题提供一定的参考价值，需要的开发者们随着小编来一起学习吧！

正如大家所知道的MLLib这个算法包apache已经宣布只维护不更新了，所以大家如果做算法本人推荐使用ML的算法包。
原理的话本人就不在讲了，因为很多资料都写的比较清晰明白，这里我只写代码
本人在这里写了一些sparkml和mllib的示例入门程序

SparkML


import org.apache.spark.ml.clustering.{KMeans, KMeansModel}
import org.apache.spark.ml.feature.{ VectorAssembler}
import org.apache.spark.rdd.RDD
import org.apache.spark.sql.types.{DoubleType,StructField, StructType}
import org.apache.spark.sql.{Row, SparkSession}
/*** Created by LiuWenSheng on 2017/10/11.*/
object MyKMeans {def main(args: Array[String]) {val sparkSession = SparkSession.builder().appName("mykmeans").master("local[2]").getOrCreate()//本例使用的数据集为鸢尾花数据集，大家可以自行网上下载。val rawData: RDD[String] = sparkSession.sparkContext.textFile("D:/testData/fun.txt")//转换数据格式val data = rawData.map(_.split(",")).map(x=>Row(x(0).toDouble,x(1).toDouble,x(2).toDouble,x(3).toDouble))//创建StructType格式val struct: StructType = StructType(StructField("f1",DoubleType,false)::StructField("f2",DoubleType,false)::StructField("f3",DoubleType,false)::StructField("f4",DoubleType,false)::Nil)//创建DataFrameval df = sparkSession.createDataFrame(data,struct)//把f1~f4转化为向量集合val vectorAssembler = new VectorAssembler().setInputCols(Array("f1","f2","f3","f4")).setOutputCol("features")val kms = new KMeans().setPredictionCol("predictionCol").setFeaturesCol("features").setMaxIter(66).setK(4)val a = vectorAssembler.transform(df)val model: KMeansModel = kms.fit(a)model.transform(a).show(100)val wssse = model.computeCost(a)println("wssse is :"+wssse)//计算聚类的中心点model.clusterCenters.foreach(println)/**                    如果想要使用pipeline的话代码如下            **/
//    val pipeLine: Pipeline = new Pipeline().setStages(Array(vectorAssembler,model))
//    val res: PipelineModel = pipeLine.fit(df)
//    val b: DataFrame = res.transform(df)
//    b.show(130)}
}

结果如下所示：

+---+---+---+---+-----------------+-------------+
| f1| f2| f3| f4|         features|predictionCol|
+---+---+---+---+-----------------+-------------+
|5.1|3.5|1.4|0.2|[5.1,3.5,1.4,0.2]|            3|
|4.9|3.0|1.4|0.2|[4.9,3.0,1.4,0.2]|            0|
|4.7|3.2|1.3|0.2|[4.7,3.2,1.3,0.2]|            0|
|4.6|3.1|1.5|0.2|[4.6,3.1,1.5,0.2]|            0|
|5.0|3.6|1.4|0.2|[5.0,3.6,1.4,0.2]|            3|
|5.4|3.9|1.7|0.4|[5.4,3.9,1.7,0.4]|            3|
|4.6|3.4|1.4|0.3|[4.6,3.4,1.4,0.3]|            0|
|5.0|3.4|1.5|0.2|[5.0,3.4,1.5,0.2]|            0|
|4.4|2.9|1.4|0.2|[4.4,2.9,1.4,0.2]|            0|
|4.9|3.1|1.5|0.1|[4.9,3.1,1.5,0.1]|            0|
+---+---+---+---+-----------------+-------------+
only showing top 10 rowswssse is :71.34822351828443
[4.725,3.1333333333333333,1.4208333333333334,0.19166666666666676]
[5.883606557377049,2.740983606557377,4.388524590163936,1.4344262295081964]
[6.8538461538461535,3.076923076923076,5.715384615384614,2.053846153846153]
[5.265384615384616,3.6807692307692306,1.503846153846154,0.2923076923076923]

SparkMLLib实现KMeans

import org.apache.spark.mllib.clustering.KMeans
import org.apache.spark.mllib.linalg.{Vector, Vectors}
import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}/*** Created by LiuWenSheng on 2017/10/16.*/
object MLLib_KMeans {def main(args: Array[String]) {val conf = new SparkConf().setMaster("local[2]").setAppName("mllibKMeans")val sc = new SparkContext(conf)val data = sc.textFile("D:/testData/fun.txt")//制作vetors向量的RDD用于计算val parseTrainData: RDD[Vector] = data.map { line =>Vectors.dense(line.split(",").take(4).map(_.toDouble))}val numIterators = 66//迭代次数val numClusters = 3 //聚的的类的种树val runs = 3 //运行次数选出最优解//开始训练val model = KMeans.train(parseTrainData,numClusters,numIterators,runs)parseTrainData.map(x=>(x.toString+"is belongs to")+model.predict(x)).collect().foreach(println(_))//计算cost （点到最近的中心平方之和）val wssse = model.computeCost(parseTrainData)println("wssse is:" + wssse)//计算中心点model.clusterCenters.foreach(println(_))println(model.predict(Vectors.dense(6.6,2.9,4.5,1.2)))}
}

结果如下所示：

[5.1,3.5,1.4,0.2]is belongs to0
[4.9,3.0,1.4,0.2]is belongs to0
[4.7,3.2,1.3,0.2]is belongs to0
[4.6,3.1,1.5,0.2]is belongs to0
[5.0,3.6,1.4,0.2]is belongs to0
[5.4,3.9,1.7,0.4]is belongs to0
[4.6,3.4,1.4,0.3]is belongs to0
[5.0,3.4,1.5,0.2]is belongs to0
[4.4,2.9,1.4,0.2]is belongs to0
[4.9,3.1,1.5,0.1]is belongs to0
[5.4,3.7,1.5,0.2]is belongs to0
[4.8,3.4,1.6,0.2]is belongs to0
[4.8,3.0,1.4,0.1]is belongs to0
[4.3,3.0,1.1,0.1]is belongs to0
[5.8,4.0,1.2,0.2]is belongs to0
[5.7,4.4,1.5,0.4]is belongs to0
[5.4,3.9,1.3,0.4]is belongs to0only show a fewwssse is:78.94506582597703
[5.005999999999999,3.4180000000000006,1.4640000000000002,0.2439999999999999]
[6.8538461538461535,3.076923076923076,5.715384615384614,2.053846153846153]
[5.883606557377049,2.740983606557377,4.388524590163936,1.4344262295081964]
2