-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathNetwork Intrusion Detection with K-Means.json
1 lines (1 loc) · 10 KB
/
Network Intrusion Detection with K-Means.json
1
{"paragraphs":[{"config":{"colWidth":12,"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true,"editorMode":"ace/mode/scala"},"settings":{"params":{},"forms":{}},"jobName":"paragraph_1460675266483_-1061947854","id":"20160414-230746_171457195","dateCreated":"Apr 14, 2016 11:07:46 PM","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:20","text":"print(\"hello world\")","dateUpdated":"Apr 14, 2016 11:08:01 PM","dateFinished":"Apr 14, 2016 11:08:38 PM","dateStarted":"Apr 14, 2016 11:08:02 PM","result":{"code":"SUCCESS","type":"TEXT","msg":"hello world"}},{"config":{"colWidth":12,"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true,"editorMode":"ace/mode/scala"},"settings":{"params":{},"forms":{}},"jobName":"paragraph_1460675281976_304477736","id":"20160414-230801_841756125","dateCreated":"Apr 14, 2016 11:08:01 PM","status":"FINISHED","progressUpdateIntervalMs":500,"focus":true,"$$hashKey":"object:67","dateUpdated":"Apr 14, 2016 11:22:28 PM","dateFinished":"Apr 14, 2016 11:22:29 PM","dateStarted":"Apr 14, 2016 11:22:28 PM","result":{"code":"SUCCESS","type":"TEXT","msg":"res2: org.apache.spark.SparkContext = org.apache.spark.SparkContext@414f8566\n"},"text":"sc"},{"config":{"colWidth":12,"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true,"editorMode":"ace/mode/scala"},"settings":{"params":{},"forms":{}},"jobName":"paragraph_1460676148642_893951922","id":"20160414-232228_770882638","dateCreated":"Apr 14, 2016 11:22:28 PM","status":"FINISHED","progressUpdateIntervalMs":500,"focus":true,"$$hashKey":"object:93","dateUpdated":"Apr 15, 2016 12:04:26 AM","dateFinished":"Apr 15, 2016 12:04:26 AM","dateStarted":"Apr 15, 2016 12:04:26 AM","result":{"code":"SUCCESS","type":"TEXT","msg":"rawData: org.apache.spark.rdd.RDD[String] = s3://spark-zeppelin/kddcup.data MapPartitionsRDD[6] at textFile at <console>:29\n"},"text":"val rawData = sc.textFile(\"s3://spark-zeppelin/kddcup.data\")"},{"config":{"colWidth":12,"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true,"editorMode":"ace/mode/scala"},"settings":{"params":{},"forms":{}},"jobName":"paragraph_1460677869532_-1423519351","id":"20160414-235109_702256016","dateCreated":"Apr 14, 2016 11:51:09 PM","status":"FINISHED","progressUpdateIntervalMs":500,"focus":true,"$$hashKey":"object:115","dateUpdated":"Apr 15, 2016 12:08:17 AM","dateFinished":"Apr 15, 2016 12:05:03 AM","dateStarted":"Apr 15, 2016 12:04:31 AM","result":{"code":"SUCCESS","type":"TEXT","msg":"(smurf.,2807886)\n(neptune.,1072017)\n(normal.,972781)\n(satan.,15892)\n(ipsweep.,12481)\n(portsweep.,10413)\n(nmap.,2316)\n(back.,2203)\n(warezclient.,1020)\n(teardrop.,979)\n(pod.,264)\n(guess_passwd.,53)\n(buffer_overflow.,30)\n(land.,21)\n(warezmaster.,20)\n(imap.,12)\n(rootkit.,10)\n(loadmodule.,9)\n(ftp_write.,8)\n(multihop.,7)\n(phf.,4)\n(perl.,3)\n(spy.,2)\n"},"text":"rawData.map(_.split(',').last).countimpoByValue().toSeq.sortBy(_._2).reverse.foreach(println)"},{"config":{"colWidth":12,"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true,"editorMode":"ace/mode/scala"},"settings":{"params":{},"forms":{}},"jobName":"paragraph_1460678079109_-1706022446","id":"20160414-235439_26601976","dateCreated":"Apr 14, 2016 11:54:39 PM","status":"FINISHED","progressUpdateIntervalMs":500,"focus":true,"$$hashKey":"object:136","dateUpdated":"Apr 15, 2016 12:05:45 AM","dateFinished":"Apr 15, 2016 12:05:45 AM","dateStarted":"Apr 15, 2016 12:05:45 AM","result":{"code":"SUCCESS","type":"TEXT","msg":"import org.apache.spark.mllib.linalg._\n"},"text":"import org.apache.spark.mllib.linalg._"},{"config":{"colWidth":12,"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true,"editorMode":"ace/mode/scala"},"settings":{"params":{},"forms":{}},"jobName":"paragraph_1460678745273_-800975373","id":"20160415-000545_108390403","dateCreated":"Apr 15, 2016 12:05:45 AM","status":"FINISHED","progressUpdateIntervalMs":500,"focus":true,"$$hashKey":"object:167","dateUpdated":"Apr 15, 2016 12:09:29 AM","dateFinished":"Apr 15, 2016 12:09:30 AM","dateStarted":"Apr 15, 2016 12:09:29 AM","result":{"code":"SUCCESS","type":"TEXT","msg":"labelsAndData: org.apache.spark.rdd.RDD[(String, org.apache.spark.mllib.linalg.Vector)] = MapPartitionsRDD[11] at map at <console>:34\ndata: org.apache.spark.rdd.RDD[org.apache.spark.mllib.linalg.Vector] = MapPartitionsRDD[12] at values at <console>:37\n"},"text":"val labelsAndData = rawData.map { line =>\n val buffer = line.split(',').toBuffer\n buffer.remove(1,3)\n val label = buffer.remove(buffer.length-1)\n val vector = Vectors.dense(buffer.map(_.toDouble).toArray)\n (label,vector)\n}\n\nval data = labelsAndData.values.cache()"},{"config":{"colWidth":12,"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true},"settings":{"params":{},"forms":{}},"jobName":"paragraph_1460678947037_-1730404478","id":"20160415-000907_10485429","dateCreated":"Apr 15, 2016 12:09:07 AM","status":"FINISHED","progressUpdateIntervalMs":500,"focus":true,"$$hashKey":"object:187","dateUpdated":"Apr 15, 2016 12:10:24 AM","dateFinished":"Apr 15, 2016 12:10:24 AM","dateStarted":"Apr 15, 2016 12:10:24 AM","result":{"code":"SUCCESS","type":"TEXT","msg":"import org.apache.spark.mllib.clustering._\n"},"text":"import org.apache.spark.mllib.clustering._"},{"config":{"colWidth":12,"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true,"editorMode":"ace/mode/scala"},"settings":{"params":{},"forms":{}},"jobName":"paragraph_1460679024193_-1384637286","id":"20160415-001024_1550207495","dateCreated":"Apr 15, 2016 12:10:24 AM","status":"FINISHED","progressUpdateIntervalMs":500,"focus":true,"$$hashKey":"object:208","dateUpdated":"Apr 15, 2016 12:11:13 AM","dateFinished":"Apr 15, 2016 12:12:08 AM","dateStarted":"Apr 15, 2016 12:11:13 AM","result":{"code":"SUCCESS","type":"TEXT","msg":"kmeans: org.apache.spark.mllib.clustering.KMeans = org.apache.spark.mllib.clustering.KMeans@44e5461e\nmodel: org.apache.spark.mllib.clustering.KMeansModel = org.apache.spark.mllib.clustering.KMeansModel@e4d55c7\n[48.34019491959669,1834.6215497618625,826.2031900016945,5.7161172049003456E-6,6.487793027561892E-4,7.961734678254053E-6,0.012437658596734055,3.205108575604837E-5,0.14352904910348827,0.00808830584493399,6.818511237273984E-5,3.6746467745787934E-5,0.012934960793560386,0.0011887482315762398,7.430952366370449E-5,0.0010211435092468404,0.0,4.082940860643104E-7,8.351655530445469E-4,334.9735084506668,295.26714620807076,0.17797031701994215,0.1780369894027256,0.057664898753273755,0.05772990937912744,0.7898841322627554,0.02117961060991181,0.028260810096298133,232.98107822302248,189.21428335201279,0.7537133898004665,0.030710978823802318,0.6050519309247977,0.006464107887637286,0.17809118431825394,0.1778858981347298,0.05792761150001271,0.05765922142401029]\n[10999.0,0.0,1.309937401E9,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0,1.0,0.0,0.0,1.0,1.0,1.0,0.0,0.0,255.0,1.0,0.0,0.65,1.0,0.0,0.0,0.0,1.0,1.0]\n"},"text":"val kmeans = new KMeans()\nval model = kmeans.run(data)\n\nmodel.clusterCenters.foreach(println)"},{"config":{"colWidth":12,"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true,"editorMode":"ace/mode/scala"},"settings":{"params":{},"forms":{}},"jobName":"paragraph_1460679073353_1175943265","id":"20160415-001113_2047941928","dateCreated":"Apr 15, 2016 12:11:13 AM","status":"FINISHED","progressUpdateIntervalMs":500,"focus":true,"$$hashKey":"object:228","dateUpdated":"Apr 15, 2016 12:31:35 AM","dateFinished":"Apr 15, 2016 12:32:04 AM","dateStarted":"Apr 15, 2016 12:31:35 AM","result":{"code":"SUCCESS","type":"TEXT","msg":"clusterLabelCount: scala.collection.Map[(Int, String),Long] = Map((0,portsweep.) -> 10412, (0,rootkit.) -> 10, (0,buffer_overflow.) -> 30, (0,phf.) -> 4, (0,pod.) -> 264, (0,perl.) -> 3, (0,spy.) -> 2, (0,ftp_write.) -> 8, (0,nmap.) -> 2316, (0,ipsweep.) -> 12481, (0,imap.) -> 12, (0,warezmaster.) -> 20, (0,satan.) -> 15892, (0,teardrop.) -> 979, (0,smurf.) -> 2807886, (0,neptune.) -> 1072017, (0,loadmodule.) -> 9, (0,guess_passwd.) -> 53, (0,normal.) -> 972781, (0,land.) -> 21, (0,multihop.) -> 7, (1,portsweep.) -> 1, (0,warezclient.) -> 1020, (0,back.) -> 2203)\n0 back. 2203\n0 buffer_overflow. 30\n0 ftp_write. 8\n0 guess_passwd. 53\n0 imap. 12\n0 ipsweep. 12481\n0 land. 21\n0 loadmodule. 9\n0 multihop. 7\n0 neptune. 1072017\n0 nmap. 2316\n0 normal. 972781\n0 perl. 3\n0 phf. 4\n0 pod. 264\n0 portsweep. 10412\n0 rootkit. 10\n0 satan. 15892\n0 smurf. 2807886\n0 spy. 2\n0 teardrop. 979\n0 warezclient. 1020\n0 warezmaster. 20\n1 portsweep. 1\n"},"text":"val clusterLabelCount = labelsAndData.map{ case (label,datum) =>\n val cluster = model.predict(datum)\n (cluster,label)\n}.countByValue\n\nclusterLabelCount.toSeq.sorted.foreach {\n case ((cluster,label),count) =>\n println(f\"$cluster%1s$label%18s$count%8s\")\n}"},{"config":{"colWidth":12,"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true},"settings":{"params":{},"forms":{}},"jobName":"paragraph_1460680295335_-431779538","id":"20160415-003135_579833305","dateCreated":"Apr 15, 2016 12:31:35 AM","status":"READY","progressUpdateIntervalMs":500,"focus":true,"$$hashKey":"object:298"}],"name":"Network Intrusion Detection with K-Means","id":"2BG6MUA2X","angularObjects":{"2B44YVSN1":[],"2AJXGMUUJ":[],"2AK8P7CPX":[],"2AM1YV5CU":[],"2AKK3QQXU":[],"2ANGGHHMQ":[]},"config":{"looknfeel":"default"},"info":{}}