{"id":1438,"date":"2022-06-15T21:53:32","date_gmt":"2022-06-15T13:53:32","guid":{"rendered":"http:\/\/www.eait.co\/?p=1438"},"modified":"2022-06-15T21:53:32","modified_gmt":"2022-06-15T13:53:32","slug":"scala%e4%b8%a4%e7%a7%8d%e7%8e%af%e5%a2%83%e5%a4%87%e5%bf%98","status":"publish","type":"post","link":"https:\/\/notes.coremix.net\/?p=1438","title":{"rendered":"scala\u4e24\u79cd\u73af\u5883\u5907\u5fd8"},"content":{"rendered":"<p>IDEA\u73af\u5883\uff1a<\/p>\n<pre class=\"brush: scala; title: ; notranslate\" title=\"\">\r\npackage wordcount\r\n\r\nimport org.apache.spark.{SparkConf, SparkContext}\r\n\r\nobject wordCountScala extends App{\r\nval conf = new SparkConf().setAppName(&quot;Wordcount&quot;).setMaster(&quot;local&quot;);  \/\/\u53ef\u5220\u6389\u8be5setMaster\r\n val sc = new SparkContext(conf)\r\n\r\n val line = sc.textFile(&quot;D:\\\\win7\u8fdc\u7a0b\\\\14\u671f\u5927\u6570\u636e\u6f6d\u5dde\u8bfe\u4ef6\\\\\u7b2c\u4e09\u9636\u6bb5\uff1a\u5b9e\u65f6\u5f00\u53d1\uff08plus\uff09\\\\2020-0105-Spark-SQL\\\\\u6570\u636e\\\\wordcount.txt&quot;)\r\n\r\n val result = line.flatMap(_.split(&quot; &quot;)).map((_,1)).reduceByKey(_+_)\r\n\r\n result.foreach(println)\r\n}\r\n<\/pre>\n<p>HDFS\u73af\u5883\uff1a<\/p>\n<pre class=\"brush: scala; title: ; notranslate\" title=\"\">\r\npackage wordcount\r\n\r\nimport org.apache.spark.{SparkConf, SparkContext}\r\n\r\nobject wordCountScala_HDFS extends App{\r\nval conf = new SparkConf().setAppName(&quot;Wordcount&quot;);\/\/\u53ef\u5220\u6389\u8be5setMaster\r\n val sc = new SparkContext(conf)\r\n\r\n val line = sc.textFile(&quot;hdfs:\/\/bigdata166:9000\/testdata\/wordcount.txt&quot;)\r\n\r\n val result = line.flatMap(_.split(&quot; &quot;)).map((_,1)).reduceByKey(_+_)\r\n\r\n result.foreach(println)\r\n result.collect()\r\n}\r\n<\/pre>\n<p>&nbsp;<\/p>\n<pre class=\"brush: bash; title: ; notranslate\" title=\"\">\r\n&#x5B;root@bigdata166 bin]# .\/spark-submit --master spark:\/\/bigdata166:7077 --class wordcount.wordCountScala_HDFS ..\/testjar\/scalaTest-1.0-SNAPSHOT.jar\r\n<\/pre>\n<p>&nbsp;<\/p>\n<p>\u5176\u5b83\u7684\u5751\uff1a<\/p>\n<p>\u6709\u65f6\u5019\u4e0d\u5173sparkshell\u7684\u65f6\u5019\u4f1a\u5bfc\u81f4\u5185\u5b58\u4e0d\u8db3\uff0c\u63a7\u5236\u53f0\u4f1a\u63d0\u793a\u67e5\u770bUI\u9875\u9762\u3002<\/p>\n<p>\u52a0collect\u8f93\u51fa\u5230\u65e5\u5fd7\uff0cprint\u8f93\u51fa\u5230outstd\u597d\u50cf<\/p>\n<p>&nbsp;<\/p>\n","protected":false},"excerpt":{"rendered":"<p>IDEA\u73af\u5883\uff1a package wordcount import org.apache.spark.{Spar [&hellip;]<\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[1],"tags":[],"class_list":["post-1438","post","type-post","status-publish","format-standard","hentry","category-uncategorized"],"blocksy_meta":[],"_links":{"self":[{"href":"https:\/\/notes.coremix.net\/index.php?rest_route=\/wp\/v2\/posts\/1438","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/notes.coremix.net\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/notes.coremix.net\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/notes.coremix.net\/index.php?rest_route=\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/notes.coremix.net\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=1438"}],"version-history":[{"count":1,"href":"https:\/\/notes.coremix.net\/index.php?rest_route=\/wp\/v2\/posts\/1438\/revisions"}],"predecessor-version":[{"id":1439,"href":"https:\/\/notes.coremix.net\/index.php?rest_route=\/wp\/v2\/posts\/1438\/revisions\/1439"}],"wp:attachment":[{"href":"https:\/\/notes.coremix.net\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=1438"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/notes.coremix.net\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=1438"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/notes.coremix.net\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=1438"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}