{"id":1352,"date":"2022-04-20T23:47:32","date_gmt":"2022-04-20T15:47:32","guid":{"rendered":"http:\/\/www.eait.co\/?p=1352"},"modified":"2022-06-15T21:18:27","modified_gmt":"2022-06-15T13:18:27","slug":"spark-%e9%85%8d%e7%bd%ae","status":"publish","type":"post","link":"https:\/\/notes.coremix.net\/?p=1352","title":{"rendered":"spark \u914d\u7f6e\u548cWord Count jar\u5305"},"content":{"rendered":"<p><strong>\u914d\u7f6e\uff1a\u89e3\u538b\uff0c\u914d\u7f6e\u6587\u4ef6\uff1a<\/strong><\/p>\n<pre class=\"brush: bash; title: ; notranslate\" title=\"\">\r\n\u914d\u7f6e\u6587\u4ef6\uff1aconf\/spark-env.sh\r\nexport JAVA_HOME=\/opt\/module\/jdk1.8.0_144\r\nexport SPARK_MASTER_HOST=bigdata166\r\nexport SPARK_MASTER_PORT=7077\r\n# \u4e0b\u9762\u7684\u53ef\u4ee5\u4e0d\u5199\uff0c\u9ed8\u8ba4\r\nexport SPARK_WORKER_CORES=1\r\nexport SPARK_WORKER_MEMORY=1024m\r\n<\/pre>\n<p>\u914d\u7f6eslaves\uff1a<\/p>\n<pre class=\"brush: bash; title: ; notranslate\" title=\"\">\r\n\u914d\u7f6e\u6587\u4ef6\uff1aconf\/slave\r\nbigdata166\r\n<\/pre>\n<p>\u542f\u52a8\u4e00\u4e0b\uff1a<\/p>\n<pre class=\"brush: bash; title: ; notranslate\" title=\"\">\r\nsbin\/start-all.sh\r\n<\/pre>\n<p>\u767b\u9646\u9875\u9762\uff088080\u7aef\u53e3\uff09bigdata166:8080<br \/>\n<img loading=\"lazy\" decoding=\"async\" width=\"703\" height=\"539\" class=\"alignnone wp-image-1353 size-full\" src=\"http:\/\/www.eait.co\/wp-content\/uploads\/2022\/04\/Pasted.png\" srcset=\"https:\/\/notes.coremix.net\/wp-content\/uploads\/2022\/04\/Pasted.png 703w, https:\/\/notes.coremix.net\/wp-content\/uploads\/2022\/04\/Pasted-300x230.png 300w\" sizes=\"auto, (max-width: 703px) 100vw, 703px\" \/><\/p>\n<p>\u5355\u673a\u7248\u914d\u7f6e\u5b8c\u6210\uff0c\u540e\u53f0166\u4e00\u4e2aworker\u4e00\u4e2amaster\u7ed3\u675f<\/p>\n<pre class=\"brush: bash; title: ; notranslate\" title=\"\">\r\nsbin\/stop-all.sh\r\n<\/pre>\n<p>&nbsp;<\/p>\n<p><strong>\u914d\u7f6e\u5b8c\u5168\u5206\u5e03\u5f0f\uff1a<\/strong><\/p>\n<p>\u6dfb\u52a0\u914d\u7f6e\u4ece\u8282\u70b9\uff0c\u5206\u53d1\uff0c\u542f\u52a8\uff08\u57fa\u4e8eslaves\u6587\u4ef6\uff09<\/p>\n<p>\u4e3b\u8282\u70b9sbin\/start-all.sh\u5373\u53ef<\/p>\n<p><img loading=\"lazy\" decoding=\"async\" width=\"489\" height=\"133\" class=\"alignnone wp-image-1354 size-full\" src=\"http:\/\/www.eait.co\/wp-content\/uploads\/2022\/04\/Pasted-1.png\" srcset=\"https:\/\/notes.coremix.net\/wp-content\/uploads\/2022\/04\/Pasted-1.png 489w, https:\/\/notes.coremix.net\/wp-content\/uploads\/2022\/04\/Pasted-1-300x82.png 300w\" sizes=\"auto, (max-width: 489px) 100vw, 489px\" \/><\/p>\n<p>\u6ce8\uff1a\u5176\u5b83\u673a\u5668\u4e0a\u8fd0\u884cstart-all\u65e0\u6cd5\u542f\u52a8master\uff0c\u9700\u8981\u624b\u52a8sbin\/start-master.sh\u624d\u884c<\/p>\n<p>&nbsp;<\/p>\n<p><strong>\u6d4b\u8bd5\u7a0b\u5e8f\uff1a<\/strong><\/p>\n<pre class=\"brush: bash; title: ; notranslate\" title=\"\">\r\nbin\/spark-submit --master spark:\/\/bigdata166:7077 --class \\\r\norg.apache.spark.examples.SparkPi examples\/jars\/spark-examples_2.11-2.1.0.jar 1000\r\n<\/pre>\n<p>\u8e29\u5751\u8bb0\u5f55:\u5148\u522b\u8ba4\u4e3a\u662f\u5305\u7684\u95ee\u9898,\u5148\u770b\u770b\u4e24\u4e2a\u8def\u5f84(submit\u548cjar\u5305\u7684\u8def\u5f84\u5bf9\u4e0d\u5bf9!)<\/p>\n<p><span style=\"color: #0000ff;\"> <span class=\"fontstyle0\">\u5982\u679c\u542f\u52a8<\/span><span class=\"fontstyle1\">spark shell<\/span><span class=\"fontstyle0\">\u65f6\u6ca1\u6709\u6307\u5b9a<\/span><span class=\"fontstyle1\">master<\/span><span class=\"fontstyle0\">\u5730\u5740\uff0c\u4f46\u662f\u4e5f\u53ef\u4ee5\u6b63\u5e38\u542f\u52a8<\/span><span class=\"fontstyle1\">spark shell<\/span><span class=\"fontstyle0\">\u548c\u6267\u884c<\/span><span class=\"fontstyle1\">spark shell<\/span><span class=\"fontstyle0\">\u4e2d\u7684\u7a0b\u5e8f\uff0c\u5176\u5b9e\u662f\u542f\u52a8\u4e86<\/span><span class=\"fontstyle1\">spark<\/span><span class=\"fontstyle0\">\u7684<\/span><span class=\"fontstyle1\">local<\/span><span class=\"fontstyle0\">\u6a21\u5f0f\uff0c\u8be5\u6a21\u5f0f\u4ec5\u5728\u672c\u673a\u542f\u52a8\u4e00\u4e2a\u8fdb\u7a0b\uff0c\u6ca1\u6709\u4e0e\u96c6\u7fa4\u5efa\u7acb\u8054\u7cfb\u3002<\/span><\/span><\/p>\n<p>\u6848\u4f8b\u7a0b\u5e8f:<\/p>\n<pre class=\"brush: scala; title: ; notranslate\" title=\"\">\r\nsc.textFile(&quot;hdfs:\/\/192.168.43.166:9000\/data\/data.txt&quot;).flatMap(_.split(&quot;\r\n&quot;)).map((_,1)).reduceByKey(_+_).saveAsTextFile(&quot;hdfs:\/\/192.168.43.166:9000\/outpu\r\nt\/spark\/wc&quot;)\r\n<\/pre>\n<p>\u81ea\u5b9a\u4e49\u7a0b\u5e8f:<\/p>\n<pre class=\"brush: scala; title: ; notranslate\" title=\"\">\r\n spark-submit --masterspark:\/\/bigdata166:7077 --class mydemo.WordCount jars\/wc.jar\\\r\nhdfs:\/\/192.168.43.166:9000\/data\/data.txt \\\r\nhdfs:\/\/192.168.43.166:9000\/output\/spark\/wc1\r\n<\/pre>\n<p><img loading=\"lazy\" decoding=\"async\" width=\"799\" height=\"847\" class=\"alignnone wp-image-1355 size-full\" src=\"http:\/\/www.eait.co\/wp-content\/uploads\/2022\/04\/Pasted-2.png\" srcset=\"https:\/\/notes.coremix.net\/wp-content\/uploads\/2022\/04\/Pasted-2.png 799w, https:\/\/notes.coremix.net\/wp-content\/uploads\/2022\/04\/Pasted-2-283x300.png 283w, https:\/\/notes.coremix.net\/wp-content\/uploads\/2022\/04\/Pasted-2-768x814.png 768w\" sizes=\"auto, (max-width: 799px) 100vw, 799px\" \/><\/p>\n<p>&nbsp;<\/p>\n<pre class=\"brush: java; title: ; notranslate\" title=\"\">\r\n spark-submit --master spark:\/\/bigdata166:7077 --class mydemo.JavaWordCount\r\njars\/wc.jar hdfs:\/\/192.168.43.166:9000\/data\/data.txt\r\n<\/pre>\n<p>&nbsp;<\/p>\n<p>&nbsp;<\/p>\n<p>&nbsp;<\/p>\n<p>&nbsp;<\/p>\n<p>&nbsp;<\/p>\n<p>&nbsp;<\/p>\n<p>&nbsp;<\/p>\n","protected":false},"excerpt":{"rendered":"<p>\u914d\u7f6e\uff1a\u89e3\u538b\uff0c\u914d\u7f6e\u6587\u4ef6\uff1a \u914d\u7f6e\u6587\u4ef6\uff1aconf\/spark-env.sh export JAVA_HOME=\/op [&hellip;]<\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[1],"tags":[],"class_list":["post-1352","post","type-post","status-publish","format-standard","hentry","category-uncategorized"],"blocksy_meta":[],"_links":{"self":[{"href":"https:\/\/notes.coremix.net\/index.php?rest_route=\/wp\/v2\/posts\/1352","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/notes.coremix.net\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/notes.coremix.net\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/notes.coremix.net\/index.php?rest_route=\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/notes.coremix.net\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=1352"}],"version-history":[{"count":6,"href":"https:\/\/notes.coremix.net\/index.php?rest_route=\/wp\/v2\/posts\/1352\/revisions"}],"predecessor-version":[{"id":1437,"href":"https:\/\/notes.coremix.net\/index.php?rest_route=\/wp\/v2\/posts\/1352\/revisions\/1437"}],"wp:attachment":[{"href":"https:\/\/notes.coremix.net\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=1352"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/notes.coremix.net\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=1352"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/notes.coremix.net\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=1352"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}