{"id":1207,"date":"2022-02-03T18:18:06","date_gmt":"2022-02-03T10:18:06","guid":{"rendered":"http:\/\/www.eait.co\/?p=1207"},"modified":"2022-02-08T16:12:06","modified_gmt":"2022-02-08T08:12:06","slug":"spark%e9%ab%98%e5%8f%af%e7%94%a8%e9%85%8d%e7%bd%ae%e6%96%b9%e6%b3%95","status":"publish","type":"post","link":"https:\/\/notes.coremix.net\/?p=1207","title":{"rendered":"Spark\u9ad8\u53ef\u7528\u914d\u7f6e\u65b9\u6cd5"},"content":{"rendered":"<p>\u67b6\u6784\u56fe\uff1a\n<\/p>\n<p><img decoding=\"async\" src=\"http:\/\/www.eait.co\/wp-content\/uploads\/2022\/02\/020322_1018_Spark1.png\" alt=\"\"\/>\n\t<\/p>\n<p><span style=\"color:black\"><strong><span style=\"font-family:Times New Roman\">1<\/span><span style=\"font-family:\u5b8b\u4f53\">\uff09 <\/span><span style=\"font-family:Times New Roman\">zookeeper <\/span><span style=\"font-family:\u5b8b\u4f53\">\u6b63\u5e38\u5b89\u88c5\u5e76\u542f\u52a8<\/span><\/strong><br \/><strong><span style=\"font-family:Times New Roman\">2<\/span><span style=\"font-family:\u5b8b\u4f53\">\uff09 \u4fee\u6539 <\/span><span style=\"font-family:Times New Roman\">spark-env.sh <\/span><span style=\"font-family:\u5b8b\u4f53\">\u6587\u4ef6\u6dfb\u52a0\u5982\u4e0b\u914d\u7f6e\uff1a<\/span><\/strong><br \/><span style=\"font-family:Times New Roman\">vi spark-env.sh<br \/><\/span><span style=\"color:red\"><span style=\"font-family:\u5b8b\u4f53\">\u6ce8\u91ca\u6389\u5982\u4e0b\u5185\u5bb9\uff1a<\/span><br \/><span style=\"color:black\"><span style=\"font-family:Times New Roman\">#SPARK_MASTER_HOST=bigdata166<br \/>#SPARK_MASTER_PORT=7077<br \/><\/span><span style=\"color:red\"><span style=\"font-family:\u5b8b\u4f53\">\u6dfb\u52a0\u4e0a\u5982\u4e0b\u5185\u5bb9\uff1a<\/span><br \/><span style=\"color:black\"><span style=\"font-family:Times New Roman\">export SPARK_DAEMON_JAVA_OPTS=&#8221;<br \/>-Dspark.deploy.recoveryMode=ZOOKEEPER<br \/>-Dspark.deploy.zookeeper.url=<span style=\"color:red\">bigdata166,bigdata167,bigdata168<br \/><span style=\"color:black\">-Dspark.deploy.zookeeper.dir=\/spark&#8221;<br \/><\/span><\/span><\/span><span style=\"font-family:Times New Roman\">3<\/span><span style=\"font-family:\u5b8b\u4f53\">\uff09 \u5206\u53d1\u914d\u7f6e\u6587\u4ef6<\/span><br \/><span style=\"font-family:Times New Roman\">xsync spark-env.sh<br \/><\/span><span style=\"font-family:Times New Roman\">4<\/span><span style=\"font-family:\u5b8b\u4f53\">\uff09 \u5728 <\/span><span style=\"font-family:Times New Roman\">bigdata166 <\/span><span style=\"font-family:\u5b8b\u4f53\">\u4e0a\u542f\u52a8\u5168\u90e8\u8282\u70b9<\/span><br \/><span style=\"font-family:Times New Roman\">sbin\/start-all.sh<br \/><\/span><span style=\"font-family:Times New Roman\">5<\/span><span style=\"font-family:\u5b8b\u4f53\">\uff09 \u5728 <\/span><span style=\"font-family:Times New Roman\">bigdata167 <\/span><span style=\"font-family:\u5b8b\u4f53\">\u4e0a\u5355\u72ec\u542f\u52a8 <\/span><span style=\"font-family:Times New Roman\">master <\/span><span style=\"font-family:\u5b8b\u4f53\">\u8282\u70b9<\/span><br \/><span style=\"font-family:Times New Roman\">sbin\/start-master.sh<\/span><\/span><br \/><span style=\"color:black\"><span style=\"font-family:Times New Roman\">6<\/span><span style=\"font-family:\u5b8b\u4f53\">\uff09 <\/span><span style=\"font-family:Times New Roman\">spark HA <\/span><span style=\"font-family:\u5b8b\u4f53\">\u96c6\u7fa4\u8bbf\u95ee<\/span><br \/><span style=\"font-family:Times New Roman\">\/opt\/module\/spark\/bin\/spark-shell \\<br \/>&#8211;master spark:\/\/bigdata166:7077,bigdata167:7077 \\<br \/>&#8211;executor-memory 2g \\<br \/>&#8211;total-executor-cores 2<\/span><\/span><\/span><\/span><\/span><\/span><\/p>\n","protected":false},"excerpt":{"rendered":"<p>\u67b6\u6784\u56fe\uff1a 1\uff09 zookeeper \u6b63\u5e38\u5b89\u88c5\u5e76\u542f\u52a82\uff09 \u4fee\u6539 spark-env.sh \u6587\u4ef6\u6dfb\u52a0\u5982\u4e0b\u914d\u7f6e\uff1avi [&hellip;]<\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[30,29],"tags":[],"class_list":["post-1207","post","type-post","status-publish","format-standard","hentry","category-bigdata","category-29"],"blocksy_meta":[],"_links":{"self":[{"href":"https:\/\/notes.coremix.net\/index.php?rest_route=\/wp\/v2\/posts\/1207","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/notes.coremix.net\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/notes.coremix.net\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/notes.coremix.net\/index.php?rest_route=\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/notes.coremix.net\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=1207"}],"version-history":[{"count":1,"href":"https:\/\/notes.coremix.net\/index.php?rest_route=\/wp\/v2\/posts\/1207\/revisions"}],"predecessor-version":[{"id":1208,"href":"https:\/\/notes.coremix.net\/index.php?rest_route=\/wp\/v2\/posts\/1207\/revisions\/1208"}],"wp:attachment":[{"href":"https:\/\/notes.coremix.net\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=1207"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/notes.coremix.net\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=1207"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/notes.coremix.net\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=1207"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}