{"id":1440,"date":"2022-06-15T23:22:08","date_gmt":"2022-06-15T15:22:08","guid":{"rendered":"http:\/\/www.eait.co\/?p=1440"},"modified":"2022-06-15T23:22:59","modified_gmt":"2022-06-15T15:22:59","slug":"spark-cache-checkpoints-mappartitionswithindex-aggregate","status":"publish","type":"post","link":"https:\/\/notes.coremix.net\/?p=1440","title":{"rendered":"spark cache checkpoints mapPartitionsWithIndex aggregate"},"content":{"rendered":"

 <\/p>\n

 <\/p>\n

\r\nscala> var rdd1 = sc.textFile("hdfs:\/\/bigdata111:9000\/spark\/test_Cache.txt")\r\nrdd1: org.apache.spark.rdd.RDD[String] = hdfs:\/\/bigdata166:9000\/spark\/test_Cache.txt MapPartitionsRDD[1] at textFile at <console>:24\r\n\r\nscala> sc.setCheckpointDir("hdfs:\/\/bigdata166:9000\/spark\/")\r\n\r\nscala> sc.setCheckpointDir("hdfs:\/\/bigdata166:9000\/spark\/checkpoint")\r\n\r\nscala> rdd1.count\r\nscala> rdd1.cache    \/\/\u7f13\u5b58\r\nscala> rdd1.count\r\nres2: Long = 1846904 \r\n\r\n\/\/\r\nscala> def fun1(index:Int,itea:Iterator[Int]):Iterator[String] = {\r\n     | itea.toList.map(x => "[partid:" + index +", value="+x+"]" ).iterator\r\n     | }\r\nfun1: (index: Int, itea: Iterator[Int])Iterator[String]\r\n\r\nscala> rdd1.mapPartitions\r\nmapPartitions   mapPartitionsWithIndex\r\n\r\nscala> rdd1.mapPartitionsWithIndex(fun1).collect\r\nres3: Array[String] = Array(\r\n    [partid:0, value=1], [partid:0, value=2], [partid:0, value=3], \r\n    [partid:1, value=4], [partid:1, value=5], [partid:1, value=6], \r\n    [partid:2, value=7], [partid:2, value=8], [partid:2, value=9])\r\n\r\n<\/pre>\n
 <\/p>\n
aggregate<\/strong><\/span>\uff1a\u805a\u5408\uff0c<\/span><\/p>\n
\u5148\u5c40\u90e8\u805a\u5408\uff0c\u7136\u540e\u5168\u5c40\u805a\u5408<\/span><\/p>\n
\u6c42\u6bcf\u4e2a\u5206\u533a\u7684\u6700\u5927\u503c\uff0c\u7136\u540e\u6c42\u548c<\/span><\/span><\/p>\n
\u73b0\u6c42\u6bcf\u4e2a\u5206\u533a\u7684\u6700\u5927\u503c<\/span><\/p>\n
\u6c42\u548c<\/span><\/p>\n
zeroValue: U\uff1a\u521d\u59cb\u5316\uff0c\u9700\u8981\u8d4b\u503c\uff1a\u521d\u59cb\u503c\u5728\u5c40\u90e8\u64cd\u4f5c\u8d77\u4f5c\u7528\uff0c\u5168\u5c40\u64cd\u4f5c\u4e5f\u8d77\u4f5c\u7528<\/span><\/p>\n
\u540e\u9762\u4e24\u4e2a\u51fd\u6570\u53c2\u6570<\/span><\/p>\n
\u7b2c\u4e00\u4e2a\u51fd\u6570\uff1a\u8868\u793a\u5c40\u90e8\u64cd\u4f5c<\/span><\/span><\/p>\n
\u7b2c\u4e8c\u4e2a\uff1a\u8868\u793a\u5168\u5c40\u64cd\u4f5c<\/span><\/p>\n
\r\nscala> var rdd1 = sc.parallelize(List(1,2,3,4,5),2)\r\nrdd1: org.apache.spark.rdd.RDD[Int] = ParallelCollectionRDD[4] at parallelize at <console>:24\r\n\r\nscala> rdd1.mapPartitionsWithIndex(fun1).collect\r\nres4: Array[String] = Array(\r\n    [partid:0, value=1], [partid:0, value=2],\r\n    [partid:1, value=3], [partid:1, value=4], [partid:1, value=5])\r\n\r\nscala> rdd1.aggregate(0)(max(_,_),_+_)\r\nres5: Int = 7\r\nscala> rdd1.aggregate(10)(max(_,_),_+_)\r\nres6: Int = 30\r\n\/\/\u7b2c\u4e00\u4e2a\u5206\u533a\u6570\u636e\uff1a10\uff08\u521d\u59cb\u503c\uff09\uff0c1\uff0c2\u2014\u2014\u2014\u201410\r\n\/\/\u7b2c\u4e8c\u4e2a\u5206\u533a\uff1a10\uff08\u521d\u59cb\u503c\uff09\uff0c3\uff0c4\uff0c5\u2014\u2014\u2014\u201410\r\n\/\/\u6c42\u548c\uff1a10\uff08\u521d\u59cb\u503c\uff09+10+10=30\r\n\r\n\/\/\u7b2c\u4e00\u4e2a\u5206\u533a\u7684\u6700\u5927\u503c\uff1a2\r\n\/\/\u4e8c\uff1a5\r\n\/\/\u6c42\u548c\uff1a2+5=7\r\n\r\nscala> rdd1.aggregate(0)(_+_,_+_)\r\nres7: Int = 15\r\n\r\nscala> rdd1.aggregate(10)(_+_,_+_)\r\nres8: Int = 45\r\n\r\n\/\/\u7b2c\u4e00\u4e2a\u5206\u533a\uff1a10+1+2=3\r\n\/\/\u7b2c\u4e8c\u4e2a\u5206\u533a\uff1a10+3+4+5=12\r\n\/\/\u6c42\u548c\uff1a10+3+12=15\r\n\r\n\/\/\u5176\u5b83\u64cd\u4f5c\uff1a<\/pre>\r\nscala> var rdd1 = sc.parallelize(List("12","34","567","8901"),2)\r\nrdd1: org.apache.spark.rdd.RDD[String] = ParallelCollectionRDD[0] at parallelize at <console>:24\r\n\r\nscala> def fun1(index:Int,iter:Iterator[String]):Iterator[String]={\r\n| iter.toList.map(x => "[partID: "+index+",value:"+x+"]").iterator}\r\nfun1: (index: Int, iter: Iterator[String])Iterator[String]\r\n\r\nscala> rdd1.mapPartitionsWithIndex(fun1).collect\r\nres1: Array[String] = Array(\r\n[partID: 0,value:12], [partID: 0,value:34],\r\n[partID: 1,value:567], [partID: 1,value:8901])\r\n\r\nscala> rdd1.aggregate("")((x,y)=>math.max(x.length,y.length).toString,(x,y)=>x+y)\r\n\r\nscala> rdd1.aggregate("")((x,y)=>math.max(x.length,y.length).toString,(x,y)=>x+y)\r\nres3: String = 42\r\n\r\nscala> rdd1.aggregate("")((x,y)=>math.max(x.length,y.length).toString,(x,y)=>x+y)\r\nres4: String = 24\r\n\r\n\u5206\u6790\uff1a\r\n\u7b2c\u4e00\u4e2a\u5206\u533a\uff1a\u201c12\u201d\uff0c\u201c34\u201d\r\n\u7b2c\u4e00\u6b21\u6bd4\u8f83\uff1a\u201c\u201d\uff0c\u201c12\u201d=2.toString ==\u300b \u201c2\u201d\r\n\u7b2c\u4e8c\u6b21\u6bd4\u8f83\uff1a\u201c2\u201d\uff0c\u201c34\u201d=2.toString ==\u300b \u201c2\u201d\r\n\u7b2c\u4e8c\u4e2a\u5206\u533a\uff1a\u201c567\u201d\uff0c\u201c8901\u201d\r\n\u7b2c\u4e00\u6b21\u6bd4\u8f83\uff1a\u201c\u201d\uff0c\u201c567\u201d=3.toString ==\u300b\u201c3\u201d\r\n\u7b2c\u4e8c\u6b21\u6bd4\u8f83\uff1a\u201c3\u201d\uff0c\u201c8901\u201d=4.toString ==\u300b \u201c4\u201d\r\n\u201c24\u201d\u6216\u8005\u201c42\u201d\r\n\r\nscala> var rdd1 = sc.parallelize(List("12","23","345",""),2)\r\nrdd1: org.apache.spark.rdd.RDD[String] = ParallelCollectionRDD[2] at parallelize at <console>:24\r\n\r\nscala> rdd1.mapPartitionsWithIndex(fun1).collect\r\nres6: Array[String] = Array([partID: 0,value:12], [partID: 0,value:23], [partID: 1,value:345], [partID: 1,value:])\r\n\r\nscala> rdd1.aggregate("")((x,y)=>math.min(x.length,y.length).toString,(x,y)=>x+y)\r\nres7: String = 10\r\n\r\nscala> rdd1.aggregate("")((x,y)=>math.min(x.length,y.length).toString,(x,y)=>x+y)\r\nres9: String = 01\r\n\r\n\u5206\u6790\uff1a\r\n\u7b2c\u4e00\u4e2a\u5206\u533a\uff1a\u201c12\u201d\uff0c\u201c34\u201d\r\n\u7b2c\u4e00\u6b21\u6bd4\u8f83\uff1a\u201c\u201d\uff0c\u201c12\u201d=0.toString ==\u300b \u201c0\u201d\r\n\u7b2c\u4e8c\u6b21\u6bd4\u8f83\uff1a\u201c0\u201d\uff0c\u201c34\u201d=1.toString ==\u300b \u201c1\u201d\r\n\u7b2c\u4e8c\u4e2a\u5206\u533a\uff1a\u201c345\u201d\uff0c\u201c\u201d\r\n\u7b2c\u4e00\u6b21\u6bd4\u8f83\uff1a\u201c\u201d\uff0c\u201c345\u201d=0.toString ==\u300b\u201c0\u201d\r\n\u7b2c\u4e8c\u6b21\u6bd4\u8f83\uff1a\u201c0\u201d\uff0c\u201c\u201d=0.toString ==\u300b \u201c0\u201d\r\n\u201c10\u201d\u6216\u8005\u201c01\u201d\r\n\r\nscala> var rdd1 = sc.parallelize(List("12","23","","345"),2)\r\nrdd1: org.apache.spark.rdd.RDD[String] = ParallelCollectionRDD[5] at parallelize at <console>:24\r\n\r\nscala> rdd1.mapPartitionsWithIndex(fun1).collect\r\nres10: Array[String] = Array([partID: 0,value:12], [partID: 0,value:23], [partID: 1,value:], [partID: 1,value:345])\r\n\r\nscala> rdd1.aggregate("")((x,y)=>math.min(x.length,y.length).toString,(x,y)=>x+y)\r\nres11: String = 11\r\n\r\n\u5206\u6790\uff1a\r\n\u7b2c\u4e00\u4e2a\u5206\u533a\uff1a\u201c12\u201d\uff0c\u201c34\u201d\r\n\u7b2c\u4e00\u6b21\u6bd4\u8f83\uff1a\u201c\u201d\uff0c\u201c12\u201d=0.toString ==\u300b \u201c0\u201d\r\n\u7b2c\u4e8c\u6b21\u6bd4\u8f83\uff1a\u201c0\u201d\uff0c\u201c34\u201d=1.toString ==\u300b \u201c1\u201d\r\n\u7b2c\u4e8c\u4e2a\u5206\u533a\uff1a\u201c\u201d\uff0c\u201c345\u201d\r\n\u7b2c\u4e00\u6b21\u6bd4\u8f83\uff1a\u201c\u201d\uff0c\u201c\u201d=0.toString ==\u300b\u201c0\u201d\r\n\u7b2c\u4e8c\u6b21\u6bd4\u8f83\uff1a\u201c0\u201d\uff0c\u201c345\u201d=1.toString ==\u300b \u201c1\u201d\r\n\u201c11\u201d\r\n<\/pre>\n
 <\/p>\n","protected":false},"excerpt":{"rendered":"
    scala> var rdd1 = sc.textFile("hd […]<\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[30],"tags":[],"class_list":["post-1440","post","type-post","status-publish","format-standard","hentry","category-bigdata"],"blocksy_meta":[],"_links":{"self":[{"href":"https:\/\/notes.coremix.net\/index.php?rest_route=\/wp\/v2\/posts\/1440","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/notes.coremix.net\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/notes.coremix.net\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/notes.coremix.net\/index.php?rest_route=\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/notes.coremix.net\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=1440"}],"version-history":[{"count":2,"href":"https:\/\/notes.coremix.net\/index.php?rest_route=\/wp\/v2\/posts\/1440\/revisions"}],"predecessor-version":[{"id":1442,"href":"https:\/\/notes.coremix.net\/index.php?rest_route=\/wp\/v2\/posts\/1440\/revisions\/1442"}],"wp:attachment":[{"href":"https:\/\/notes.coremix.net\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=1440"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/notes.coremix.net\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=1440"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/notes.coremix.net\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=1440"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}