{"id":1465,"date":"2022-06-19T21:36:41","date_gmt":"2022-06-19T13:36:41","guid":{"rendered":"http:\/\/www.eait.co\/?p=1465"},"modified":"2022-06-19T21:36:41","modified_gmt":"2022-06-19T13:36:41","slug":"spark-idea%e4%bb%a3%e7%a0%81%e5%88%9d%e6%ad%a5","status":"publish","type":"post","link":"https:\/\/notes.coremix.net\/?p=1465","title":{"rendered":"spark IDEA\u4ee3\u7801\u521d\u6b65"},"content":{"rendered":"

IDEA\u901a\u8fc7DS+Row\u67e5\u8be2\u6570\u636e<\/p>\n

\r\npackage day0106\r\n\r\nimport java.util.Properties\r\n\r\nimport org.apache.spark.sql.{Row, SparkSession}\r\nimport org.apache.spark.sql.types.{IntegerType, StringType, StructField, StructType}\r\n\r\n\/**\r\n * \u6570\u636e\u5199\u5165MySQL\r\n * *\/\r\nobject Demo3 extends App {\r\n  \/\/\u83b7\u53d6spark\u73af\u5883\r\n  val spark = SparkSession.builder().appName("DF").master("local").getOrCreate()\r\n\r\n\r\n\r\n  \/\/\u8bfb\u53d6\u6570\u636e\r\n  var stuRDD = spark.sparkContext.textFile("D:\\\\testdata\\\\student.txt").map(_.split("\\t"))\r\n\r\n\r\n  \/\/\u901a\u8fc7Struct Type\u521b\u5efaSchema\r\n  val schema = StructType(\r\n    List(\r\n      StructField("id",IntegerType),\r\n      StructField("name",StringType),\r\n      StructField("age",IntegerType)\r\n    )\r\n  )\r\n\r\n  \/\/\u5c06\u6570\u636e\u6620\u5c04\u5230Row\uff0c\u521b\u5efaDF\r\n  val RowRDD = stuRDD.map(s => Row(s(0).toInt,s(1),s(2).toInt))\r\n  val stuDF = spark.createDataFrame(RowRDD,schema)\r\n\r\n  \/\/\u6ce8\u518c\u8868\r\n  stuDF.createOrReplaceTempView("student")\r\n\r\n  \/\/sql\u64cd\u4f5c\r\n  val result = spark.sql("select * from student order by age desc")\r\n\r\n  \/\/\u5199\u5165MySQL\u4e2d\r\n  val myPro = new Properties()\r\n  myPro.setProperty("user","root")\r\n  myPro.setProperty("password","000000")\r\n  myPro.setProperty("driver","com.mysql.jdbc.Driver")\r\n\r\n  result.write.mode("overwrite").jdbc("jdbc:mysql:\/\/bigdata166:3306\/company?serverTimezone=UTC&characterEncoding=utf-8","student",myPro)\r\n\r\n  spark.stop()\r\n\r\n}\r\n\r\n<\/pre>\n
IDEA\u901a\u8fc7case class \u521b\u5efaDF<\/p>\n
\r\n\r\n<\/pre>\n
IDEA\u6570\u636e\u5199\u5165mysql<\/p>\n
\r\npackage day0106\r\n\r\nimport java.util.Properties\r\n\r\nimport org.apache.spark.sql.{Row, SparkSession}\r\nimport org.apache.spark.sql.types.{IntegerType, StringType, StructField, StructType}\r\n\r\n\/**\r\n * \u6570\u636e\u5199\u5165MySQL\r\n * *\/\r\nobject Demo3 extends App {\r\n  \/\/\u83b7\u53d6spark\u73af\u5883\r\n  val spark = SparkSession.builder().appName("DF").master("local").getOrCreate()\r\n\r\n\r\n\r\n  \/\/\u8bfb\u53d6\u6570\u636e\r\n  var stuRDD = spark.sparkContext.textFile("D:\\\\testdata\\\\student.txt").map(_.split("\\t"))\r\n\r\n\r\n  \/\/\u901a\u8fc7Struct Type\u521b\u5efaSchema\r\n  val schema = StructType(\r\n    List(\r\n      StructField("id",IntegerType),\r\n      StructField("name",StringType),\r\n      StructField("age",IntegerType)\r\n    )\r\n  )\r\n\r\n  \/\/\u5c06\u6570\u636e\u6620\u5c04\u5230Row\uff0c\u521b\u5efaDF\r\n  val RowRDD = stuRDD.map(s => Row(s(0).toInt,s(1),s(2).toInt))\r\n  val stuDF = spark.createDataFrame(RowRDD,schema)\r\n\r\n  \/\/\u6ce8\u518c\u8868\r\n  stuDF.createOrReplaceTempView("student")\r\n\r\n  \/\/sql\u64cd\u4f5c\r\n  val result = spark.sql("select * from student order by age desc")\r\n\r\n  \/\/\u5199\u5165MySQL\u4e2d\r\n  val myPro = new Properties()\r\n  myPro.setProperty("user","root")\r\n  myPro.setProperty("password","000000")\r\n  myPro.setProperty("driver","com.mysql.jdbc.Driver")\r\n\r\n  result.write.mode("overwrite").jdbc("jdbc:mysql:\/\/bigdata166:3306\/company?serverTimezone=UTC&characterEncoding=utf-8","student",myPro)\r\n\r\n  spark.stop()\r\n\r\n}\r\n\r\n<\/pre>\n","protected":false},"excerpt":{"rendered":"
IDEA\u901a\u8fc7DS+Row\u67e5\u8be2\u6570\u636e package day0106 import java.util.Prope […]<\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[1],"tags":[],"class_list":["post-1465","post","type-post","status-publish","format-standard","hentry","category-uncategorized"],"blocksy_meta":[],"_links":{"self":[{"href":"https:\/\/notes.coremix.net\/index.php?rest_route=\/wp\/v2\/posts\/1465","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/notes.coremix.net\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/notes.coremix.net\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/notes.coremix.net\/index.php?rest_route=\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/notes.coremix.net\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=1465"}],"version-history":[{"count":1,"href":"https:\/\/notes.coremix.net\/index.php?rest_route=\/wp\/v2\/posts\/1465\/revisions"}],"predecessor-version":[{"id":1469,"href":"https:\/\/notes.coremix.net\/index.php?rest_route=\/wp\/v2\/posts\/1465\/revisions\/1469"}],"wp:attachment":[{"href":"https:\/\/notes.coremix.net\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=1465"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/notes.coremix.net\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=1465"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/notes.coremix.net\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=1465"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}