{"id":1470,"date":"2022-06-19T21:47:41","date_gmt":"2022-06-19T13:47:41","guid":{"rendered":"http:\/\/www.eait.co\/?p=1470"},"modified":"2022-06-19T21:47:41","modified_gmt":"2022-06-19T13:47:41","slug":"%e9%ab%98%e7%ba%a7sql%e8%ae%b0%e5%bd%95%ef%bc%9a","status":"publish","type":"post","link":"https:\/\/notes.coremix.net\/?p=1470","title":{"rendered":"\u9ad8\u7ea7sql\u8bb0\u5f55\uff1a"},"content":{"rendered":"
\r\n\r\npackage day0108\r\n\r\n\r\nimport java.text.SimpleDateFormat\r\n\r\nimport org.apache.log4j.{Level, Logger}\r\nimport org.apache.spark.sql.SparkSession\r\n\r\nobject SparkSqlDemo {\r\n  def main(args: Array[String]): Unit = {\r\n\r\n    \/\/\u53bb\u9664\u65e0\u7528INFO\r\n    Logger.getLogger("org.apache.spark").setLevel(Level.ERROR)\r\n    Logger.getLogger("org.eclipse.jetty.server").setLevel(Level.OFF)\r\n\r\n    \/\/\u83b7\u53d6sparksession\r\n    val spark = SparkSession.builder().master("local").appName("sqlDemo").getOrCreate()\r\n    \/\/\u8bfb\u53d6\u6570\u636e,\u8868\u7ed3\u6784\u6620\u5c04\uff0c\u751f\u6210DF\uff0c\u6ce8\u518c\u8868\r\n    import spark.sqlContext.implicits._\r\n    \/\/\u521b\u5efa\u5b66\u751f\u8868\r\n    spark.sparkContext.textFile("D:\\\\testdata\\\\streaming\\\\spark_sql_test_data\\\\Student.csv").\r\n      map(_.split(","))\r\n      .map(x => Student(x(0),x(1),x(2),x(3),x(4)))\r\n      .toDF\r\n      .createOrReplaceTempView("student")\r\n\r\n    \/\/\u521b\u5efa\u6210\u7ee9\u8868\r\n    spark.sparkContext.textFile("D:\\\\testdata\\\\streaming\\\\spark_sql_test_data\\\\Score.csv").\r\n      map(_.split(","))\r\n      .map(x => Score(x(0),x(1),x(2)))\r\n      .toDF\r\n      .createOrReplaceTempView("score")\r\n\r\n    \/\/\u521b\u5efa\u8bfe\u7a0b\u8868\r\n    spark.sparkContext.textFile("D:\\\\testdata\\\\streaming\\\\spark_sql_test_data\\\\Course.csv").\r\n      map(_.split(","))\r\n      .map(x => Course(x(0),x(1),x(2)))\r\n      .toDF\r\n      .createOrReplaceTempView("course")\r\n    \/\/\u521b\u5efa\u6559\u5e08\u8868\r\n    spark.sparkContext.textFile("D:\\\\testdata\\\\streaming\\\\spark_sql_test_data\\\\Teacher.csv").\r\n      map(_.split(","))\r\n      .map(x => Teacher(x(0),x(1),x(2),x(3),x(4),x(5)))\r\n      .toDF\r\n      .createOrReplaceTempView("teacher")\r\n\r\n\/\/    spark.sql("select * from student").show()\r\n\/\/    spark.sql("select * from score").show()\r\n\/\/    spark.sql("select * from course").show()\r\n\/\/    spark.sql("select * from teacher").show(false)\r\n\r\n\r\n\r\n    \/\/\u6ce8\u610f\uff1a\u6392\u5e8f\u4f7f\u7528String\u7c7b\u578b\u6309\u7167\u5b57\u5178\u987a\u5e8f\u6392\u5e8f,\u9700\u8981\u8fdb\u884c\u7c7b\u578b\u8f6c\u6362\r\n\/\/    spark.sql("select degree from score order by int(degree) desc ").show()\r\n\r\n    \/\/\u67e5\u8be2\u6ca1\u95e8\u8bfe\u7a0b\u7684\u5e73\u5747\u6210\u7ee9;spark-sql\u5bf9\u4e8e\u5927\u5c0f\u5199\u4e5f\u662f\u4e0d\u654f\u611f\u7684\r\n\/\/    spark.sql("select cnum,avg(cast(degree as int))  from score group by cnum").show()\r\n\/\/    spark.sql("select cnum,avg(degree)  from SCORE group by cnum").show()\r\n\r\n\r\n    \/\/\u67e5\u8be2score\u4e2d\u81f3\u5c11\u67095\u540d\u5b66\u751f\u9009\u8bfe\uff0c\u5e76\u4e14\u8bfe\u7a0b\u7f16\u53f7\u662f3\u5f00\u5934\u7684\u8bfe\u7a0b\u7684\u5e73\u5747\u5206\r\n\/\/    spark.sql("select cnum,avg(degree) from score where cnum like '3%' group by cnum having count(1) >=5").show()\r\n\r\n    \/\/\u67e5\u8be2score\u8868\u4e2d\u9009\u62e9\u591a\u95e8\u8bfe\u7a0b\u7684\u540c\u5b66\uff0c\u6700\u9ad8\u5206\u7684\u5b66\u751f\r\n\/\/    spark.sql("select snum,degree from score where" +\r\n\/\/      " snum in (select t.snum from score t group by t.snum having count(t.snum)>1) " +\r\n\/\/      "and degree = (select max(degree) from score) ").show()\r\n\r\n\r\n    \/\/\u67e5\u8be2\u5b66\u751f\u7684\u5b66\u53f7\u4e3a108\u7684\u540c\u5e74\u7684\u6240\u6709\u5b66\u751f\r\n\/\/    spark.sql("select * from student where " +\r\n\/\/      "substring(sbirthday,0,4) = " +\r\n\/\/      "(select substring(sbirthday,0,4) from student where snum = '108')" ).show()\r\n\r\n    \/\/\u67e5\u8be2\u9009\u4fee\u8bfe\u7a0b\u4eba\u6570\u5927\u4e8e5\uff0c\u7684\u8bfe\u7a0b\u7684\u6559\u5e08\u7684\u59d3\u540d\r\n\/\/    spark.sql("select tname from teacher e " +\r\n\/\/      "join course c on e.tnum = c.tnum " +\r\n\/\/      "join (select cnum from score group by cnum having count(1)>5) t on t.cnum = c.cnum").show()\r\n\r\n    \/\/\u67e5\u8be2\u6210\u7ee9\u6bd4\u5e73\u5747\u6210\u7ee9\u4f4e\u7684\u5b66\u751f\u6210\u7ee9\r\n\/\/    spark.sql("select * from score s where s.degree < (select avg(degree) from score c where s.snum = c.snum)").show()\r\n\r\n    \/\/\u67e5\u8be2\u6240\u6709\u6ca1\u6709\u8bb2\u8bfe\u7684\u6559\u5e08\r\n\/\/    spark.sql("select * from teacher t where t.tnum not in " +\r\n\/\/      "(select tnum from course c where c.cnum in " +\r\n\/\/      "(select cnum from score))").show(false)\r\n\r\n    \/\/\u67e5\u8be2\u5b66\u751f\u8868\u7684\u5e74\u9f84\r\n\r\n    spark.sql("select sname , " +\r\n          "(" +\r\n          "cast("+getDate("yyyy")+" as int) " +\r\n          "- " +\r\n          "cast(substring(sbirthday,0,4) as int) " +\r\n          ") as age from student").show()\r\n\r\n    spark.sql("select sname ,"+getDate("yyyy")+" - substring(sbirthday,0,4) from student").show()\r\n\r\n\r\n    spark.stop()\r\n\r\n\r\n  }\r\n\r\n  \/\/\u83b7\u53d6\u5f53\u524d\u65f6\u95f4\r\n  def getDate(time:String): String ={\r\n    val now:Long = System.currentTimeMillis()\r\n    val df:SimpleDateFormat = new SimpleDateFormat(time)\r\n    df.format(now)\r\n  }\r\n\r\n}\r\n\r\n<\/pre>\n","protected":false},"excerpt":{"rendered":"
package day0108 import java.text.SimpleDateFormat impor […]<\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[1],"tags":[],"class_list":["post-1470","post","type-post","status-publish","format-standard","hentry","category-uncategorized"],"blocksy_meta":[],"_links":{"self":[{"href":"https:\/\/notes.coremix.net\/index.php?rest_route=\/wp\/v2\/posts\/1470","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/notes.coremix.net\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/notes.coremix.net\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/notes.coremix.net\/index.php?rest_route=\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/notes.coremix.net\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=1470"}],"version-history":[{"count":2,"href":"https:\/\/notes.coremix.net\/index.php?rest_route=\/wp\/v2\/posts\/1470\/revisions"}],"predecessor-version":[{"id":1472,"href":"https:\/\/notes.coremix.net\/index.php?rest_route=\/wp\/v2\/posts\/1470\/revisions\/1472"}],"wp:attachment":[{"href":"https:\/\/notes.coremix.net\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=1470"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/notes.coremix.net\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=1470"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/notes.coremix.net\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=1470"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}