{"id":1295,"date":"2022-02-28T20:23:40","date_gmt":"2022-02-28T12:23:40","guid":{"rendered":"http:\/\/www.eait.co\/?p=1295"},"modified":"2022-02-28T20:23:40","modified_gmt":"2022-02-28T12:23:40","slug":"flume%ef%bc%88%e4%ba%8c%ef%bc%89","status":"publish","type":"post","link":"https:\/\/notes.coremix.net\/?p=1295","title":{"rendered":"Flume\uff08\u4e8c\uff09"},"content":{"rendered":"<p><strong>\u76d1\u63a7\u6587\u4ef6\u5939\uff1a<\/strong><\/p>\n<p>\u5c16\u53eb\u63d0\u793a\uff1a \u5728\u4f7f\u7528Spooling Directory Source\u65f6<\/p>\n<p>1) \u4e0d\u8981\u5728\u76d1\u63a7\u76ee\u5f55\u4e2d\u521b\u5efa\u5e76\u6301\u7eed\u4fee\u6539\u6587\u4ef6<\/p>\n<p>2) \u4e0a\u4f20\u5b8c\u6210\u7684\u6587\u4ef6\u4f1a\u4ee5.COMPLETED\u7ed3\u5c3e<\/p>\n<p>3) \u88ab\u76d1\u63a7\u6587\u4ef6\u5939\u6bcf500\u6beb\u79d2\u626b\u63cf\u4e00\u6b21\u6587\u4ef6\u53d8\u52a8<\/p>\n<pre class=\"brush: bash; title: ; notranslate\" title=\"\">\r\n#1 Agent\r\na3.sources = r3\r\na3.sinks = k3\r\na3.channels = c3\r\n\r\n#2 source\r\na3.sources.r3.type = spooldir\r\na3.sources.r3.spoolDir = \/opt\/module\/flume-1.8.0\/uploads\r\na3.sources.r3.fileSuffix = .COMPLETEDs\r\na3.sources.r3.fileHeader = true\r\n#\u5ffd\u7565\u6240\u6709\u4ee5.tmp\u7ed3\u5c3e\u7684\u6587\u4ef6\uff0c\u4e0d\u4e0a\u4f20\r\na3.sources.r3.ignorePattern = (&#x5B;^ ]*\\.tmp)\r\n\r\n# 3 sink\r\na3.sinks.k3.type = hdfs\r\na3.sinks.k3.hdfs.path = hdfs:\/\/bigdata166:9000\/flume\/%H\r\n#\u4e0a\u4f20\u6587\u4ef6\u7684\u524d\u7f00\r\na3.sinks.k3.hdfs.filePrefix = upload-\r\n#\u662f\u5426\u6309\u7167\u65f6\u95f4\u6eda\u52a8\u6587\u4ef6\u5939\r\na3.sinks.k3.hdfs.round = true\r\n#\u591a\u5c11\u65f6\u95f4\u5355\u4f4d\u521b\u5efa\u4e00\u4e2a\u65b0\u7684\u6587\u4ef6\u5939\r\na3.sinks.k3.hdfs.roundValue = 1\r\n#\u91cd\u65b0\u5b9a\u4e49\u65f6\u95f4\u5355\u4f4d\r\na3.sinks.k3.hdfs.roundUnit = hour\r\n#\u662f\u5426\u4f7f\u7528\u672c\u5730\u65f6\u95f4\u6233\r\na3.sinks.k3.hdfs.useLocalTimeStamp = true\r\n#\u79ef\u6512\u591a\u5c11\u4e2aEvent\u624dflush\u5230HDFS\u4e00\u6b21\r\na3.sinks.k3.hdfs.batchSize = 100\r\n#\u8bbe\u7f6e\u6587\u4ef6\u7c7b\u578b\uff0c\u53ef\u652f\u6301\u538b\u7f29\r\na3.sinks.k3.hdfs.fileType = DataStream\r\n#\u591a\u4e45\u751f\u6210\u4e00\u4e2a\u65b0\u7684\u6587\u4ef6\r\na3.sinks.k3.hdfs.rollInterval = 600\r\n#\u8bbe\u7f6e\u6bcf\u4e2a\u6587\u4ef6\u7684\u6eda\u52a8\u5927\u5c0f\u5927\u6982\u662f128M\r\na3.sinks.k3.hdfs.rollSize = 134217700\r\n#\u6587\u4ef6\u7684\u6eda\u52a8\u4e0eEvent\u6570\u91cf\u65e0\u5173   \u9632\u6b62\u521b\u5efa\u591a\u4e2a\u5c0f\u6587\u4ef6\r\na3.sinks.k3.hdfs.rollCount = 0\r\n#\u6700\u5c0f\u526f\u672c\u6570\r\na3.sinks.k3.hdfs.minBlockReplicas = 1\r\n\r\n# Use a channel which buffers events in memory\r\na3.channels.c3.type = memory\r\na3.channels.c3.capacity = 1000\r\na3.channels.c3.transactionCapacity = 100\r\n\r\n# Bind the source and sink to the channel\r\na3.sources.r3.channels = c3\r\na3.sinks.k3.channel = c3\r\n<\/pre>\n<p><strong>\u542f\u52a8\uff1a<\/strong><\/p>\n<pre class=\"brush: bash; title: ; notranslate\" title=\"\">\r\n\/opt\/module\/flume1.8.0\/bin\/flume-ng agent \\\r\n--conf \/opt\/module\/flume1.8.0\/conf\/ \\\r\n--name a3 \\\r\n--conf-file \/opt\/module\/flume1.8.0\/jobconf\/flume-dir.conf\r\n\r\n<\/pre>\n","protected":false},"excerpt":{"rendered":"<p>\u76d1\u63a7\u6587\u4ef6\u5939\uff1a \u5c16\u53eb\u63d0\u793a\uff1a \u5728\u4f7f\u7528Spooling Directory Source\u65f6 1) \u4e0d\u8981\u5728\u76d1\u63a7\u76ee\u5f55\u4e2d\u521b [&hellip;]<\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[1],"tags":[],"class_list":["post-1295","post","type-post","status-publish","format-standard","hentry","category-uncategorized"],"blocksy_meta":[],"_links":{"self":[{"href":"https:\/\/notes.coremix.net\/index.php?rest_route=\/wp\/v2\/posts\/1295","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/notes.coremix.net\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/notes.coremix.net\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/notes.coremix.net\/index.php?rest_route=\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/notes.coremix.net\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=1295"}],"version-history":[{"count":1,"href":"https:\/\/notes.coremix.net\/index.php?rest_route=\/wp\/v2\/posts\/1295\/revisions"}],"predecessor-version":[{"id":1296,"href":"https:\/\/notes.coremix.net\/index.php?rest_route=\/wp\/v2\/posts\/1295\/revisions\/1296"}],"wp:attachment":[{"href":"https:\/\/notes.coremix.net\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=1295"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/notes.coremix.net\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=1295"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/notes.coremix.net\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=1295"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}