{"id":1301,"date":"2022-03-02T22:08:54","date_gmt":"2022-03-02T14:08:54","guid":{"rendered":"http:\/\/www.eait.co\/?p=1301"},"modified":"2022-03-02T22:08:54","modified_gmt":"2022-03-02T14:08:54","slug":"flume%e5%90%84%e7%a7%8d%e6%8b%a6%e6%88%aa%e5%99%a8%ef%bc%88%e4%b8%80%ef%bc%89","status":"publish","type":"post","link":"https:\/\/notes.coremix.net\/?p=1301","title":{"rendered":"Flume\u5404\u79cd\u62e6\u622a\u5668\uff08\u4e00\uff09"},"content":{"rendered":"<h4><span style=\"font-family: '\u5fae\u8f6f\u96c5\u9ed1',sans-serif;\">\u65f6\u95f4\u6233\u62e6\u622a\u5668<\/span><\/h4>\n<p>Timestamp.conf<\/p>\n<pre class=\"brush: bash; title: ; notranslate\" title=\"\">\r\n\r\n#1.\u5b9a\u4e49agent\u540d\uff0c source\u3001channel\u3001sink\u7684\u540d\u79f0\r\na4.sources = r1\r\na4.channels = c1\r\na4.sinks = k1\r\n\r\n#2.\u5177\u4f53\u5b9a\u4e49source\r\na4.sources.r1.type = spooldir\r\na4.sources.r1.spoolDir = \/opt\/module\/flume-1.8.0\/upload\r\n\r\n#\u5b9a\u4e49\u62e6\u622a\u5668\uff0c\u4e3a\u6587\u4ef6\u6700\u540e\u6dfb\u52a0\u65f6\u95f4\u6233\r\na4.sources.r1.interceptors = i1\r\na4.sources.r1.interceptors.i1.type = org.apache.flume.interceptor.TimestampInterceptor$Builder\r\n\r\n#\u5177\u4f53\u5b9a\u4e49channel\r\na4.channels.c1.type = memory\r\na4.channels.c1.capacity = 10000\r\na4.channels.c1.transactionCapacity = 100\r\n\r\n\r\n#\u5177\u4f53\u5b9a\u4e49sink\r\na4.sinks.k1.type = hdfs\r\na4.sinks.k1.hdfs.path = hdfs:\/\/bigdata166:9000\/flume-interceptors\/%H\r\na4.sinks.k1.hdfs.filePrefix = events-\r\na4.sinks.k1.hdfs.fileType = DataStream\r\n\r\n#\u4e0d\u6309\u7167\u6761\u6570\u751f\u6210\u6587\u4ef6\r\na4.sinks.k1.hdfs.rollCount = 0\r\n#HDFS\u4e0a\u7684\u6587\u4ef6\u8fbe\u5230128M\u65f6\u751f\u6210\u4e00\u4e2a\u6587\u4ef6\r\na4.sinks.k1.hdfs.rollSize = 134217728\r\n#HDFS\u4e0a\u7684\u6587\u4ef6\u8fbe\u523060\u79d2\u751f\u6210\u4e00\u4e2a\u6587\u4ef6\r\na4.sinks.k1.hdfs.rollInterval = 60\r\n\r\n#\u7ec4\u88c5source\u3001channel\u3001sink\r\na4.sources.r1.channels = c1\r\na4.sinks.k1.channel = c1\r\n<\/pre>\n<pre class=\"brush: bash; title: ; notranslate\" title=\"\">\r\n\/opt\/module\/flume-1.8.0\/bin\/flume-ng agent -n a4 \\\r\n-f \/opt\/module\/flume-1.8.0\/jobconf\/Timestamp.conf \\\r\n-c \/opt\/module\/flume-1.8.0\/conf \\\r\n-Dflume.root.logger=INFO,console\r\n<\/pre>\n<p>&nbsp;<\/p>\n<p>&nbsp;<\/p>\n<h4><span style=\"font-family: '\u5fae\u8f6f\u96c5\u9ed1',sans-serif;\">\u4e3b\u673a\u540d\u62e6\u622a\u5668<\/span><\/h4>\n<p>Host.conf<\/p>\n<pre class=\"brush: bash; title: ; notranslate\" title=\"\">\r\n\r\n#1.\u5b9a\u4e49agent\r\na1.sources= r1\r\na1.sinks = k1\r\na1.channels = c1\r\n\r\n#2.\u5b9a\u4e49source\r\na1.sources.r1.type = exec\r\na1.sources.r1.channels = c1\r\na1.sources.r1.command = tail -F \/opt\/Shalltest\r\n#\u62e6\u622a\u5668\r\na1.sources.r1.interceptors = i1\r\na1.sources.r1.interceptors.i1.type = host\r\n\r\n#\u53c2\u6570\u4e3atrue\u65f6\u7528IP192.168.1.111\uff0c\u53c2\u6570\u4e3afalse\u65f6\u7528\u4e3b\u673a\u540d\uff0c\u9ed8\u8ba4\u4e3atrue\r\na1.sources.r1.interceptors.i1.useIP = false\r\na1.sources.r1.interceptors.i1.hostHeader = agentHost\r\n\r\n #3.\u5b9a\u4e49sinks\r\na1.sinks.k1.type=hdfs\r\na1.sinks.k1.channel = c1\r\na1.sinks.k1.hdfs.path = hdfs:\/\/bigdata166:9000\/flumehost\/%H\r\na1.sinks.k1.hdfs.filePrefix = Shall_%{agentHost}\r\n#\u5f80\u751f\u6210\u7684\u6587\u4ef6\u52a0\u540e\u7f00\u540d.log\r\na1.sinks.k1.hdfs.fileSuffix = .log\r\na1.sinks.k1.hdfs.fileType = DataStream\r\na1.sinks.k1.hdfs.writeFormat = Text\r\na1.sinks.k1.hdfs.rollInterval = 10\r\na1.sinks.k1.hdfs.useLocalTimeStamp = true\r\n \r\na1.channels.c1.type = memory\r\na1.channels.c1.capacity = 1000\r\na1.channels.c1.transactionCapacity = 100\r\n \r\na1.sources.r1.channels = c1\r\na1.sinks.k1.channel = c1\r\n<\/pre>\n<pre class=\"brush: bash; title: ; notranslate\" title=\"\">\r\nbin\/flume-ng agent -c conf\/ -f jobconf\/Host.conf -n a1 -Dflume.root.logger=INFO,console\r\n<\/pre>\n<p>&nbsp;<\/p>\n<h4><span lang=\"EN-US\" style=\"font-family: \u5b8b\u4f53;\">UUID<\/span><span style=\"font-family: \u5b8b\u4f53;\">\u62e6\u622a\u5668 <\/span><\/h4>\n<p>uuid.conf<\/p>\n<p>&nbsp;<\/p>\n<pre class=\"brush: bash; title: ; notranslate\" title=\"\">\r\na1.sources = r1\r\na1.sinks = k1\r\na1.channels = c1\r\n\r\na1.sources.r1.type = exec\r\na1.sources.r1.channels = c1\r\na1.sources.r1.command = tail -F \/opt\/Shalltest\r\na1.sources.r1.interceptors = i1\r\n#type\u7684\u53c2\u6570\u4e0d\u80fd\u5199\u6210uuid\uff0c\u5f97\u5199\u5177\u4f53\uff0c\u5426\u5219\u627e\u4e0d\u5230\u7c7b\r\na1.sources.r1.interceptors.i1.type = org.apache.flume.sink.solr.morphline.UUIDInterceptor$Builder\r\n#\u5982\u679cUUID\u5934\u5df2\u7ecf\u5b58\u5728,\u5b83\u5e94\u8be5\u4fdd\u5b58\r\na1.sources.r1.interceptors.i1.preserveExisting = true\r\na1.sources.r1.interceptors.i1.prefix = UUID_\r\n\r\n#\u5982\u679csink\u7c7b\u578b\u6539\u4e3aHDFS\uff0c\u90a3\u4e48\u5728HDFS\u7684\u6587\u672c\u4e2d\u6ca1\u6709headers\u7684\u4fe1\u606f\u6570\u636e\r\na1.sinks.k1.type = logger\r\n\r\na1.channels.c1.type = memory\r\na1.channels.c1.capacity = 1000\r\na1.channels.c1.transactionCapacity = 100\r\n\r\na1.sources.r1.channels = c1\r\na1.sinks.k1.channel = c1\r\n<\/pre>\n<pre class=\"brush: bash; title: ; notranslate\" title=\"\">\r\nbin\/flume-ng agent -c conf\/ -f jobconf\/uuid.conf -n a1 -Dflume.root.logger==INFO,console\r\n<\/pre>\n<p>&nbsp;<\/p>\n<p>&nbsp;<\/p>\n","protected":false},"excerpt":{"rendered":"<p>\u65f6\u95f4\u6233\u62e6\u622a\u5668 Timestamp.conf #1.\u5b9a\u4e49agent\u540d\uff0c source\u3001channel\u3001sink\u7684 [&hellip;]<\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[1],"tags":[],"class_list":["post-1301","post","type-post","status-publish","format-standard","hentry","category-uncategorized"],"blocksy_meta":[],"_links":{"self":[{"href":"https:\/\/notes.coremix.net\/index.php?rest_route=\/wp\/v2\/posts\/1301","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/notes.coremix.net\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/notes.coremix.net\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/notes.coremix.net\/index.php?rest_route=\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/notes.coremix.net\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=1301"}],"version-history":[{"count":2,"href":"https:\/\/notes.coremix.net\/index.php?rest_route=\/wp\/v2\/posts\/1301\/revisions"}],"predecessor-version":[{"id":1305,"href":"https:\/\/notes.coremix.net\/index.php?rest_route=\/wp\/v2\/posts\/1301\/revisions\/1305"}],"wp:attachment":[{"href":"https:\/\/notes.coremix.net\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=1301"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/notes.coremix.net\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=1301"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/notes.coremix.net\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=1301"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}