[TOC] # 简介 **flume** cloudera公司研发 适合下游数据消费者不多的情况 适合数据安全性要求不多的情况 适合与hadoop生态圈对接的操作 **kafka** linkedin公司研发 适合数据下游消费众多的情况 适合数据安全性要求比较高的操作,支持replication # 代码 配置flume(flume-kafka.conf) ~~~ # define a1.sources = r1 a1.sinks = k1 a1.channels = c1 # source a1.sources.r1.type = exec # 加-c +0表示从头开始读并开始监控,这个文件有可能被很多人监控过 a1.sources.r1.command = tail -F -c +0 /opt/module/datas/flume.log a1.sources.r1.shell = /bin/bash -c # sink a1.sinks.k1.type = org.apache.flume.sink.kafka.KafkaSink a1.sinks.k1.kafka.bootstrap.servers = master:9092,slave1:9092,slave2:9092 a1.sinks.k1.kafka.topic = first a1.sinks.k1.kafka.flumeBatchSize = 20 a1.sinks.k1.kafka.producer.acks = 1 a1.sinks.k1.kafka.producer.linger.ms = 1 # channel a1.channels.c1.type = memory a1.channels.c1.capacity = 1000 a1.channels.c1.transactionCapacity = 100 # bind a1.sources.r1.channels = c1 a1.sinks.k1.channel = c1 ~~~