[root@linux01 install]# tar -zxf flume-ng-1.6.0-cdh5.14.0.tar.gz -C ../soft
[root@linux01 soft]# mv apache-flume-1.6.0-cdh5.14.0-bin/ flume160
[root@linux01 soft]# cd flume160/
[root@linux01 flume160]# cd conf/
[root@linux01 conf]# cp flume-env.sh.template flume-env.sh
[root@linux01 conf]# vi ./flume-env.sh
22 export JAVA_HOME=/opt/soft/jdk180
25 # export JAVA_OPTS="-Xms3000m -Xmx3000m -Dcom.sun.management.jmxremote"
[root@linux01 conf]# yum install -y nc
[root@linux01 conf]# nc -lk 44444
[root@linux01 conf]# yum install telnet-server
[root@linux01 conf]# yum install telnet.*
[root@linux01 conf]# telnet localhost 44444
println("--------------------------------------------")
[root@linux01 kb15conf]# vi ./netcat-flume-logger.conf
a1.sources=r1
a1.channels=c1
a1.sinks=k1
a1.sources.r1.type=netcat
a1.sources.r1.bind=localhost
a1.sources.r1.port=44444
a1.channels.c1.type=memory
a1.channels.c1.capacity=1000
a1.channels.c1.transactionCapacity=100
a1.sinks.k1.type=logger
a1.sources.r1.channels=c1
a1.sinks.k1.channel=c1
[root@linux01 flume160]# ./bin/flume-ng agent --name a1 --conf ./conf/ --conf-file ./conf/kb15conf/netcat-flume-logger.conf -Dflume.root.logger=INFO,console(先)
[root@linux01 conf]# telnet localhost 44444(后)
println("--------------------------------------------")
logger
[root@linux01 flume160]#./bin/flume-ng agent --name a2 --conf ./conf/ --conf-file ./conf/kb15conf/file-flume-logger.conf -Dflume.root.logger=INFO,console(先)
[root@linux01 kb15conf]# echo hello spark >> ./flumefile.log(后)
[root@linux01 kb15conf]# echo hello flume >> ./flumefile.log(后)
[root@linux01 kb15conf]# vi ./file-flume-hdfs.conf
a2.sources=r1
a2.channels=c1
a2.sinks=k1
a2.sources.r1.type=exec
a2.sources.r1.command=tail -f /opt/soft/flume160/conf/kb15conf/flumefile.log
a2.channels.c1.type=memory
a2.channels.c1.capacity=1000
a2.channels.c1.transactionCapacity=100
a2.sinks.k1.type=hdfs
a2.sinks.k1.hdfs.fileType=DataStream
a2.sinks.k1.hdfs.filePrefix=flumetohdfs
a2.sinks.k1.hdfs.fileSuffix=.txt
a2.sinks.k1.hdfs.path=hdfs://192.168.111.131:9000/kb15file
a2.sources.r1.channels=c1
a2.sinks.k1.channel=c1
上传到hdfs
[root@linux01 kb15conf]# hdfs dfsadmin -safemode leave
[root@linux01 flume160]# ./bin/flume-ng agent --name a2 --conf ./conf/ --conf-file ./conf/kb15conf/file-flume-hdfs.conf -Dflume.root.logger=INFO,console(先)
[root@linux01 kb15conf]# echo hello spark >> ./flumefile.log(后)
[root@linux01 kb15conf]# echo hello flume >> ./flumefile.log(后)
