Flume 部署

cd /opt/softs  # 打开flume压缩包所在目录
tar -xvf apache-flume-1.9.0-bin.tar.gz -C /opt/module/  # 解压到 /opt/modul
cd /opt/module  # 打开解压到的目录
mv apache-flume-1.9.0-bin flume-1.9.0  # 重命名

vim /etc/profile

# Flume
export FLUME=/opt/module/flume-1.9.0
export PATH=$PATH:$FLUME_HOME

scp /etc/profile slave1:/etc/
scp /etc/profile slave2:/etc/

cd /opt/module/flume-1.9.0/conf
cp flume-env.sh.template flume-env.sh
vim flume-env.sh

# 添加一行
export JAVA_HOME=/opt/module/jdk1.8.0_212

cd /opt/module/flume-1.9.0/lib
rm -rf guava

cd /opt/module
scp -r flume-1.9.0 slave1:/opt/module/
scp -r flume-1.9.0 slave2:/opt/module

cd /opt/module/flume-1.9.0/lib
cp $HADOOP_HOME/share/hadoop/common/*.jar ./
cp $HADOOP_HOME/share/hadoop/common/lib/*.jar ./
cp $HADOOP_HOME/share/hadoop/hdfs/*.jar ./

cd /opt/module/flume-1.9.0/conf

# 创建相关路径
mkdir /ver/log/flume/ -p
hadoop dfs -mkdir nginx-events

touch fileToHDFS.conf

a1.sources = r1
a1.channels = c1
a1.sinks = k1

a1.sources.r1.type = TAILDIR
a1.sources.r1.channels = c1

# 存储记录位置
a1.sources.r1.positionFile = /var/log/flume/tail_dir.json
a1.sources.r1.filegroups = f1 f2
a1.sources.r1.filegroups.f1 = /var/log/nginx/access.log
a1.sources.r1.filegroups.f2 = /var/log/nginx/error.log

a1.sinks.k1.type = hdfs
a1.sinks.k1.channel = c1
# 保存到HDFS的位置
a1.sinks.k1.hdfs.path = hdfs://master:8020/user/root/nginx-events
# 文件前缀
a1.sinks.k1.hdfs.filePrefix = event-

a1.sinks.k1.hdfs.fileType = DataStream
# 随时间滚动 0 表示不滚动 这里设置1Mb 开启新文件
a1.sinks.k1.hdfs.rollSize = 1048576

# 随event滚动 开启新文件数量 0 关闭 event 开启新文件
a1.sinks.k1.hdfs.rollCount = 0

# 随时间滚动 这里设置60s
a1.sinks.k1.hdfs.rollInterval = 30

# 防止flume自动开启新文件 不管hdfs备份
a1.sinks.k1.hdfs.minBlockReplicas = 1

a1.channels.c1.type = memory
# 存储在通道中的最大事件数
a1.channels.c1.capacity = 10000
# 每次传输 通道从源最大获取到的event
a1.channels.c1.transactionCapacity = 100
#  允许最大占用内存
a1.channels.c1.byteCapacityBufferPercentage = 20
a1.channels.c1.byteCapacity = 800000

a1.sources.r1.channels = c1
a1.sinks.k1.chanels = c1

$FLUME_HOME/bin/flume-ng agent -c $FLUME_HOME/conf/ -f $FLUME_HOME/conf/fileToHDFS.conf -n a1 -Dflume.root.logger=INFO,console