hudi集群环境搭建（hudi+hadoop+spark+zk+kafka）

一、集群环境配置 1.集群配置

hostname	slave1	slave2	slave3
ip	192.168.100.164	192.168.100.163	192.168.100.162
内存	16G	16G	8G
usernmae	root	root	root

安装常用工具

yum install -y epel-release
yum install -y net-tools 
yum install -y vim

2.集群常用脚本

用户bin目录下

2.1 集群分发脚本xsync

#!/bin/bash
#1. 判断参数个数
if [ $# -lt 1 ]
then
  echo Not Enough Arguement!
  exit;
fi
#2. 遍历集群所有机器
for host in slave1 slave2 slave3
do
#!/bin/bash
#1. 判断参数个数
if [ $# -lt 1 ]
then
  echo Not Enough Arguement!
  exit;
fi
#2. 遍历集群所有机器
for host in slave1 slave2 slave3
do
  echo ====================  $host  ====================
  #3. 遍历所有目录，挨个发送
  for file in $@
  do
    #4 判断文件是否存在
    if [ -e $file ]
    then
      #5. 获取父目录
      pdir=$(cd -P $(dirname $file); pwd)
      #6. 获取当前文件的名称
      fname=$(basename $file)
      ssh $host "mkdir -p $pdir"
      rsync -av $pdir/$fname $host:$pdir
    else
      echo $file does not exists!
    fi
  done
done

2.2 集群命令脚本xcall.sh

#! /bin/bash
 
for i in slave1 slave2 slave3
do
    echo --------- $i ----------
    ssh $i "$*"
done

2.3 群起hadoop集群脚本 hdp.sh

#!/bin/bash
if [ $# -lt 1 ]
then
    echo "No Args Input..."
    exit ;
fi
case $1 in
"start")
        echo " =================== 启动 hadoop集群 ==================="

        echo " --------------- 启动 hdfs ---------------"
        ssh slave1 "/opt/module/hadoop-2.7.3/sbin/start-dfs.sh"
        echo " --------------- 启动 yarn ---------------"
        ssh slave2 "/opt/module/hadoop-2.7.3/sbin/start-yarn.sh"
        echo " --------------- 启动 historyserver ---------------"
        ssh slave1 "/opt/module/hadoop-2.7.3/sbin/mr-jobhistory-daemon.sh start historyserver"
;;
"stop")
        echo " =================== 关闭 hadoop集群 ==================="

        echo " --------------- 关闭 historyserver ---------------"
        ssh slave1 "/opt/module/hadoop-2.7.3/sbin/mr-jobhistory-daemon.sh stop historyserverr"
        echo " --------------- 关闭 yarn ---------------"
        ssh slave2 "/opt/module/hadoop-2.7.3/sbin/stop-yarn.sh"
        echo " --------------- 关闭 hdfs ---------------"
        ssh slave1 "/opt/module/hadoop-2.7.3/sbin/stop-dfs.sh"
;;
*)
    echo "Input Args Error..."
;;
esac

2.4 群起zookeeper集群脚本zk.sh

#!/bin/bash

case $1 in
"start"){
	for i in slave1 slave2 slave3
	do
        echo ---------- zookeeper $i 启动 ------------
		ssh $i "/opt/module/zookeeper-3.4.6/bin/zkServer.sh start"
	done
};;
"stop"){
	for i in slave1 slave2 slave3
	do
        echo ---------- zookeeper $i 停止 ------------    
		ssh $i "/opt/module/zookeeper-3.4.6/bin/zkServer.sh stop"
	done
};;
"status"){
	for i in slave1 slave2 slave3
	do
        echo ---------- zookeeper $i 状态 ------------    
		ssh $i "/opt/module/zookeeper-3.4.6/bin/zkServer.sh status"
	done
};;
esac

2.5 群起kafka集群脚本kf.sh

#! /bin/bash

case $1 in
"start"){
    for i in slave1 slave2 slave3
    do
        echo " --------启动 $i Kafka-------"
        ssh $i "/opt/module/kafka_2.12-2.4.1/bin/kafka-server-start.sh -daemon /opt/module/kafka_2.12-2.4.1/config/server.properties"
    done
};;
"stop"){
    for i in slave1 slave2 slave3
    do
        echo " --------停止 $i Kafka-------"
        ssh $i "/opt/module/kafka_2.12-2.4.1/bin/kafka-server-stop.sh stop"
    done
};;
esac

3.环境配置

	slave1	slave2	slave3
HDFS	NameNode DataNode	DataNode	DataNode SecondaryNameNode
Yarn	NodeManager	Resourcemanager NodeManager	NodeManager
zk	zk	zk	zk
kafka	kafka	kafka	kafka

/opt/software ：软件压缩包

/opt/module ：解压后的软件

3.1 jdk和maven

vi /etc/profile.d/my_env.sh

#JAVA_HOME
export JAVA_HOME=/opt/module/jdk1.8.0_212
export PATH=$PATH:$JAVA_HOME/bin

#MAVEN_HOME
export MAVEN_HOME=/opt/module/maven-3.8.4
export PATH=$PATH:$MAVEN_HOME/bin

source /etc/profile.d/my_env.sh

3.2 hadoop2.7.3 3.2.1 HADOOP_HOME

vi /etc/profile.d/my_env.sh

#HADOOP_HOME
export HADOOP_HOME=/opt/module/hadoop-2.7.3
export HADOOP_CONF_DIR=$HADOOP_HOME/etc/hadoop
export HADOOP_COMMON_HOME=$HADOOP_HOME
export HADOOP_HDFS_HOME=$HADOOP_HOME
export HADOOP_YARN_HOME=$HADOOP_HOME
export HADOOP_MAPRED_HOME=$HADOOP_HOME
export PATH=$PATH:$HADOOP_HOME/bin:$HADOOP_HOME/sbin

source /etc/profile.d/my_env.sh

3.2.1 core-site.xml


	
    
        fs.defaultFS
        hdfs://slave1:8020


    
        hadoop.tmp.dir
        /opt/module/hadoop-2.7.3/data



    
        hadoop.http.staticuser.user
        root



    
        hadoop.proxyuser.root.hosts
        *


    
        hadoop.proxyuser.root.groups
        *


    
        hadoop.proxyuser.root.users
        *

3.2.2 hdfs-site.xml


	
	
        dfs.namenode.http-address
        slave1:9870
    
    
	
    
        dfs.namenode.secondary.http-address
        slave3:9868
    
    
    
    
        dfs.replication
        3

3.2.3 yarn-site.xml


	
    
        yarn.nodemanager.aux-services
        mapreduce_shuffle
    
    
    
    
        yarn.resourcemanager.hostname
        slave2
    
    
    
    
        yarn.nodemanager.env-whitelist
        JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_MAPRED_HOME
    
    
    
    
        yarn.scheduler.minimum-allocation-mb
        512
    
    
        yarn.scheduler.maximum-allocation-mb
        4096
    
    
    
    
        yarn.nodemanager.resource.memory-mb
        4096
    
    
    
    
        yarn.nodemanager.vmem-check-enabled
        false
    
    
    
        yarn.log-aggregation-enable
        true
    

    
      
        yarn.log.server.url  
        http://slave1:19888/jobhistory/logs
    

    
    
        yarn.log-aggregation.retain-seconds
        604800

3.2.4 mapred-site.xml


	
    
        mapreduce.framework.name
        yarn
    
    
	
   	 	mapreduce.jobhistory.address
   	 	slave1:10020
	

    
        mapreduce.jobhistory.webapp.address
        slave1:19888

3.2.5 workers

slave1
slave2
slave3

3.2.6 hadoop-env.sh

export HDFS_NAMENODE_USER=root
export HDFS_DATANODE_USER=root
export HDFS_SECONDARYNAMENODE_USER=root
export YARN_RESOURCEMANAGER_USER=root
export YARN_NODEMANAGER_USER=root

启动前格式化namenode

$HADOOP_HOME/bin/hdfs namenode -format

关闭防火墙

systemctl stop firewalld

在slave2开启节点均衡计划

$HADOOP_HOME/sbin/start-balancer.sh -threshold 10
stop-balancer.sh

3.2.7 启动测试

http://slave1:9870

http://slave2:8088

http://slave1:19888

3.3 hudi0.9

上传并解压hudi安装包

hudi测试启动

./hudi-cli/hudi-cli.sh

配置完进行集群分发

3.4 spark3.0.0 3.4.1 scala2.12.0

#SCALA_HOME
export SCALA_HOME=/opt/module/scala-2.12.10
export PATH=$PATH:$SCALA_HOME/bin

#SPARK_HOME
export SPARK_HOME=/opt/module/spark-3.0.0-bin-hadoop2.7
export PATH=$PATH:$SPARK_HOME/bin

3.4.2 spark_env.sh

export JAVA_HOME=/opt/module/jdk1.8.0_212
export SCALA_HOME=/opt/module/scala-2.12.10

3.4.3 测试启动

$SPARK_HOME/bin/spark-shell --master local[2]

3.4.4 spark集成hudi

1）上传相关jar包至/root/hudi-jars

2）启动spark

$SPARK_HOME/bin/spark-shell 
--master local[2] 
--jars /root/hudi-jars/hudi-spark3-bundle_2.12-0.9.0.jar,
/root/hudi-jars/spark_unused-1.0.0.jar,/root/hudi-jars/spark-avro_2.12-3.0.1.jar 
--conf "spark.serializer=org.apache.spark.serializer.KryoSerializer"

配置完进行集群分发

3.5 zookerper3.4.6

上传解压至/opt/module

3.5.1 环境变量

#ZOOKERPER_HOME
export ZOOKERPER_HOME=/opt/module/zookeeper-3.4.6
export PATH=$PATH:$ZOOKERPER_HOME/bin

3.5.2 配置服务器编号

zookeeper目录下

mkdir zkData
#在zkData目录内
vim myid
1

注意集群每个都需要配编号，分别为1、2、3

3.5.3 zoo.cfg

dataDir=/opt/module/zookeeper-3.4.6/zkData

server.1=slave1:2888:3888
server.2=slave2:2888:3888
server.3=slave3:2888:3888

配置完进行集群分发

3.5.4 测试

zk.sh start
zk.sh status

3.6 kafka2.12 3.6.1 环境变量

#KAFKA_HOME
export KAFKA_HOME=/opt/module/kafka_2.12-2.4.1
export PATH=$PATH:$KAFKA_HOME/bin

3.6.2 server.properties

kafka目录下

mkdir logs
vim server.properties

修改或者增加以下内容：
#broker的全局唯一编号，不能重复
broker.id=0
#删除topic功能使能
delete.topic.enable=true
#kafka运行日志存放的路径
log.dirs=/opt/module/kafka_2.12-2.4.1/data
#配置连接Zookeeper集群地址
zookeeper.connect=slave1,slave2,slave3:2181/kafka

注意修改其他服务器的broker.id

3.6.3启动测试

kf.sh start

hudi集群环境搭建（hudi+hadoop+spark+zk+kafka）

大数据系统相关栏目本月热门文章