Hadoop双节点&Hive搭建自动化脚本

cat install_jdk-hadoop-hive.sh
#!/bin/bash
DIR=/usr/local/src
JDK_FILE="jdk-8u291-linux-x64.tar.gz"
JDK_DIR="/usr/local"

#hadoop node节点
HADOOPnode1=10.0.7.1
HADOOPnode2=10.0.7.2

#安装hadoop多节点时候，需要先在hadoop 2上执行，然后在在hadoop1上执行

HADOOP_FILE='hadoop-2.7.7.tar.gz'
HADOOP_DIR='/usr/local'

HIVE_FILE='apache-hive-2.3.9-bin.tar.gz'
HIVE_DIR='/usr/local'
MYSQLip=10.0.7.1
HIVEmima=hive

#安装hive 之前先操作mysql数据库,重新初始化时候必须清空hive库
#create user 'hive' identified by 'hive';
#grant all privileges on *.* to 'hive'@'%' with grant option;
#create database hive;
#alter database hive character set latin1;


color () {
    RES_COL=60
    MOVE_TO_COL="echo -en \033[${RES_COL}G"
    SETCOLOR_SUCCESS="echo -en \033[1;32m"
    SETCOLOR_FAILURE="echo -en \033[1;32m"
    SETCOLOR_WARNING="echo -en \033[1;33m"
    SETCOLOR_NORMAL="echo -en E[0m"
    echo -n "$2" && $MOVE_TO_COL
    echo -n "["
    if [ $1 = "success" -o $1 = "0" ] ;then
	    ${SETCOLOR_SUCCESS}
	    echo -n $" OK "
    elif [ $1 = "failure" -o $1 = "1" ] ; then
	    ${SETCOLOR_FAILURE}
	    echo -n $"FAILED"
    else
	    ${SETCOLOR_WARNING}
	    echo -n $"WARNING"
    fi
    ${SETCOLOR_NORMAL}
    echo -n "]"
    echo
}

initialize () {
sed -i '/SELINUX=enforcing/c SELINUX=disabled' /etc/selinux/config &&setenforce 0
systemctl disable --now firewalld
}

install_jdk () {
if ! [ -f "$DIR/$JDK_FILE" ] ;then
	color 1 "$JDK_FILE 文件不存在"
	exit;
elif [ -d $JDK_DIR/jdk ] ;then
	color 1 "JDK 已经安装"
        exit;
else  
	[ -d "$JDK_DIR" ] || mkdir -pv $JDK_DIR
fi

tar xvf $DIR/$JDK_FILE -C $JDK_DIR
cd $JDK_DIR && ln -s jdk1.8.* jdk

cat > /etc/profile.d/jdk.sh << EOF
export JAVA_HOME=$JDK_DIR/jdk
export JRE_HOME=$JAVA_HOME/jre
export CLASSPATH=$JAVA_HOME/lib/:$JRE_HOME/lib/
export PATH=$PATH:$JAVA_HOME/bin
EOF
source /etc/profile.d/jdk.sh
java -version && color 0 "JDK 安装完成" || { color 1 "JDK 安装失败" ; exit;}

sleep 3

source /etc/profile.d/jdk.sh
yum install -y rsync

echo $HADOOPnode1 hadoop1 >> /etc/hosts
echo $HADOOPnode2 hadoop2 >> /etc/hosts

}

install_hadoop2 () {
hostnamectl set-hostname hadoop2

if ! [ -f "$DIR/$HADOOP_FILE" ] ;then
        color 1 "$HADOOP_FILE 文件不存在"
        wget https://archive.apache.org/dist/hadoop/common/hadoop-2.7.7/hadoop-2.7.7.tar.gz
        exit;
elif [ -d $HADOOP_DIR/hadoop ] ;then
        color 1 "Hadoop 已经安装"
        exit;
else
        [ -d "$HADOOP_DIR" ] || mkdir -pv $HADOOP_DIR
fi

tar -xvf $DIR/$HADOOP_FILE -C $HADOOP_DIR

cd $HADOOP_DIR && ln -sv hadoop* hadoop

cat > /etc/profile.d/hadoop.sh << EOF
export HADOOP_HOME=$HADOOP_DIR/hadoop
export PATH=$PATH:$HADOOP_HOME/bin
export PATH=$PATH:$HADOOP_HOME/sbin
export HADOOP_CONF_DIR=${HADOOP_HOME}/etc/hadoop
EOF

source /etc/profile.d/hadoop.sh
hadoop version && color 0 "Hadoop2 安装完成" || { color 1 "Hadoop 安装失败" ; exit;}

sleep 3
source /etc/profile.d/hadoop.sh

}

install_hadoop1 () {

hostnamectl set-hostname hadoop2

if ! [ -f "$DIR/$HADOOP_FILE" ] ;then
	color 1 "$HADOOP_FILE 文件不存在"
        wget https://archive.apache.org/dist/hadoop/common/hadoop-2.7.7/hadoop-2.7.7.tar.gz
	exit;
elif [ -d $HADOOP_DIR/hadoop ] ;then
	color 1 "Hadoop 已经安装"
        exit;
else  
	[ -d "$HADOOP_DIR" ] || mkdir -pv $HADOOP_DIR
fi

tar -xvf $DIR/$HADOOP_FILE -C $HADOOP_DIR

cd $HADOOP_DIR && ln -sv hadoop* hadoop

cat > /etc/profile.d/hadoop.sh << EOF
export HADOOP_HOME=$HADOOP_DIR/hadoop
export PATH=$PATH:$HADOOP_HOME/bin
export PATH=$PATH:$HADOOP_HOME/sbin
export HADOOP_CONF_DIR=${HADOOP_HOME}/etc/hadoop
EOF

source /etc/profile.d/hadoop.sh

hadoop version && color 0 "Hadoop1 安装完成" || { color 1 "Hadoop 安装失败" ; exit;}

sleep 3
source /etc/profile.d/hadoop.sh
#配置hadoop环境脚本文件中的JAVA_HOME参数
mkdir $HADOOP_DIR/hadoop/tmp
cd $HADOOP_DIR/hadoop/etc/hadoop/

sed -i 's/export JAVA_HOME=${JAVA_HOME}/export JAVA_HOME="/usr/local/jdk"/' $HADOOP_DIR/hadoop/etc/hadoop/hadoop-env.sh

echo export JAVA_HOME="/usr/local/jdk" >>  $HADOOP_DIR/hadoop/etc/hadoop/mapred-env.sh
echo export JAVA_HOME="/usr/local/jdk" >>  $HADOOP_DIR/hadoop/etc/hadoop/yarn-env.sh

#修改hadoop配置文件

#core-site.xml
cat > $HADOOP_DIR/hadoop/etc/hadoop/core-site.xml << EOF






    fs.defaultFS
    hdfs://$HADOOPnode1:8020


    
    hadoop.tmp.dir
    $HADOOP_DIR/hadoop/tmp



    hadoop.proxyuser.root.hosts
    *


    hadoop.proxyuser.root.groups
    *



EOF
#hdfs-site.xml  hdfs文件系统模块设置

cat > $HADOOP_DIR/hadoop/etc/hadoop/hdfs-site.xml << EOF





  dfs.namenode.secondary.http-address
  $HADOOPnode2:9868




  dfs.webhdfs.enabled
  true



EOF
#mapred-site.xml #MapReduce模块设置
cat > $HADOOP_DIR/hadoop/etc/hadoop/mapred-site.xml << EOF




    mapreduce.framework.name
    yarn



    mapreduce.jobhistory.address
    $HADOOPnode2:10020



    mapreduce.jobhistory.webapp.address
    $HADOOPnode2:19888


EOF

#yarn-site.xml
cat > $HADOOP_DIR/hadoop/etc/hadoop/yarn-site.xml << EOF





    yarn.resourcemanager.hostname
    $HADOOPnode1



    yarn.nodemanager.aux-services
    mapreduce_shuffle



    yarn.log-aggregation-enable
    false


EOF
#在etc/hadoop/slaves增加子节点，使用IP地址即可
cd $HADOOP_DIR/hadoop/etc/hadoop/
cat > $HADOOP_DIR/hadoop/etc/hadoop/slaves << EOF
$HADOOPnode1
$HADOOPnode2
EOF

#依次分发配置文件到子节点

rsync -av $HADOOP_DIR/hadoop/etc/hadoop/ $HADOOPnode2:$HADOOP_DIR/hadoop/etc/hadoop/

sleep 5
#格式化NameNode
cd $HADOOP_DIR/hadoop/
./bin/hdfs namenode -format

#启动HDFS
$HADOOP_DIR/hadoop/sbin/start-dfs.sh
#启动YARN
$HADOOP_DIR/hadoop/sbin/start-yarn.sh

jps 
sleep 3
}

install_Hive () {
if ! [ -f "$DIR/$HIVE_FILE" ] ;then
        color 1 "$HIVE_FILE 文件不存在"
        wget https://mirror.tuna.tsinghua.edu.cn/apache/hive/hive-2.3.9/apache-hive-2.3.9-bin.tar.gz --no-check-certificate
        exit;
elif [ -d $HIVE_DIR/hive ] ;then
        color 1 "Hive 已经安装"
        exit;
else
        [ -d "$HIVE_DIR" ] || mkdir -pv $HIVE_DIR
fi
tar -xvf $DIR/$HIVE_FILE -C $HIVE_DIR
cd $HIVE_DIR && ln -sv apache-hive* hive

#下载mysql jar包，将jar包放在hive lib目录下
cd $HIVE_DIR/hive/lib/
wget https://mirror.tuna.tsinghua.edu.cn/mysql/downloads/Connector-J/mysql-connector-java-8.0.26.tar.gz --no-check-certificate

tar -xvf mysql-connector-java-8.0.26.tar.gz
mv mysql-connector-java-8.0.26/mysql-connector-java-8.0.26.jar .
rm -rf mysql-connector-java-8.0.26 mysql-connector-java-8.0.26.tar.gz
#升级hive中的guava依赖的hadoop版本
cd $HIVE_DIR/hive/lib/
mv guava*.jar guava.jar-bak
cd $HADOOP_DIR/hadoop/share/hadoop/hdfs/lib/

ln -sv $HADOOP_DIR/hadoop/share/hadoop/hdfs/lib/guava-*.jar $HIVE_DIR/hive/lib/

cat > /etc/profile.d/hive.sh << EOF
export HIVE_HOME=$HIVE_DIR/hive
export PATH=$PATH:$HIVE_HOME/bin
EOF
source /etc/profile.d/hive.sh

sleep 1

source /etc/profile.d/hive.sh

color 0 "Hive 安装完成"
#关闭hadoop安全模式
hdfs dfsadmin -safemode leave
#创建hive数仓目录
hadoop fs -mkdir -p $HIVE_DIR/hive/warehouse
hadoop fs -mkdir -p $HIVE_DIR/hive/tmp
#授权
hadoop fs -chmod 777 $HIVE_DIR/hive/warehouse
hadoop fs -chmod 777 $HIVE_DIR/hive/tmp
#检查这两个目录是否成功创建
hadoop fs -ls $HIVE_DIR/hive/

#配置hive环境
cd $HIVE_DIR/hive/conf/
cp hive-log4j2.properties.template hive-log4j2.properties
sed -i 's/property.hive.log.dir = ${sys:java.io.tmpdir}/${sys:user.name}/property.hive.log.dir = $HIVE_DIR/hive/log/' hive-log4j2.properties

cp hive-exec-log4j2.properties.template hive-exec-log4j2.properties
sed -i 's/property.hive.log.dir = ${sys:java.io.tmpdir}/${sys:user.name}/property.hive.log.dir =$HIVE_DIR/hive/exelog/' hive-exec-log4j2.properties

#配置hive-site文件
cd $HIVE_DIR/hive/conf
cat > $HIVE_DIR/hive/conf/hive-site.xml << EOF



    
    
        hive.exec.scratchdir
        $HIVE_DIR/hive/tmp
    
    
    
        hive.scratch.dir.permission
        733
    
    
    
        hive.metastore.warehouse.dir
        $HIVE_DIR/hive/warehouse
    
    
    
        javax.jdo.option.ConnectionURL
        jdbc:mysql://$MYSQLip:3306/hive?createDatabaseIfNotExist=true
    
    
    
        javax.jdo.option.ConnectionDriverName
        com.mysql.cj.jdbc.Driver
    
    
    
        javax.jdo.option.ConnectionUserName
        hive
    
    
    
        javax.jdo.option.ConnectionPassword
        $HIVEmima
    
    
    
        hive.cli.print.header
        true
    
    
    
        hive.cli.print.current.db
        true
    
    
        hive.server2.enable.doAs
        false
    
    
        hive.metastore.sasl.enabled
        false
    
    
        hive.server2.authenication
        NONE
    


EOF
#启动前先检查hadoop是否已经启动
#初始化Metastore元信息Schema
$HIVE_DIR/hive/bin/schematool -initSchema -dbType mysql

#后台启动hiveserver2
mkdir $HIVE_DIR/hive/log
nohup $HIVE_DIR/hive/bin/hiveserver2 >> $HIVE_DIR/hive/log/nohup.hs2.log 2>&1 &

#后台启动metastore
nohup hive --service metastore >> $HIVE_DIR/hive/log/metastore.log 2>&1 &

#查看hive的两个进程
ps -ef |grep hive

}

stop_hadoop () {
$HADOOP_DIR/hadoop/sbin/stop-dfs.sh
$HADOOP_DIR/hadoop/sbin/stop-yarn.sh
}

start_hadoop () {
$HADOOP_DIR/hadoop/sbin/start-dfs.sh
$HADOOP_DIR/hadoop/sbin/start-yarn.sh 
}

start_hive () {

nohup $HIVE_DIR/hive/bin/hiveserver2 >> $HIVE_DIR/hive/log/nohup.hs2.log 2>&1 &

nohup hive --service metastore >> $HIVE_DIR/hive/log/metastore.log 2>&1 &

}

#hadoop2上执行，把下面注释打开
#initialize
#install_jdk
#install_hadoop2

#hadoop1上执行，把下面注释打开
#initialize
#install_jdk
#install_hadoop1
#install_Hive

#启动、关停的命令
#hadoop
#stop_hadoop
#start_hadoop
#hive
#start_hive