Hadoop HA搭建

• 准备环境
• • 修改静态IP
  • 关闭防火墙
  • 配置主机名、IP映射关系
  • 关闭防火墙
  • 配置SSH免密认证
  • 安装jdk
• Hadoop HA搭建
• • 解压安装Hadoop
  • 配置环境变量
  • 修改配置文件
• 启动HDFS HA
• Yarn
• • 修改配置文件

# 打开配置文件
vim /etc/sysconfig/network-scripts/ifcfg-ens33

# BOOTPROTO=dhcp
BOOTPROTO=static
# onBOOT=no
onBOOT=yes
# IP地址
IPADDR=10.192.210.3
# 子网掩码
NETMASK=255.255.254.0
# 网关
GATEWAY=10.192.210.1

# 重载所有ifcfg或route到connection（不会立即生效）
nmcli c reload
# 立即生效connection
nmcli c up ens33

[root@hadoop1 ~]# systemctl stop firewalld    # 关闭防火墙
[root@hadoop1 ~]# systemctl disable firewalld # 关闭防火墙开机自启动
## 安装vim/tree
```shell
[root@hadoop1 ~]# yum -y install vim
[root@hadoop1 ~]# yum -y install tree

[root@hadoop1 ~]# vim /etc/hostname	# 配置主机名
hadoop1
[root@hadoop1 ~]# vim /etc/hosts	# 配置IP映射关系
10.192.210.3 centos
[root@hadoop1 ~]# reboot			# 重启

# 在oracle官网下载jdk的tar文件
# 解压安装jdk
tar -zxf jdk-11.0.12_linux-x64_bin.tar.gz
# 配置环境变量（系统或用户,我配的是用户级别）
vim /etc/profile	# 系统
vim .bashrc	# 用户
JAVA_HOME=jdk1.8.0_291
HADOOP_HOME=hadoop-3.3.1
PATH=$PATH:$JAVA_HOME/bin:$HADOOP_HOME/bin:$HADOOP_HOME/sbin
export JAVA_HOME
export HADOOP_HOME
export PATH

tar -zxf hadoop-3.3.1.tar.gz

vim .bashrc
JAVA_HOME=/usr/java/latest
HADOOP_HOME=/usr/hadoop-2.9.2
PATH=$PATH:$JAVA_HOME/bin:$HADOOP_HOME/bin:$HADOOP_HOME/sbin
export JAVA_HOME
export PATH
export HADOOP_HOME

• core-site.xml

    
    
    
        fs.defaultFS
        hdfs://mycluster
    
    
    
    
        hadoop.tmp.dir
        /opt/HA/hadoop-2.8.4/data/tmp
    
    
        hadoop.tmp.dir
        /usr/local/hadoop-2.10.1/hadoop-${user.name}
    
    
    
	
		ha.zookeeper.quorum
		hadoop1:2181,hadoop2:2181,hadoop3:2181
	
    

• hdfs-site.xml

    
    
        dfs.ha.fencing.methods
        sshfence
    
    
    
        dfs.ha.fencing.ssh.private-key-files
        /root/.ssh/id_rsa
    

    
    
        dfs.ha.automatic-failover.enabled
        true
    
    
    
        dfs.client.failover.proxy.provider.mycluster
        org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider
    

    
    
        dfs.nameservices
        mycluster
    
    
    
        dfs.ha.namenodes.mycluster
        nn1,nn2
    
    
    
        dfs.namenode.rpc-address.mycluster.nn1
        hadoop1:9000
    
    
    
        dfs.namenode.http-address.mycluster.nn1
        hadoop1:50070
    
    
    
        dfs.namenode.rpc-address.mycluster.nn2
        hadoop2:9000
    
    
    
        dfs.namenode.http-address.mycluster.nn2
        hadoop2:50070
    

# 启动zookeeper
[root@hadoop1 ~]# zkServer.sh start
[root@hadoop2 ~]# zkServer.sh start
[root@hadoop3 ~]# zkServer.sh start
# 启动journalnode
[root@hadoop1 ~]# hadoop-daemon.sh start journalnode
[root@hadoop2 ~]# hadoop-daemon.sh start journalnode
[root@hadoop3 ~]# hadoop-daemon.sh start journalnode
# 格式化nn1，并启动
[root@hadoop1 ~]# hdfs namenode -format
[root@hadoop1 ~]# hadoop-daemon.sh start namenode
# 同步nn1元数据信息，并启动
[root@hadoop2 ~]# hdfs namenode -bootstrapStandby
[root@hadoop2 ~]# hadoop-daemon.sh start namenode
# 初始化HA在Zookeeper中状态
[root@hadoop1 ~]# hdfs zkfc -formatZK
# 启动HDFS HA
[root@hadoop1 ~]# hdfs zkfc -formatZK

• yarn-site.xml

    yarn.resourcemanager.ha.enabled
    true


    yarn.resourcemanager.cluster-id
    yarncluster


    yarn.resourcemanager.ha.rm-ids
    rm1,rm2


    yarn.resourcemanager.hostname.rm1
    hadoop1


    yarn.resourcemanager.hostname.rm2
    hadoop2


    yarn.resourcemanager.zk-address
    hadoop1:2181,hadoop2:2181,hadoop3:2181


    yarn.nodemanager.aux-services
    mapreduce_shuffle

• mapred-site.xml

    
        mapreduce.framework.name
        yarn