一：hadoop3.1.3离线安装

环境初始化

前提准备
192.168.100.29 hadoop01
192.168.100.28 hadoop02
192.168.100.27 hadoop03
192.168.100.26 hadoop04
修改ip和主机名
#修改ip
vi /etc/sysconfig/network-scripts/ifcfg-ens192
IPADDR=“192.168.100.151”
#重启网卡服务
systemctl restart network
#修改主机名(新增的服务器)
hostnamectl set-hostname server19
#安装常用的基础工具
yum -y install sudo
yum install -y vim
#yum -y install lrzsz
yum -y install wget
yum -y install net-tools
#主节点编辑host文件
vim /etc/hosts
192.168.100.29 hadoop01 hadoop01
192.168.100.28 hadoop02 hadoop02
192.168.100.27 hadoop03 hadoop03
192.168.100.26 hadoop04 hadoop04
#配置免密
#集群服务器
cd /root/
ssh-keygen -t rsa
#然后三个回车
#复制主节点公钥到扩展的节点机器上(主节点上执行)
#ssh-copy-id -i /root/.ssh/id_rsa.pub root@ip
#主节点执行
ssh-copy-id -i /root/.ssh/id_rsa.pub root@hadoop01
ssh-copy-id -i /root/.ssh/id_rsa.pub root@hadoop02
ssh-copy-id -i /root/.ssh/id_rsa.pub root@hadoop03
#主节点上传文件分发脚本
资源如下
https://download.csdn.net/download/csdn_gyb/31849299
#主节点执行同步
sh /home/scp.sh /etc/hosts
#上传安装包到/tmp路径下
#关闭防火墙并查看状态(集群服务器)
systemctl stop firewalld
systemctl status firewalld

#安装ntp服务(所有服务器)为了集群时间同步
yum -y install ntp
service ntpd start
systemctl enable ntpd.service
#查看ntp服务状态
service ntpd status
#设置时间同步(从主节点同步)

安装mysql和jdk

#卸载清除主节点mysql
#先检查是否安装有mysql
rpm -qa|grep -i mysql
#停止mysql服务
systemctl stop mysqld
#删除软件命令
rpm -e –nodeps 包名
#如果上面命令提示依赖包错误，则使用以下命令尝试：
rpm -ev 包名 --nodeps
#查找卸载残留的mysql的目录
find / -name mysql #然后使用 rm -rf 将查询到的目录全部删除
#删除老版本mysql的配置文件
rm -rf /etc/my.cnf
rm -rf /etc/my.cnf.d
#mysql安装(见子节点)暂时先不安装
#安装jdk
#解压jdk到指定目录
tar -zxvf /tmp/jdk-8u181-linux-x64.tar.gz -C /usr/share/
对应路径如下(配置环境变量要用到该路径)
/usr/share/jdk1.8.0_181
#然后配置jdk环境变量(见子节点)
#然后把jdk目录复制到其他节点
sh /home/scp.sh /usr/share/jdk1.8.0_181
sh /home/scp.sh /etc/profile
安装hadoop及其配置

安装配置hadoop

#解压hadoop到指定路径
tar -zxvf /tmp/hadoop-3.1.3.tar.gz -C /home/
#到hadoop目录下建立目录
cd /home/hadoop-3.1.3
mkdir tmp
mkdir hdfs
mkdir hdfs/data
mkdir hdfs/name
#修改对应的配置文件(见子节点)
/home/hadoop-3.1.3/etc/hadoop/core-site.xml
/home/hadoop-3.1.3/etc/hadoop/hdfs-site.xml
/home/hadoop-3.1.3/etc/hadoop/yarn-site.xml
/home/hadoop-3.1.3/etc/hadoop/mapred-site.xml
/home/hadoop-3.1.3/etc/hadoop/yarn-env.sh
#配置workers
vim /home/hadoop-3.1.3/etc/hadoop/workers
#加入集群主机名称
hadoop01
hadoop02
hadoop03
hadoop04
#该文件夹分发到集群
sh scp.sh /home/hadoop-3.1.3/
#配置hadoop的环境变量
vi /etc/profile
文件末尾添加
export HADOOP_HOME=/home/hadoop-3.1.3
export PATH= P A T H : PATH: PATH:HADOOP_HOME/bin:$HADOOP_HOME/sbin
source /etc/profile
#修改hadoop-3.1.3/sbin/目录下的start-dfs.sh、start-yarn.sh文件，在文件头添加如下用户设置(对应的stop-dfs.sh、stop-yarn.sh也要修改，此处为hadoop3的特殊配置处，不修改，启动失败)。
start-dfs.sh
HDFS_DATANODE_USER=root
HDFS_DATANODE_SECURE_USER=hdfs
HDFS_NAMENODE_USER=root
HDFS_SECONDARYNAMENODE_USER=root
start-yarn.sh

YARN_RESOURCEMANAGER_USER=root
HADOOP_SECURE_DN_USER=yarn
YARN_NODEMANAGER_USER=root
对应配置如下
core-site.xml




    
        fs.defaultFS
        hdfs://hadoop04:9000
    
    
        io.file.buffer.size
        131072
    
    
        hadoop.tmp.dir
        /home/hadoop-3.1.3/tmp
    
    
        hadoop.proxyuser.root.hosts
        *
    
    
        hadoop.proxyuser.root.groups
        *
    
    
        hadoop.http.staticuser.user
        root

hdfs-site.xml





   
        dfs.namenode.name.dir
        /home/hadoop-3.1.3/hdfs/name
    
	
   
        dfs.datanode.data.dir
        /home/hadoop-3.1.3/hdfs/data
    
	
    
        dfs.replication
        3
    
	
    
        dfs.webhdfs.enabled
        true
    

    
        dfs.namenode.http-address
        hadoop04:9870
    

    
        dfs.namenode.secondary.http-address
        hadoop04:9001

yarn-site.xml





    
        yarn.nodemanager.aux-services
        mapreduce_shuffle
    
    
         yarn.resourcemanager.webapp.address
         hadoop04:8088
    
    
        yarn.resourcemanager.hostname
        hadoop04
    

  
    To enable RM web ui2 application.
    yarn.webapp.ui2.enable
    true
  
    
        yarn.nodemanager.env-whitelist
        JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_MAPRED_HOME
    
    
        yarn.scheduler.minimum-allocation-mb
        512
    
    
        yarn.scheduler.maximum-allocation-mb
        4096
    
    
        yarn.nodemanager.resource.memory-mb
        4096
    
    
        yarn.nodemanager.pmem-check-enabled
        false
    
    
        yarn.nodemanager.vmem-check-enabled
        false

mapred-site.xml





    
        mapreduce.framework.name
        yarn

yarn-env.sh

export JAVA_HOME=/usr/share/jdk1.8.0_181

hadoop-env.sh

export JAVA_HOME=/usr/share/jdk1.8.0_181
export HADOOP_HOME=/home/hadoop-3.1.3

start-dfs.sh，stop-dfs.sh&start-yarn.sh，stop-yarn.sh

在/hadoop/sbin路径下：
将start-dfs.sh，stop-dfs.sh两个文件顶部添加以下参数

#!/usr/bin/env bash
HDFS_DATANODE_USER=root
HADOOP_SECURE_DN_USER=hdfs
HDFS_NAMENODE_USER=root
HDFS_SECONDARYNAMENODE_USER=root

还有，start-yarn.sh，stop-yarn.sh顶部也需添加以下：
#!/usr/bin/env bash
YARN_RESOURCEMANAGER_USER=root
HADOOP_SECURE_DN_USER=yarn
YARN_NODEMANAGER_USER=root

一：hadoop3.1.3离线安装

大数据系统相关栏目本月热门文章