03.完全分布式搭建

• 虚拟机准备
• 免密登录配置
• 集群规划
• 端口变化：
• 相关配置
• 启动
• yarn测试
• 历史服务器
• xcall脚本
• 集群文件分发
• • scp和rsync区别：
  • rsync是否安装：
  • rsync语法：
• 脚本工具xsync
• 设置集群时钟同步

node111 node112 node113

ip修改：
	vim /etc/sysconfig/network-scripts/ifcfg-ens33
主机名修改：
	/etc/hostname 

生成密钥：
	ssh-keygen -t rsa 
		三个回车
拷贝密钥：
	ssh-copy-id node111
	ssh-copy-id node112
	ssh-copy-id node113

node111	NN DN RM NM
node112	DN NM
node113	2NN	NM

		hadoop2	hadoop3
NN: 	8020	9820
NNUI:	50070	9870 

hadoop-env.sh

export JAVA_HOME=/opt/softs/jdk1.8.0_212

core-site.xml

	
		fs.defaultFS
		hdfs://node111:9820
	
	 
	
		hadoop.tmp.dir
		/opt/softs/hadoop-3.1.3/datas
	
	
	
		hadoop.http.staticuser.user
		miao
	 
	
		hadoop.proxyuser.bduser.hosts
		*
	 
	
		hadoop.proxyuser.bduser.groups
		*
	 
	
		hadoop.proxyuser.bduser.users
		*
	

hdfs-site.xml

	
		dfs.replication
		3
	
	
	
		dfs.namenode.http-address
		node111:9870
	
	
	
		dfs.namenode.secondary.http-address
		node113:9868
	

workers

node111
node112
node113

yarn-env.sh

export JAVA_HOME=/opt/softs/jdk1.8.0_212

yarn-site.xml

	
		yarn.nodemanager.aux-services
		mapreduce_shuffle
	
	
	
		yarn.resourcemanager.hostname
		node111
	
	
	
		yarn.nodemanager.env-whitelist
		JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPA
	TH_PREPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_MAPRED_HOME
	
	
	
	
		yarn.scheduler.minimum-allocation-mb
		512
	 
	
	
		yarn.scheduler.maximum-allocation-mb
		2048
	
	
	
		yarn.nodemanager.vmem-pmem-ratio
		4
	
	
	
	
		yarn.nodemanager.resource.memory-mb
		2048
	
	
	
	
		yarn.nodemanager.pmem-check-enabled
		false
	
	
		yarn.nodemanager.vmem-check-enabled
		false
	

mapred-env.sh

export JAVA_HOME=/opt/softs/jdk1.8.0_212

mapred-site.xml

		mapreduce.framework.name
		yarn
	
	
		yarn.app.mapreduce.am.env
		HADOOP_MAPRED_HOME=${HADOOP_HOME}
	
	
	mapreduce.map.env
	HADOOP_MAPRED_HOME=${HADOOP_HOME}
	
	
	mapreduce.reduce.env
	HADOOP_MAPRED_HOME=${HADOOP_HOME}
	
	
		yarn.app.mapreduce.am.env
		HADOOP_MAPRED_HOME=${HADOOP_HOME}
	
	
		mapreduce.map.env
		HADOOP_MAPRED_HOME=${HADOOP_HOME}
	
	
		mapreduce.reduce.env
		HADOOP_MAPRED_HOME=${HADOOP_HOME}
	

hdfs namenode -format
start-dfs.sh
start-yarn.sh

[miao@node111 hadoop]$ yarn jar
/opt/softs/hadoop-3.1.3/share/hadoop/mapreduce/hadoop-mapreduce-examples-3.1.3.jar
pi 2 50

mapred-site.xml

	
		mapreduce.jobhistory.address
		node112:10020
	
	
	
		mapreduce.jobhistory.webapp.address
		node112:19888
	

ps:

对每个节点都执行相同的操作命令
cd /usr/local/bin
sudo touch xcall
sudo chmod 777 xcall
vi xcall

#!/bin/bash 
paramcount=$#
if [ $paramcount -eq 0 ]
then
	echo "no commond to exec!"
exit
fi 
for ((hostip=111;$hostip<=113;hostip++))
do
	echo "==========node$hostip==========="
	ssh node$hostip $*
done

rsync支持符号链接（软链接） 而scp不支持
rsync只对差异文件做更新。scp是把所有文件都复制过去

rpm -qa|grep rsync

rsync [-ravl] [$user@$host:]source [$user@$host:]dest
-r：递归文件夹
-a：归档
-v：显示复制过程
-l：拷贝符号连接

cd /usr/local/bin
sudo touch xsync
sudo chmod 777 xsync

#!/bin/bash 
if [ $# -lt 1 ]
then
  echo Not Enough Arguement!
  exit;
fi 
for host in node111 node112 node113
do
  echo ====================  $host  ==================== 
  for file in $@
  do 
	if [ -e $file ]
	then 
	  pdir=$(cd -P $(dirname $file); pwd) 
	  fname=$(basename $file)
	  ssh $host "mkdir -p $pdir"
	  rsync -av $pdir/$fname $host:$pdir
	else
	  echo $file does not exists!
	fi
  done
done